[dpdk-dev] [PATCH v3 1/5] app: add muli process crypto application

[Arek Kusztal <arkadiuszx.kusztal@intel.com>]

This patch adds test application that can show
usage of cryptodev in multi process environment.
More can be found in mp_crypto.rst in tools guides.

Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
---
 app/Makefile                          |   1 +
 app/meson.build                       |   3 +-
 app/test-mp-crypto/Makefile           |  15 ++
 app/test-mp-crypto/main.c             | 169 ++++++++++++
 app/test-mp-crypto/meson.build        |   8 +
 app/test-mp-crypto/mp_crypto.c        |  26 ++
 app/test-mp-crypto/mp_crypto.h        | 178 ++++++++++++
 app/test-mp-crypto/mp_crypto_ipc.c    |  32 +++
 app/test-mp-crypto/mp_crypto_parser.c | 493 ++++++++++++++++++++++++++++++++++
 app/test-mp-crypto/mp_crypto_parser.h | 148 ++++++++++
 10 files changed, 1072 insertions(+), 1 deletion(-)
 create mode 100644 app/test-mp-crypto/Makefile
 create mode 100644 app/test-mp-crypto/main.c
 create mode 100644 app/test-mp-crypto/meson.build
 create mode 100644 app/test-mp-crypto/mp_crypto.c
 create mode 100644 app/test-mp-crypto/mp_crypto.h
 create mode 100644 app/test-mp-crypto/mp_crypto_ipc.c
 create mode 100644 app/test-mp-crypto/mp_crypto_parser.c
 create mode 100644 app/test-mp-crypto/mp_crypto_parser.h

diff --git a/app/Makefile b/app/Makefile
index 0392a7d..abacadf 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -13,6 +13,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_FIB) += test-fib
 DIRS-$(CONFIG_RTE_TEST_FLOW_PERF) += test-flow-perf
 DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test-pipeline
 DIRS-$(CONFIG_RTE_LIBRTE_IPSEC) += test-sad
+DIRS-$(CONFIG_RTE_LIBRTE_IPSEC) += test-mp-crypto
 
 ifeq ($(CONFIG_RTE_LIBRTE_BBDEV),y)
 DIRS-$(CONFIG_RTE_TEST_BBDEV) += test-bbdev
diff --git a/app/meson.build b/app/meson.build
index 585b908..d6ec4e8 100644
--- a/app/meson.build
+++ b/app/meson.build
@@ -18,7 +18,8 @@ apps = [
 	'test-flow-perf',
 	'test-pipeline',
 	'test-pmd',
-	'test-sad']
+	'test-sad',
+	'test-mp-crypto']
 
 # for BSD only
 lib_execinfo = cc.find_library('execinfo', required: false)
diff --git a/app/test-mp-crypto/Makefile b/app/test-mp-crypto/Makefile
new file mode 100644
index 0000000..9fc1f3c
--- /dev/null
+++ b/app/test-mp-crypto/Makefile
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2020 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+
+APP = dpdk-test-mp-crypto
+
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -O3
+
+# all source are stored in SRCS-y
+SRCS-y := main.c mp_crypto.c mp_crypto_parser.c mp_crypto_ipc.c
+
+include $(RTE_SDK)/mk/rte.app.mk
diff --git a/app/test-mp-crypto/main.c b/app/test-mp-crypto/main.c
new file mode 100644
index 0000000..ce150b5
--- /dev/null
+++ b/app/test-mp-crypto/main.c
@@ -0,0 +1,169 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include <rte_hexdump.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev.h>
+#include <rte_cycles.h>
+#include <rte_atomic.h>
+#include <signal.h>
+#include <inttypes.h>
+
+#include "mp_crypto_parser.h"
+#include "mp_crypto.h"
+
+static void sigkill_handler(int __rte_unused sig,
+				siginfo_t *siginfo __rte_unused,
+				void *context __rte_unused)
+{
+	mp_crypto_exit_flag = 1;
+	printf("\nInterrupted, finalizing...");
+}
+
+static int
+mp_app_init(int argc, char *argv[])
+{
+	/* init EAL */
+	int ret = rte_eal_init(argc, argv)
+;
+	if (ret < 0)
+		rte_exit(-1, "Invalid EAL arguments!\n");
+
+	argc -= ret;
+	argv += ret;
+
+	struct sigaction sigkill_action;
+
+	memset(&sigkill_action, 0, sizeof(sigkill_action));
+	sigkill_action.sa_sigaction = sigkill_handler;
+	sigkill_action.sa_flags = SA_SIGINFO;
+
+	if (sigaction(SIGINT, &sigkill_action, NULL) < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot init sigation");
+		return -1;
+	}
+
+	if (get_options(argc, argv) != 0) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Get cmdln options returned an error\n");
+		return -1;
+	};
+
+	/* Set driver id for this process */
+	mp_app_driver_id =
+		rte_cryptodev_driver_id_get(mp_app_params->devtype_name);
+	MP_APP_LOG(INFO, COL_BLUE, "- Setting driver %d for this process",
+		mp_app_driver_id);
+
+	/* Register IPC and allocate memzones */
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		MP_APP_LOG_2(INFO, COL_NORM, "- Starting PRIMARY process");
+		if (rte_mp_action_register(MP_APP_IPC_NAME,
+			mp_crypto_primary_handler)) {
+			RTE_LOG(ERR, USER1, "Cannot register IPC callback");
+			return -1;
+		}
+		/* Setup memzone for shared data */
+		mp_app_process_mz = rte_memzone_reserve(MP_APP_PROC_SHARED_NAME,
+				sizeof(struct mp_app_process_data), 0, 0);
+		if (mp_app_process_mz == NULL) {
+			RTE_LOG(ERR, USER1,
+				"%s: cannot create memzone for process",
+				__func__);
+			return -1;
+		}
+		mp_shared_data = mp_app_process_mz->addr;
+		rte_spinlock_init(&mp_shared_data->sessions.lock);
+	} else {
+		MP_APP_LOG_2(INFO, COL_NORM, "- Starting SECONDARY process");
+		if (rte_mp_action_register(MP_APP_IPC_NAME,
+			mp_crypto_secondary_handler)) {
+			RTE_LOG(ERR, USER1, "Cannot register IPC callback");
+			return -1;
+		}
+		/* Setup memzone for shared data */
+		mp_app_process_mz =
+			rte_memzone_lookup(MP_APP_PROC_SHARED_NAME);
+		if (mp_app_process_mz == NULL) {
+			MP_APP_LOG(ERR, COL_RED,
+				"Cannot find memzone by name %s",
+			MP_APP_PROC_SHARED_NAME);
+			return -1;
+		}
+		mp_shared_data = mp_app_process_mz->addr;
+	}
+
+	mp_shared_data->proc_counter++;
+	mp_shared_data->proc_counter_total++;
+	MP_APP_LOG(INFO, COL_GREEN, "Number of processes = %d",
+		mp_shared_data->proc_counter);
+
+	return 0;
+}
+
+void mp_crypto_exit_app(void)
+{
+	const int timeout = 10;
+	int counter = 0;
+	struct rte_mp_msg icp_msg;
+
+	memset(&icp_msg, 0, sizeof(MP_APP_IPC_NAME));
+	mp_crypto_exit_flag = 1;
+	if (mp_shared_data == NULL)
+		return;
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		/* Inform of exit intention,
+		 * wait until all processes finish
+		 */
+
+		memcpy(icp_msg.name, MP_APP_IPC_NAME, sizeof(MP_APP_IPC_NAME));
+		memcpy(icp_msg.param, PRIMARY_PROC_EXIT,
+			sizeof(PRIMARY_PROC_EXIT));
+		icp_msg.len_param = sizeof(PRIMARY_PROC_EXIT);
+		icp_msg.num_fds = 0;
+		if (rte_mp_sendmsg(&icp_msg) < 0) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Error when sending IPC to secondary processes");
+			return;
+		}
+		while (mp_shared_data->proc_counter > 1 && counter++
+				< timeout) {
+			rte_delay_ms(1000);
+			MP_APP_LOG(INFO, COL_NORM,
+			"Waiting for %d out of %d seconds", counter, timeout);
+		}
+		if (counter < timeout) {
+			MP_APP_LOG_2(INFO, COL_GREEN,
+			"All secondary processes exited normally");
+		} else {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"One or more processes did not exit normally");
+		}
+
+		mp_shared_data->proc_counter = 0;
+	} else {
+		/* Inform primary of exiting */
+		mp_shared_data->proc_counter--;
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	if (mp_app_init(argc, argv) < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Error when initializing");
+		goto err;
+	};
+
+	mp_crypto_exit_app();
+	return 0;
+err:
+	mp_crypto_exit_app();
+
+	return 1;
+}
diff --git a/app/test-mp-crypto/meson.build b/app/test-mp-crypto/meson.build
new file mode 100644
index 0000000..12a6d49
--- /dev/null
+++ b/app/test-mp-crypto/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2020 Intel Corporation
+
+sources = files('mp_crypto.c',
+		'mp_crypto_parser.c',
+		'mp_crypto_ipc.c',
+		'main.c')
+deps += ['cryptodev']
\ No newline at end of file
diff --git a/app/test-mp-crypto/mp_crypto.c b/app/test-mp-crypto/mp_crypto.c
new file mode 100644
index 0000000..3437397
--- /dev/null
+++ b/app/test-mp-crypto/mp_crypto.c
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#include "mp_crypto.h"
+#include "mp_crypto_parser.h"
+
+int			mp_app_driver_id;
+/* Global driver id, one per mp_app */
+int			mp_app_device_id;
+/* For now we use only one device type, so for session
+ * init only one need to be provided
+ */
+struct mp_app_dev	mp_app_devs[MP_APP_MAX_DEVS];
+/* Global devices list */
+uint16_t		mp_app_devs_cnt;
+/* Global device counter */
+uint8_t			mp_app_max_queues;
+/* Per process queue counter */
+const struct rte_memzone *mp_app_process_mz;
+struct mp_app_process_data *mp_shared_data;
+/* Data shared across processes
+ * memzone name = MP_PROC_SHARED_MZ
+ */
+
+int mp_crypto_exit_flag;
+/* Global exit flag */
diff --git a/app/test-mp-crypto/mp_crypto.h b/app/test-mp-crypto/mp_crypto.h
new file mode 100644
index 0000000..da89501
--- /dev/null
+++ b/app/test-mp-crypto/mp_crypto.h
@@ -0,0 +1,178 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#ifndef _MP_CRYPTO_SAMPLE_APP_
+#define _MP_CRYPTO_SAMPLE_APP_
+
+#include <stdint.h>
+#include <rte_hexdump.h>
+#include <rte_cryptodev.h>
+
+/* Intel QuickAssist Technology Symmetric service PMD name */
+#define CRYPTODEV_NAME_QAT_SYM_PMD	"crypto_qat"
+/* Maximum number of devices to configure with this app */
+#define MP_APP_MAX_DEVS			64
+/* Maximum number of queue pairs per device */
+#define MP_APP_QUEUE_PAIRS_NUM		8
+
+#define MP_APP_PROC_SHARED_NAME		"MP_PROC_SHARED_MZ"
+/* Memzone name for shared data across processes */
+#define MP_APP_IPC_NAME			"MP_APP_IPC_NAME"
+
+/* Session pool information */
+#define MP_APP_SESSION_POOL_NAME	"MP_APP_SESSION_POOL_NAME"
+#define MP_APP_PRIV_SESSION_POOL_NAME	"MP_APP_PRIV_SESSPOL_NAME"
+
+#define MP_APP_SESSION_POOL_NAME_LOC		"MP_APP_SESSP_NAME_LOC"
+#define MP_APP_PRIV_SESSION_POOL_NAME_LOC	"MP_APP_PRIV_SPOL_NLOC"
+
+#define MAX_NUM_OF_SESSIONS		(16)
+
+/* Crypto op information */
+#define MP_APP_CRYPTO_OP_POOL_NAME	"MP_APP_OP_NAME"
+/* Mbuf information */
+#define MP_APP_MBUFPOOL_NAME		"MP_APP_MBUF_NAME"
+
+extern int mp_crypto_exit_flag;
+/* Global exit flag */
+
+/*
+ * IPC COMMANDS
+ */
+#define PRIMARY_PROC_EXIT		"PRIMARY_EXIT"
+#define SECONDARY_PROC_EXIT		"SECONDARY_EXIT"
+
+#define MP_APP_DEV_NAME_LEN	64
+/* Max name length */
+
+/* Op pool constants */
+#define MP_APP_NUM_MBUFS					(4096)
+/* Same number as default/max ops */
+#define MP_APP_MBUF_CACHE_SIZE				(256)
+#define MP_APP_DEFAULT_NUM_XFORMS			(2)
+#define MP_APP_MAXIMUM_IV_LENGTH			(16)
+/* Mbuf constants */
+#define MP_APP_MBUF_SIZE			(sizeof(struct rte_mbuf) + \
+		RTE_PKTMBUF_HEADROOM + MBUF_DATAPAYLOAD_SIZE)
+/* qps constants */
+#define MP_CRYPTO_QP_DESC_NUM		(4096)
+#define NP_CRYPTO_OPS_TO_ENQ		(160000)
+#define NP_CRYPTO_OPS_TO_DEQ		(160000)
+/* Enqueue constants */
+#define MP_CRYPTO_BURST_NUM		(64)
+#define MP_CRYPTO_OPS_NUM		(MP_APP_NUM_MBUFS)
+/* Device information */
+#define MP_CRYPTO_MAX_DEVS		(64)
+
+extern struct rte_crypto_op *mp_crypto_ops[];
+/* Per process set of rte crypto ops */
+extern struct rte_crypto_op *mp_crypto_ops_ret[];
+/* Per process set of return rte crypto ops */
+extern struct rte_mbuf *mp_crypto_mbufs[];
+/* Per process set of rte mbufs */
+
+/* Name of the device */
+struct mp_app_dev_name {
+	char name[MP_APP_DEV_NAME_LEN];
+};
+
+extern struct rte_cryptodev_sym_session *mp_crypto_local_sessions[];
+/* Array of private sessions */
+
+/* Symmetric session + ref count*/
+struct mp_app_shared_sym_session {
+	struct rte_cryptodev_sym_session *session;
+	/* Pointer to symmetric session */
+	int refcnt;
+	/* Reference count, process that created this session
+	 * does not increment this value
+	 */
+};
+
+/* Data for session array to be shared */
+struct mp_app_session_array {
+	struct mp_app_shared_sym_session sym_sessions[MAX_NUM_OF_SESSIONS];
+	/* Array of pointers to sessions */
+	int sym_session_counter;
+	/* Counter of allocated sessions */
+	rte_spinlock_t lock;
+	/* Spinlock guarding this array */
+};
+
+/* Data to be shared across processes */
+struct mp_app_process_data {
+	uint16_t proc_counter;
+	/* Counter of processes */
+	uint16_t proc_counter_total;
+	/* Number of processes that joined, not decremented
+	 * can be used for naming in particular processes
+	 */
+	uint16_t devices_number;
+	/* Number of devices probed by primary process */
+	struct mp_app_dev_name prim_dev_name[MP_APP_MAX_DEVS];
+	/* Names of devices probed by primary process */
+	struct mp_app_session_array sessions;
+	/* Array of sessions to be visible by all processes */
+};
+
+extern const struct rte_memzone *mp_app_process_mz;
+extern struct mp_app_process_data *mp_shared_data;
+/* Data shared across processes
+ * memzone name = MP_PROC_SHARED_MZ
+ */
+
+struct mp_app_dev {
+	int8_t id;
+	/* Cryptodev id of this dev */
+	int queue_pair_flag[MP_APP_QUEUE_PAIRS_NUM];
+	/* 1 means qp was configured for this device,
+	 * 0 not configured by this process, but still
+	 * could be initialized by another
+	 * -2 means this qp is to be configured
+	 */
+	uint16_t max_queue_pairs;
+	/* Per device info */
+	uint8_t probed;
+	/* If device was probed by EAL */
+	uint8_t configured;
+	/* Was this device configured */
+	const struct rte_memzone *shared_data;
+	/* This data is shared across processes
+	 * memzone name = MZ_DEV_SHARED_DATA_DEV_[ID]
+	 */
+};
+
+extern int			mp_app_driver_id;
+/* Global driver id, one per mp_app */
+extern int			mp_app_device_id;
+/* For now we use only one device type, so for session
+ * init only one need to be provided
+ */
+extern struct mp_app_dev	mp_app_devs[];
+/* Global devices list */
+extern uint16_t			mp_app_devs_cnt;
+/* Global device counter */
+extern uint8_t			mp_app_max_queues;
+/* Per process queue counter */
+
+void mp_crypto_exit_app(void);
+/* Exit function for both primary and secondary */
+
+/*
+ * Primary process IPC handler
+ */
+int
+mp_crypto_primary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer);
+int
+mp_crypto_secondary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer);
+
+#define IV_OFFSET			(sizeof(struct rte_crypto_op) + \
+		sizeof(struct rte_crypto_sym_op) + DEFAULT_NUM_XFORMS * \
+		sizeof(struct rte_crypto_sym_xform))
+
+#define MBUF_DATAPAYLOAD_SIZE		(2048)
+#define DEFAULT_NUM_XFORMS			(2)
+
+#endif
diff --git a/app/test-mp-crypto/mp_crypto_ipc.c b/app/test-mp-crypto/mp_crypto_ipc.c
new file mode 100644
index 0000000..9d5a8cb
--- /dev/null
+++ b/app/test-mp-crypto/mp_crypto_ipc.c
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#include "mp_crypto.h"
+
+/*
+ * Primary process IPC handler
+ */
+int
+mp_crypto_primary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer)
+{
+	(void)peer;
+	if (!memcmp(SECONDARY_PROC_EXIT, (const char *)mp_msg->param,
+		sizeof(SECONDARY_PROC_EXIT))) {
+		RTE_LOG(ERR, USER1, "One of secondary processes exiting...");
+	}
+	return 0;
+}
+
+int
+mp_crypto_secondary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer)
+{
+	(void)peer;
+	if (!memcmp(PRIMARY_PROC_EXIT, (const char *)mp_msg->param,
+		sizeof(PRIMARY_PROC_EXIT)))	{
+		RTE_LOG(ERR, USER1, "Primary process exiting...");
+		mp_crypto_exit_flag = 1;
+	}
+	return 0;
+}
diff --git a/app/test-mp-crypto/mp_crypto_parser.c b/app/test-mp-crypto/mp_crypto_parser.c
new file mode 100644
index 0000000..8edae17
--- /dev/null
+++ b/app/test-mp-crypto/mp_crypto_parser.c
@@ -0,0 +1,493 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include <getopt.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <errno.h>
+
+#include <rte_string_fns.h>
+#include "mp_crypto_parser.h"
+#include "mp_crypto.h"
+
+struct mp_crypto_app_parameters *mp_app_params;
+
+static void
+usage(char *progname)
+{
+	/* TODO, find better way of formatting columns... */
+	printf("%s [EAL options] -- [options]"
+	"\noptions:"
+	"\n  --devtype [device name]: \t\t\tdevice name, the same name need to be used"
+	" across all processes.\n\t\t\t\t\t\t--Example: --devtype=crypto_qat"
+	"\n  --config-dev [dev_id,]: \t\t\tid of device that should be"
+	" configured by this process. Note that order of ids depends on the"
+	" Cryptodev\n\t\t\t\t\t\tglobal array placement so BDF of smaller numbers will come"
+	" first.\n\t\t\t\t\t\t--Example: -w 03:01.2 -w 03:01.1 -w 03:01.3 --config-dev 0,2"
+	" will configure devices 03:01.1 and 03:01.3."
+	"\n  --qp-config=[dev_id]:[qp_id,];...: \t\tqueue_pairs qp_id's to be configured dev_id's"
+	"\n\t\t\t\t\t\t--Example: --qp-config=0:0,1;1:1;0:1; - will configure qp's 0,1 on device 0"
+	"' 1 on device 1, 0,1 on device 2.'"
+	"\n  --session-mask=[mask]\t\t\t\tsession to be shared for all processes, session list is in"
+	" mp_crypto_vectors.c file.\n\t\t\t\t\t\tIf session mask will not be set it still can be configured"
+	" interactively by user for certain process and the used by this process only"
+	"\n\t\t\t\t\t\t--Example --sesion-mask=0x3 will configure session 0 and 1."
+	"\n  --enq=[dev_id]:[qp_id]:[ops]:[vector_id]:\tEnqueue operation for this process"
+	"\n\t\t\t\t\t\t- dev_id: device selected the same way as in --config-dev option"
+	"\n\t\t\t\t\t\t- qp_id: queue pair to bu used for enqueue operation"
+	"\n\t\t\t\t\t\t- ops: 0 means it will run in infinite loop (ctrl-c will inform other processes),"
+	"other than that any positive number"
+	"\n\t\t\t\t\t\t- vector_id: vector id to be used, vector array can be found"
+	" in mp_crypto_vectors.c file. "
+	"\n\t\t\t\t\t\t- Only one can be specified by process"
+	"\n  --deq=[dev_id]:[qp_id]:[ops]:[vector_id]:\tDequeue operation for this process"
+	"\n\t\t\t\t\t\t- dev_id: device selected the same way as in --config-dev option"
+	"\n\t\t\t\t\t\t- qp_id: queue pair to bu used for dequeue operation"
+	"\n\t\t\t\t\t\t- ops: 0 means it will run in infinite loop (ctrl-c will inform other processes),"
+	"other than that any positive number"
+	"\n\t\t\t\t\t\t- vector_id: vector id to be used, vector array can be found"
+	" in mp_crypto_vectors.c file. "
+	"\n\t\t\t\t\t\t- Only one can be specified by process"
+	"\n  --print-stats: \t\t\t\tPrint stats at then end of program."
+	"\n",
+	progname);
+}
+
+static struct option lgopts[] = {
+	{ MP_DEV_CONFIGURE, required_argument, 0, 0 },
+	{ MP_QP_CONFIGURE, required_argument, 0, 0 },
+	{ MP_ENQ, required_argument, 0, 0 },
+	{ MP_DEQ, required_argument, 0, 0 },
+	{ MP_SESSION_MASK, required_argument, 0, 0 },
+	{ MP_PRINT_STATS, 0, 0, 0 },
+	{ MP_DEVTYPE_NAME, required_argument, 0, 0 },
+	{ NULL, 0, 0, 0 }
+};
+
+int16_t
+get_options(int argc, char *argv[])
+{
+	mp_app_params = rte_zmalloc_socket(NULL,
+					sizeof(struct mp_crypto_app_parameters),
+					0, rte_socket_id());
+
+	if (mp_app_params == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Failed to allocate for test data\n");
+		return -1;
+	}
+
+	options_default(mp_app_params);
+
+	if (options_parse(mp_app_params, argc, argv) != 0) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Parsing one or more user options failed");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+parse_config_dev(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	char *end = NULL;
+	const char *start = arg;
+	uint64_t num;
+	char str[32];
+
+	while (1) {
+		memset(str, 0, sizeof(str));
+		end = strchr(start, ',');
+		if (end) {
+			memcpy(str, start, end - start);
+			errno = 0;
+			num = strtoull(str, NULL, 10);
+			if (errno) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Invalid device provided '%s'", str);
+				return -1;
+			}
+			if (num >= MP_CRYPTO_MAX_DEVS) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Device number not supported %"PRIu64"", num);
+				return -1;
+			}
+			/* Sanity check, unfortunately c standard does not
+			 * force errno to be set when no conversion
+			 * can by performed (except for ERANGE)
+			 */
+			if (num == 0) {
+				if (start[0] != '0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+				if (start[1] != ',') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+			}
+			mp_params->dev_to_configure_mask |= 1LU << (num);
+			start = end + 1;
+			if (*start == 0)
+				break;
+		} else {
+			end = strchr(start, '\0');
+			memcpy(str, start, end - start);
+			errno = 0;
+			num = strtoull(str, NULL, 10);
+			if (errno) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Invalid device provided '%s'", str);
+				return -1;
+			}
+			if (num >= 64) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Device number not supported %"PRIu64"", num);
+				return -1;
+			}
+			/* Sanity check, unfortunately c standard does not force
+			 * errno to be set when no conversion can by performed
+			 * (except for ERANGE)
+			 */
+			if (num == 0) {
+				if (start[0] != '0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+				if (start[1] != '\0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+			}
+			mp_params->dev_to_configure_mask |= 1LU << (num);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* Veeeery simple parser */
+static int mp_parse_qps(const char *arg)
+{
+	char str[64] = { };
+	int dev_id = -1;
+	const char *start = arg;
+	const char *end;
+	int finish = 0;
+
+	while (1) {
+		end = strchr(start, ':');
+		if (end == NULL)
+			return 0;
+		memcpy(str, start, end - start);
+		dev_id = strtol(str, NULL, 10);
+		start = end + 1;
+		if (*start == '\0') {
+			MP_APP_LOG_2(ERR, COL_RED,
+				"Parsing queue pairs: error parsing");
+			return -1;
+		}
+		const char *curr = start;
+
+		while (1) {
+			memset(str, 0, sizeof(str));
+			if (*curr == ',' || *curr == ';' || *curr == '\0') {
+				memcpy(str, start, curr - start);
+				int qp_id = strtol(str, NULL, 10);
+
+				if (qp_id > (MP_APP_QUEUE_PAIRS_NUM - 1)) {
+					MP_APP_LOG(WARNING, COL_YEL,
+					"Cannot create qp: %d, maximum qp number allowed %d (%d queues)",
+					qp_id, MP_APP_QUEUE_PAIRS_NUM - 1,
+					MP_APP_QUEUE_PAIRS_NUM);
+				}
+
+				mp_app_devs[dev_id].queue_pair_flag[qp_id] =
+						QP_TO_CONFIGURE;
+			}
+			if (*curr == ',') {
+				start = curr + 1;
+				curr++;
+				continue;
+			} else if (*curr == ';') {
+				start = curr + 1;
+				break;
+			} else if (*curr == '\0') {
+				finish = 1;
+				break;
+			}
+			curr++;
+		}
+		if (finish)
+			break;
+	}
+
+	return 0;
+}
+
+static int
+parse_qp_config(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	strncpy(mp_params->qp_config, arg, MP_APP_QP_PARAM_LEN - 1);
+	if (mp_parse_qps(arg)) {
+		MP_APP_LOG_2(ERR, COL_RED, "- Parsing error, qpairs string");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+parse_enq(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	char str[64] = { };
+	const char *start = arg;
+	/* dev id */
+	char *end = strchr(start, ':');
+	int i = 0;
+
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.dev_id = strtol(str, NULL, 10);
+	/* qp id */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.qp_id = strtol(str, NULL, 10);
+	/* ops no */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.ops_no = strtol(str, NULL, 10);
+	/* vector ids */
+	start = end + 1;
+	while ((end = strchr(start, ',')) != NULL) {
+		memset(str, 0, sizeof(str));
+		memcpy(str, start, end - start);
+		mp_params->enq_param.vector_number[i] = strtoul(str, NULL, 0);
+		start = end + 1;
+		i++;
+	}
+	if (i == 0)
+		goto err;
+
+	MP_APP_LOG(INFO, COL_BLUE, "Run enqueue on device %d",
+			mp_params->enq_param.dev_id);
+	MP_APP_LOG(INFO, COL_BLUE, "Run enqueue on qp %d",
+			mp_params->enq_param.qp_id);
+	i = 0;
+	while (mp_params->enq_param.vector_number[i] > 0 &&
+			i < MP_APP_MAX_VECTORS)	{
+		MP_APP_LOG(INFO, COL_BLUE, "Run enqueue vector %d",
+			mp_params->enq_param.vector_number[i]);
+		i++;
+	}
+
+	mp_params->enq_param.checkpoint = 1000000;
+
+	return 0;
+err:
+	MP_APP_LOG_2(ERR, COL_RED, "Error parsing enq");
+	return -1;
+}
+
+static int
+parse_deq(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	char str[64] = { };
+	const char *start = arg;
+	/* Dev id */
+	char *end = strchr(start, ':');
+	int i = 0;
+
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.dev_id = strtol(str, NULL, 10);
+	/* qp id */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.qp_id = strtol(str, NULL, 10);
+	/* ops no */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.ops_no = strtol(str, NULL, 10);
+
+	/* vector no */
+	start = end + 1;
+	while ((end = strchr(start, ',')) != NULL) {
+		memset(str, 0, sizeof(str));
+		memcpy(str, start, end - start);
+		mp_params->deq_param.vector_number[i] = strtoul(str, NULL, 0);
+		start = end + 1;
+		i++;
+	}
+	if (i == 0)
+		goto err;
+
+	MP_APP_LOG(INFO, COL_BLUE, "Run dequeue on device %d",
+			mp_params->deq_param.dev_id);
+	MP_APP_LOG(INFO, COL_BLUE, "Run dequeue on qp %d",
+			mp_params->deq_param.qp_id);
+	i = 0;
+	while (mp_params->deq_param.vector_number[i] > 0 &&
+			i < MP_APP_MAX_VECTORS)	{
+		MP_APP_LOG(INFO, COL_BLUE, "Run dequeue vector %d",
+				mp_params->deq_param.vector_number[i]);
+		i++;
+	}
+
+	mp_params->deq_param.checkpoint = 1000000;
+
+	return 0;
+err:
+	MP_APP_LOG_2(ERR, COL_RED, "Error parsing deq");
+	return -1;
+}
+
+static int
+parse_print_stats(struct mp_crypto_app_parameters *mp_params,
+			const char *arg __rte_unused)
+{
+	mp_params->print_stats = 1;
+	return 0;
+}
+
+static int
+parse_session_mask(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	char *end = NULL;
+
+	mp_params->session_mask = strtoull(arg, &end, 16);
+
+	return 0;
+}
+
+static int
+parse_devtype(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	if (arg == NULL) {
+		RTE_LOG(ERR, USER1, "--%s param argument is null\n",
+			MP_DEVTYPE_NAME);
+	}
+
+	if (strlen(arg) > (sizeof(mp_params->devtype_name) - 1)) {
+		RTE_LOG(ERR, USER1, "--%s different lengths\n",
+			MP_DEVTYPE_NAME);
+		return 0;
+	}
+
+	strlcpy(mp_params->devtype_name, arg,
+			sizeof(mp_params->devtype_name));
+
+	return 0;
+};
+
+typedef int (*option_parser_t)(struct mp_crypto_app_parameters
+			*mp_params,	const char *arg);
+
+struct long_opt_parser {
+	const char *lgopt_name;
+	option_parser_t parser_fn;
+};
+
+static int
+opts_parse_long(int opt_idx, struct mp_crypto_app_parameters *mp_params)
+{
+	struct long_opt_parser parsermap[] = {
+		{ MP_DEV_CONFIGURE, parse_config_dev },
+		{ MP_QP_CONFIGURE, parse_qp_config },
+		{ MP_ENQ, parse_enq },
+		{ MP_DEQ, parse_deq },
+		{ MP_PRINT_STATS, parse_print_stats },
+		{ MP_SESSION_MASK, parse_session_mask },
+		{ MP_DEVTYPE_NAME, parse_devtype },
+	};
+	unsigned int i;
+
+	for (i = 0; i < RTE_DIM(parsermap); i++) {
+		if (strncmp(lgopts[opt_idx].name, parsermap[i].lgopt_name,
+				strlen(lgopts[opt_idx].name)) == 0) {
+			return parsermap[i].parser_fn(mp_params, optarg);
+		}
+	}
+
+	return 0;
+}
+
+int
+options_parse(struct mp_crypto_app_parameters *mp_params,
+					int argc, char **argv)
+{
+	int opt, retval;
+	int opt_idx;
+
+	while ((opt = getopt_long(argc, argv, "h", lgopts, &opt_idx))
+			!= EOF) {
+		switch (opt) {
+		case 'h':
+			usage(argv[0]);
+			rte_exit(0, "Select options as above.\n");
+			break;
+		case 0:
+			retval = opts_parse_long(opt_idx, mp_params);
+			if (retval != 0)
+				return retval;
+			break;
+		default:
+			RTE_LOG(ERR, USER1, "Parse error after %s\n",
+					lgopts[opt_idx].name);
+			usage(argv[0]);
+			return 0;
+		}
+	}
+
+	return 0;
+}
+
+void
+options_default(struct mp_crypto_app_parameters *mp_params)
+{
+	int i = 0;
+
+	for (i = 0; i < MP_APP_MAX_VECTORS; i++) {
+		mp_params->enq_param.dev_id = -1;
+		mp_params->enq_param.qp_id = -1;
+		mp_params->enq_param.vector_number[i] = -1;
+		mp_params->deq_param.dev_id = -1;
+		mp_params->deq_param.qp_id = -1;
+		mp_params->deq_param.vector_number[i] = -1;
+	}
+
+	mp_params->enq_param.ops_no = 0;
+	mp_params->deq_param.ops_no = 0;
+	mp_params->print_stats = 0;
+}
diff --git a/app/test-mp-crypto/mp_crypto_parser.h b/app/test-mp-crypto/mp_crypto_parser.h
new file mode 100644
index 0000000..cf35e09
--- /dev/null
+++ b/app/test-mp-crypto/mp_crypto_parser.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#ifndef _MP_CRYPTO_SAMPLE_APP_PARSER_
+#define _MP_CRYPTO_SAMPLE_APP_PARSER_
+
+#include <rte_hexdump.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev.h>
+
+/* Make debug colorful! */
+#define COL_NORM	"\x1B[0m"
+#define COL_WHITE	"\x1B[37m"
+#define COL_RED		"\x1B[31m"
+#define COL_GREEN	"\x1B[32m"
+#define COL_YEL		"\x1B[33m"
+#define COL_BLUE	"\x1B[34m"
+#define COL_MAG		"\x1B[35m"
+
+#define MP_APP_LOG(level, color, str, args...) \
+	do {		\
+		printf("%s", color);			\
+		RTE_LOG(level, USER1, str, args);	\
+		printf("%s\n", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_LOG_2(level, color, str) \
+	do {		\
+		printf("%s", color);			\
+		RTE_LOG(level, USER1, str);	\
+		printf("%s\n", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_LOG_NO_RET(level, color, str, args...) \
+	do {		\
+		printf("\r%s", color);			\
+		RTE_LOG(level, USER1, str, args);	\
+		printf("%s", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_QP_PARAM_LEN		(64 * 4)
+#define MP_APP_ENQ_PARAM_LEN	1024
+
+#define EMPTY_FLAGS		0
+
+#define MP_DEVTYPE_NAME		("devtype")
+#define MP_DEV_CONFIGURE	("config-dev")
+#define MP_QP_CONFIGURE		("qp-config")
+#define MP_ENQ				("enq")
+#define MP_DEQ				("deq")
+#define MP_SESSION_MASK		("session-mask")
+#define MP_PRINT_STATS		("print-stats")
+
+#define MP_APP_MAX_VECTORS	64
+
+extern const char *comp_perf_test_type_strs[];
+/* Command line parameters */
+extern struct mp_crypto_app_parameters *mp_app_params;
+/* Parser params */
+
+static const char livesign_print_char[4] = { '-', '\\', '|', '/'};
+
+int16_t
+get_options(int argc, char *argv[]);
+
+struct mp_crypto_app_enqdeq {
+	int dev_id;
+	int qp_id;
+	int vector_number[MP_APP_MAX_VECTORS];
+	int ops_no;
+	int checkpoint;
+};
+
+#define QP_TO_CONFIGURE		(-2)
+
+struct mp_crypto_app_parameters {
+	char devtype_name[RTE_DEV_NAME_MAX_LEN];
+	/* Driver to be used in this process */
+	char qp_config[MP_APP_QP_PARAM_LEN];
+	/* Queue Pairs configuration per device in process
+	 * in format q0,q1;q0,q1;, '-' means queue pair will not
+	 * be configured
+	 * Example: queue_pairs="0,1;0,-;-,1;" means that
+	 * device 0 will configure queue pairs 0 and 1,
+	 * device 1 will configure queue pairs 0
+	 * device 2 will configure queue pairs 1
+	 * Devices are order dependent
+	 */
+	char flow_config[MP_APP_ENQ_PARAM_LEN];
+	/* Enqueue configuration per process
+	 * Format "[dev_id]=qp_id:[op,]
+	 * Example: [0]=0:[enq, deq];[1]=0:[enq]
+	 * Mean that for this process qp 0 on device 0 will be
+	 * enqueuing and dequeuing in one queue pair,
+	 * meanwhile device 0 will only enqueue data on qpair 0.
+	 * Other process can then dequeue this data with
+	 * [1]=0:[deq]
+	 */
+	uint64_t dev_to_configure_mask;
+	/* Devices to configure, uint64 bitmask
+	 * 1 means dev 0, 2 dev 1, 4 dev... etc
+	 */
+	uint64_t session_mask;
+	/* Session to be created by this process,
+	 * if session was already created this step will be ommited.
+	 * Usage: session-mask=0x6 -> create session number 1 and 2.
+	 * Number of session refer to predefined array of sessions
+	 */
+	char enq[MP_APP_ENQ_PARAM_LEN];
+	struct mp_crypto_app_enqdeq enq_param;
+	char deq[MP_APP_ENQ_PARAM_LEN];
+	struct mp_crypto_app_enqdeq deq_param;
+	/* Enqueue/dequeue string used by this process.
+	 * Usage: [dev_id]:[qp_id]:[crypto_vector],[crypto_vector]...
+	 * Example 2:1:0,1,2, -> device no 2 on qp 1 enqueues ops from
+	 * vectors 0, 1, 2 .note ',' comma needs to be put after last arg
+	 */
+	int print_stats;
+	/* Print stats on the end on flow function */
+
+	uint16_t qp_id;
+	uint16_t waiting_qp_id;
+
+	int16_t configure_device;
+	int16_t setup_qp;
+	int16_t create_session_pool;
+	int16_t create_op_pool;
+	int16_t init_sessions;
+	int16_t build_ops;
+	int16_t dequeue;
+	int16_t enqueue;
+	int16_t dump_mempools;
+};
+
+int
+options_parse(struct mp_crypto_app_parameters *mp_params, int argc,
+			char **argv);
+void
+options_default(struct mp_crypto_app_parameters *mp_params);
+
+int
+options_check(struct mp_crypto_app_parameters *mp_params);
+
+#endif
-- 
2.1.0