[dpdk-dev] [dpdk-dev v6 1/4] cryptodev: add crypto data-path service APIs

[Fan Zhang <roy.fan.zhang@intel.com>]

This patch adds data-path service APIs for enqueue and dequeue
operations to cryptodev. The APIs support flexible user-define
enqueue and dequeue behaviors and operation mode.

Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
Signed-off-by: Piotr Bronowski <piotrx.bronowski@intel.com>
---
 lib/librte_cryptodev/rte_crypto.h             |   9 +
 lib/librte_cryptodev/rte_crypto_sym.h         |  44 ++-
 lib/librte_cryptodev/rte_cryptodev.c          |  45 +++
 lib/librte_cryptodev/rte_cryptodev.h          | 335 +++++++++++++++++-
 lib/librte_cryptodev/rte_cryptodev_pmd.h      |  47 ++-
 .../rte_cryptodev_version.map                 |  10 +
 6 files changed, 481 insertions(+), 9 deletions(-)

diff --git a/lib/librte_cryptodev/rte_crypto.h b/lib/librte_cryptodev/rte_crypto.h
index fd5ef3a87..f009be9af 100644
--- a/lib/librte_cryptodev/rte_crypto.h
+++ b/lib/librte_cryptodev/rte_crypto.h
@@ -438,6 +438,15 @@ rte_crypto_op_attach_asym_session(struct rte_crypto_op *op,
 	return 0;
 }
 
+/** Crypto data-path service types */
+enum rte_crypto_dp_service {
+	RTE_CRYPTO_DP_SYM_CIPHER_ONLY = 0,
+	RTE_CRYPTO_DP_SYM_AUTH_ONLY,
+	RTE_CRYPTO_DP_SYM_CHAIN,
+	RTE_CRYPTO_DP_SYM_AEAD,
+	RTE_CRYPTO_DP_N_SERVICE
+};
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_cryptodev/rte_crypto_sym.h b/lib/librte_cryptodev/rte_crypto_sym.h
index f29c98051..518e4111b 100644
--- a/lib/librte_cryptodev/rte_crypto_sym.h
+++ b/lib/librte_cryptodev/rte_crypto_sym.h
@@ -50,6 +50,18 @@ struct rte_crypto_sgl {
 	uint32_t num;
 };
 
+/**
+ * Crypto IO Data without length info.
+ * Supposed to be used to pass input/output data buffers with lengths
+ * defined when creating crypto session.
+ */
+struct rte_crypto_data {
+	/** virtual address of the data buffer */
+	void *base;
+	/** IOVA of the data buffer */
+	rte_iova_t iova;
+};
+
 /**
  * Synchronous operation descriptor.
  * Supposed to be used with CPU crypto API call.
@@ -57,12 +69,32 @@ struct rte_crypto_sgl {
 struct rte_crypto_sym_vec {
 	/** array of SGL vectors */
 	struct rte_crypto_sgl *sgl;
-	/** array of pointers to IV */
-	void **iv;
-	/** array of pointers to AAD */
-	void **aad;
-	/** array of pointers to digest */
-	void **digest;
+
+	union {
+
+		/* Supposed to be used with CPU crypto API call. */
+		struct {
+			/** array of pointers to IV */
+			void **iv;
+			/** array of pointers to AAD */
+			void **aad;
+			/** array of pointers to digest */
+			void **digest;
+		};
+
+		/* Supposed to be used with rte_cryptodev_dp_sym_submit_vec()
+		 * call.
+		 */
+		struct {
+			/** vector to IV */
+			struct rte_crypto_data *iv_vec;
+			/** vecor to AAD */
+			struct rte_crypto_data *aad_vec;
+			/** vector to Digest */
+			struct rte_crypto_data *digest_vec;
+		};
+	};
+
 	/**
 	 * array of statuses for each operation:
 	 *  - 0 on success
diff --git a/lib/librte_cryptodev/rte_cryptodev.c b/lib/librte_cryptodev/rte_cryptodev.c
index 1dd795bcb..8a28511f9 100644
--- a/lib/librte_cryptodev/rte_cryptodev.c
+++ b/lib/librte_cryptodev/rte_cryptodev.c
@@ -1914,6 +1914,51 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
 	return dev->dev_ops->sym_cpu_process(dev, sess, ofs, vec);
 }
 
+int
+rte_cryptodev_get_dp_service_ctx_data_size(uint8_t dev_id)
+{
+	struct rte_cryptodev *dev;
+	int32_t size = sizeof(struct rte_crypto_dp_service_ctx);
+	int32_t priv_size;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
+		return -1;
+
+	dev = rte_cryptodev_pmd_get_dev(dev_id);
+
+	if (*dev->dev_ops->get_drv_ctx_size == NULL ||
+		!(dev->feature_flags & RTE_CRYPTODEV_FF_DATA_PLANE_SERVICE)) {
+		return -1;
+	}
+
+	priv_size = (*dev->dev_ops->get_drv_ctx_size)(dev);
+	if (priv_size < 0)
+		return -1;
+
+	return RTE_ALIGN_CEIL((size + priv_size), 8);
+}
+
+int
+rte_cryptodev_dp_configure_service(uint8_t dev_id, uint16_t qp_id,
+	enum rte_crypto_dp_service service_type,
+	enum rte_crypto_op_sess_type sess_type,
+	union rte_cryptodev_session_ctx session_ctx,
+	struct rte_crypto_dp_service_ctx *ctx, uint8_t is_update)
+{
+	struct rte_cryptodev *dev;
+
+	if (!rte_cryptodev_get_qp_status(dev_id, qp_id))
+		return -1;
+
+	dev = rte_cryptodev_pmd_get_dev(dev_id);
+	if (!(dev->feature_flags & RTE_CRYPTODEV_FF_DATA_PLANE_SERVICE)
+			|| dev->dev_ops->configure_service == NULL)
+		return -1;
+
+	return (*dev->dev_ops->configure_service)(dev, qp_id, ctx,
+		service_type, sess_type, session_ctx, is_update);
+}
+
 /** Initialise rte_crypto_op mempool element */
 static void
 rte_crypto_op_init(struct rte_mempool *mempool,
diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h
index 7b3ebc20f..fdcfcbe14 100644
--- a/lib/librte_cryptodev/rte_cryptodev.h
+++ b/lib/librte_cryptodev/rte_cryptodev.h
@@ -466,7 +466,8 @@ rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
 /**< Support symmetric session-less operations */
 #define RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA		(1ULL << 23)
 /**< Support operations on data which is not byte aligned */
-
+#define RTE_CRYPTODEV_FF_DATA_PLANE_SERVICE		(1ULL << 24)
+/**< Support accelerated specific raw data as input */
 
 /**
  * Get the name of a crypto device feature flag
@@ -1351,6 +1352,338 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
 	struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs,
 	struct rte_crypto_sym_vec *vec);
 
+/**
+ * Get the size of the data-path service context for all registered drivers.
+ *
+ * @param	dev_id		The device identifier.
+ *
+ * @return
+ *   - If the device supports data-path service, return the context size.
+ *   - If the device does not support the data-plane service, return -1.
+ */
+__rte_experimental
+int
+rte_cryptodev_get_dp_service_ctx_data_size(uint8_t dev_id);
+
+/**
+ * Union of different crypto session types, including session-less xform
+ * pointer.
+ */
+union rte_cryptodev_session_ctx {
+	struct rte_cryptodev_sym_session *crypto_sess;
+	struct rte_crypto_sym_xform *xform;
+	struct rte_security_session *sec_sess;
+};
+
+/**
+ * Submit a data vector into device queue but the driver will not start
+ * processing until rte_cryptodev_dp_sym_submit_vec() is called.
+ *
+ * @param	qp		Driver specific queue pair data.
+ * @param	service_data	Driver specific service data.
+ * @param	vec		The array of job vectors.
+ * @param	ofs		Start and stop offsets for auth and cipher
+ *				operations.
+ * @param	opaque		The array of opaque data for dequeue.
+ * @return
+ *   - The number of jobs successfully submitted.
+ */
+typedef uint32_t (*cryptodev_dp_sym_submit_vec_t)(
+	void *qp, uint8_t *service_data, struct rte_crypto_sym_vec *vec,
+	union rte_crypto_sym_ofs ofs, void **opaque);
+
+/**
+ * Submit single job into device queue but the driver will not start
+ * processing until rte_cryptodev_dp_sym_submit_vec() is called.
+ *
+ * @param	qp		Driver specific queue pair data.
+ * @param	service_data	Driver specific service data.
+ * @param	data		The buffer vector.
+ * @param	n_data_vecs	Number of buffer vectors.
+ * @param	ofs		Start and stop offsets for auth and cipher
+ *				operations.
+ * @param	iv		IV data.
+ * @param	digest		Digest data.
+ * @param	aad		AAD data.
+ * @param	opaque		The opaque data for dequeue.
+ * @return
+ *   - On success return 0.
+ *   - On failure return negative integer.
+ */
+typedef int (*cryptodev_dp_submit_single_job_t)(
+	void *qp, uint8_t *service_data, struct rte_crypto_vec *data,
+	uint16_t n_data_vecs, union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_data *iv, struct rte_crypto_data *digest,
+	struct rte_crypto_data *aad, void *opaque);
+
+/**
+ * Inform the queue pair to start processing or finish dequeuing all
+ * submitted/dequeued jobs.
+ *
+ * @param	qp		Driver specific queue pair data.
+ * @param	service_data	Driver specific service data.
+ * @param	n		The total number of submitted jobs.
+ */
+typedef void (*cryptodev_dp_sym_opeartion_done_t)(void *qp,
+		uint8_t *service_data, uint32_t n);
+
+/**
+ * Typedef that the user provided for the driver to get the dequeue count.
+ * The function may return a fixed number or the number parsed from the opaque
+ * data stored in the first processed job.
+ *
+ * @param	opaque		Dequeued opaque data.
+ **/
+typedef uint32_t (*rte_cryptodev_get_dequeue_count_t)(void *opaque);
+
+/**
+ * Typedef that the user provided to deal with post dequeue operation, such
+ * as filling status.
+ *
+ * @param	opaque		Dequeued opaque data. In case
+ *				RTE_CRYPTO_HW_DP_FF_GET_OPAQUE_ARRAY bit is
+ *				set, this value will be the opaque data stored
+ *				in the specific processed jobs referenced by
+ *				index, otherwise it will be the opaque data
+ *				stored in the first processed job in the burst.
+ * @param	index		Index number of the processed job.
+ * @param	is_op_success	Driver filled operation status.
+ **/
+typedef void (*rte_cryptodev_post_dequeue_t)(void *opaque, uint32_t index,
+		uint8_t is_op_success);
+
+/**
+ * Dequeue symmetric crypto processing of user provided data.
+ *
+ * @param	qp			Driver specific queue pair data.
+ * @param	service_data		Driver specific service data.
+ * @param	get_dequeue_count	User provided callback function to
+ *					obtain dequeue count.
+ * @param	post_dequeue		User provided callback function to
+ *					post-process a dequeued operation.
+ * @param	out_opaque		Opaque pointer array to be retrieve from
+ *					device queue. In case of
+ *					*is_opaque_array* is set there should
+ *					be enough room to store all opaque data.
+ * @param	is_opaque_array		Set 1 if every dequeued job will be
+ *					written the opaque data into
+ *					*out_opaque* array.
+ * @param	n_success_jobs		Driver written value to specific the
+ *					total successful operations count.
+ *
+ * @return
+ *  - Returns number of dequeued packets.
+ */
+typedef uint32_t (*cryptodev_dp_sym_dequeue_t)(void *qp, uint8_t *service_data,
+	rte_cryptodev_get_dequeue_count_t get_dequeue_count,
+	rte_cryptodev_post_dequeue_t post_dequeue,
+	void **out_opaque, uint8_t is_opaque_array,
+	uint32_t *n_success_jobs);
+
+/**
+ * Dequeue symmetric crypto processing of user provided data.
+ *
+ * @param	qp			Driver specific queue pair data.
+ * @param	service_data		Driver specific service data.
+ * @param	out_opaque		Opaque pointer to be retrieve from
+ *					device queue.
+ *
+ * @return
+ *   - 1 if the job is dequeued and the operation is a success.
+ *   - 0 if the job is dequeued but the operation is failed.
+ *   - -1 if no job is dequeued.
+ */
+typedef int (*cryptodev_dp_sym_dequeue_single_job_t)(
+		void *qp, uint8_t *service_data, void **out_opaque);
+
+/**
+ * Context data for asynchronous crypto process.
+ */
+struct rte_crypto_dp_service_ctx {
+	void *qp_data;
+
+	union {
+		/* Supposed to be used for symmetric crypto service */
+		struct {
+			cryptodev_dp_submit_single_job_t submit_single_job;
+			cryptodev_dp_sym_submit_vec_t submit_vec;
+			cryptodev_dp_sym_opeartion_done_t submit_done;
+			cryptodev_dp_sym_dequeue_t dequeue_opaque;
+			cryptodev_dp_sym_dequeue_single_job_t dequeue_single;
+			cryptodev_dp_sym_opeartion_done_t dequeue_done;
+		};
+	};
+
+	/* Driver specific service data */
+	uint8_t drv_service_data[];
+};
+
+/**
+ * Configure one DP service context data. Calling this function for the first
+ * time the user should unset the *is_update* parameter and the driver will
+ * fill necessary operation data into ctx buffer. Only when
+ * rte_cryptodev_dp_submit_done() is called the data stored in the ctx buffer
+ * will not be effective.
+ *
+ * @param	dev_id		The device identifier.
+ * @param	qp_id		The index of the queue pair from which to
+ *				retrieve processed packets. The value must be
+ *				in the range [0, nb_queue_pair - 1] previously
+ *				supplied to rte_cryptodev_configure().
+ * @param	service_type	Type of the service requested.
+ * @param	sess_type	session type.
+ * @param	session_ctx	Session context data.
+ * @param	ctx		The data-path service context data.
+ * @param	is_update	Set 1 if ctx is pre-initialized but need
+ *				update to different service type or session,
+ *				but the rest driver data remains the same.
+ * @return
+ *   - On success return 0.
+ *   - On failure return negative integer.
+ */
+__rte_experimental
+int
+rte_cryptodev_dp_configure_service(uint8_t dev_id, uint16_t qp_id,
+	enum rte_crypto_dp_service service_type,
+	enum rte_crypto_op_sess_type sess_type,
+	union rte_cryptodev_session_ctx session_ctx,
+	struct rte_crypto_dp_service_ctx *ctx, uint8_t is_update);
+
+/**
+ * Submit single job into device queue but the driver will not start
+ * processing until rte_cryptodev_dp_sym_submit_vec() is called.
+ *
+ * @param	ctx		The initialized data-path service context data.
+ * @param	data		The buffer vector.
+ * @param	n_data_vecs	Number of buffer vectors.
+ * @param	ofs		Start and stop offsets for auth and cipher
+ *				operations.
+ * @param	iv		IV data.
+ * @param	digest		Digest data.
+ * @param	aad		AAD data.
+ * @param	opaque		The array of opaque data for dequeue.
+ * @return
+ *   - On success return 0.
+ *   - On failure return negative integer.
+ */
+__rte_experimental
+static __rte_always_inline int
+rte_cryptodev_dp_submit_single_job(struct rte_crypto_dp_service_ctx *ctx,
+		struct rte_crypto_vec *data, uint16_t n_data_vecs,
+		union rte_crypto_sym_ofs ofs,
+		struct rte_crypto_data *iv, struct rte_crypto_data *digest,
+		struct rte_crypto_data *aad, void *opaque)
+{
+	return (*ctx->submit_single_job)(ctx->qp_data, ctx->drv_service_data,
+		data, n_data_vecs, ofs, iv, digest, aad, opaque);
+}
+
+/**
+ * Submit a data vector into device queue but the driver will not start
+ * processing until rte_cryptodev_dp_sym_submit_vec() is called.
+ *
+ * @param	ctx	The initialized data-path service context data.
+ * @param	vec	The array of job vectors.
+ * @param	ofs	Start and stop offsets for auth and cipher operations.
+ * @param	opaque	The array of opaque data for dequeue.
+ * @return
+ *   - The number of jobs successfully submitted.
+ */
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_cryptodev_dp_sym_submit_vec(struct rte_crypto_dp_service_ctx *ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void **opaque)
+{
+	return (*ctx->submit_vec)(ctx->qp_data, ctx->drv_service_data, vec,
+			ofs, opaque);
+}
+
+/**
+ * Command the queue pair to start processing all submitted jobs from last
+ * rte_cryptodev_init_dp_service() call.
+ *
+ * @param	ctx	The initialized data-path service context data.
+ * @param	n		The total number of submitted jobs.
+ */
+__rte_experimental
+static __rte_always_inline void
+rte_cryptodev_dp_submit_done(struct rte_crypto_dp_service_ctx *ctx, uint32_t n)
+{
+	(*ctx->submit_done)(ctx->qp_data, ctx->drv_service_data, n);
+}
+
+/**
+ * Dequeue symmetric crypto processing of user provided data.
+ *
+ * @param	ctx			The initialized data-path service
+ *					context data.
+ * @param	get_dequeue_count	User provided callback function to
+ *					obtain dequeue count.
+ * @param	post_dequeue		User provided callback function to
+ *					post-process a dequeued operation.
+ * @param	out_opaque		Opaque pointer array to be retrieve from
+ *					device queue. In case of
+ *					*is_opaque_array* is set there should
+ *					be enough room to store all opaque data.
+ * @param	is_opaque_array		Set 1 if every dequeued job will be
+ *					written the opaque data into
+ *					*out_opaque* array.
+ * @param	n_success_jobs		Driver written value to specific the
+ *					total successful operations count.
+ *
+ * @return
+ *   - Returns number of dequeued packets.
+ */
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_cryptodev_dp_sym_dequeue(struct rte_crypto_dp_service_ctx *ctx,
+	rte_cryptodev_get_dequeue_count_t get_dequeue_count,
+	rte_cryptodev_post_dequeue_t post_dequeue,
+	void **out_opaque, uint8_t is_opaque_array,
+	uint32_t *n_success_jobs)
+{
+	return (*ctx->dequeue_opaque)(ctx->qp_data, ctx->drv_service_data,
+		get_dequeue_count, post_dequeue, out_opaque, is_opaque_array,
+		n_success_jobs);
+}
+
+/**
+ * Dequeue Single symmetric crypto processing of user provided data.
+ *
+ * @param	ctx			The initialized data-path service
+ *					context data.
+ * @param	out_opaque		Opaque pointer to be retrieve from
+ *					device queue. The driver shall support
+ *					NULL input of this parameter.
+ *
+ * @return
+ *   - 1 if the job is dequeued and the operation is a success.
+ *   - 0 if the job is dequeued but the operation is failed.
+ *   - -1 if no job is dequeued.
+ */
+__rte_experimental
+static __rte_always_inline int
+rte_cryptodev_dp_sym_dequeue_single_job(struct rte_crypto_dp_service_ctx *ctx,
+		void **out_opaque)
+{
+	return (*ctx->dequeue_single)(ctx->qp_data, ctx->drv_service_data,
+		out_opaque);
+}
+
+/**
+ * Inform the queue pair dequeue jobs finished.
+ *
+ * @param	ctx	The initialized data-path service context data.
+ * @param	n	The total number of jobs already dequeued.
+ */
+__rte_experimental
+static __rte_always_inline void
+rte_cryptodev_dp_dequeue_done(struct rte_crypto_dp_service_ctx *ctx, uint32_t n)
+{
+	(*ctx->dequeue_done)(ctx->qp_data, ctx->drv_service_data, n);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.h b/lib/librte_cryptodev/rte_cryptodev_pmd.h
index 81975d72b..bf0260c87 100644
--- a/lib/librte_cryptodev/rte_cryptodev_pmd.h
+++ b/lib/librte_cryptodev/rte_cryptodev_pmd.h
@@ -316,6 +316,41 @@ typedef uint32_t (*cryptodev_sym_cpu_crypto_process_t)
 	(struct rte_cryptodev *dev, struct rte_cryptodev_sym_session *sess,
 	union rte_crypto_sym_ofs ofs, struct rte_crypto_sym_vec *vec);
 
+/**
+ * Typedef that the driver provided to get service context private date size.
+ *
+ * @param	dev	Crypto device pointer.
+ *
+ * @return
+ *   - On success return the size of the device's service context private data.
+ *   - On failure return negative integer.
+ */
+typedef int (*cryptodev_dp_get_service_ctx_size_t)(
+	struct rte_cryptodev *dev);
+
+/**
+ * Typedef that the driver provided to configure data-path service.
+ *
+ * @param	dev		Crypto device pointer.
+ * @param	qp_id		Crypto device queue pair index.
+ * @param	ctx		The data-path service context data.
+ * @param	service_type	Type of the service requested.
+ * @param	sess_type	session type.
+ * @param	session_ctx	Session context data.
+ * @param	is_update	Set 1 if ctx is pre-initialized but need
+ *				update to different service type or session,
+ *				but the rest driver data remains the same.
+ *				buffer will always be one.
+ * @return
+ *   - On success return 0.
+ *   - On failure return negative integer.
+ */
+typedef int (*cryptodev_dp_configure_service_t)(
+	struct rte_cryptodev *dev, uint16_t qp_id,
+	struct rte_crypto_dp_service_ctx *ctx,
+	enum rte_crypto_dp_service service_type,
+	enum rte_crypto_op_sess_type sess_type,
+	union rte_cryptodev_session_ctx session_ctx, uint8_t is_update);
 
 /** Crypto device operations function pointer table */
 struct rte_cryptodev_ops {
@@ -348,8 +383,16 @@ struct rte_cryptodev_ops {
 	/**< Clear a Crypto sessions private data. */
 	cryptodev_asym_free_session_t asym_session_clear;
 	/**< Clear a Crypto sessions private data. */
-	cryptodev_sym_cpu_crypto_process_t sym_cpu_process;
-	/**< process input data synchronously (cpu-crypto). */
+	union {
+		cryptodev_sym_cpu_crypto_process_t sym_cpu_process;
+		/**< process input data synchronously (cpu-crypto). */
+		struct {
+			cryptodev_dp_get_service_ctx_size_t get_drv_ctx_size;
+			/**< Get data path service context data size. */
+			cryptodev_dp_configure_service_t configure_service;
+			/**< Initialize crypto service ctx data. */
+		};
+	};
 };
 
 
diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map
index 02f6dcf72..d384382d3 100644
--- a/lib/librte_cryptodev/rte_cryptodev_version.map
+++ b/lib/librte_cryptodev/rte_cryptodev_version.map
@@ -105,4 +105,14 @@ EXPERIMENTAL {
 
 	# added in 20.08
 	rte_cryptodev_get_qp_status;
+
+	# added in 20.11
+	rte_cryptodev_dp_configure_service;
+	rte_cryptodev_get_dp_service_ctx_data_size;
+	rte_cryptodev_dp_submit_single_job;
+	rte_cryptodev_dp_sym_submit_vec;
+	rte_cryptodev_dp_submit_done;
+	rte_cryptodev_dp_sym_dequeue;
+	rte_cryptodev_dp_sym_dequeue_single_job;
+	rte_cryptodev_dp_dequeue_done;
 };
-- 
2.20.1