[PATCH v2 00/11] fixes and improvements to cnxk crypto PMD

[PATCH v2 00/11] fixes and improvements to cnxk crypto PMD

[Tejasree Kondoj]

This series adds cn10k scatter gather support and improvements
to cnxk crypto PMD.

v2:
* Fixed clang build failure

Anoob Joseph (2):
  crypto/cnxk: use version field directly
  common/cnxk: ensure flush inval completion with CSR read

Gowrishankar Muthukrishnan (2):
  common/cnxk: fix incorrect auth key length
  crypto/cnxk: fix order of ECFPM params

Tejasree Kondoj (7):
  crypto/cnxk: make sg version check const
  crypto/cnxk: use direct mode for zero aad length
  crypto/cnxk: set ctx for AE
  common/cnxk: add errata function for CPT set ctx
  common/cnxk: replace CPT revision check with caps
  crypto/cnxk: support cn10k IPsec SG mode
  crypto/cnxk: add model check for pdcp chain

 drivers/common/cnxk/hw/cpt.h              |  14 +-
 drivers/common/cnxk/roc_cpt.c             |  16 ++
 drivers/common/cnxk/roc_errata.h          |   9 +
 drivers/common/cnxk/roc_se.h              |   7 +-
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c |  67 +++----
 drivers/crypto/cnxk/cn10k_ipsec_la_ops.h  | 222 ++++++++++++++++++++--
 drivers/crypto/cnxk/cn9k_cryptodev_ops.c  |  20 +-
 drivers/crypto/cnxk/cn9k_ipsec_la_ops.h   |   4 +-
 drivers/crypto/cnxk/cnxk_ae.h             |  69 +++++--
 drivers/crypto/cnxk/cnxk_cryptodev_ops.c  |  44 +++--
 drivers/crypto/cnxk/cnxk_se.h             |   6 +-
 drivers/crypto/cnxk/cnxk_sg.h             |  23 +++
 drivers/event/cnxk/cn10k_eventdev.c       |   3 +-
 13 files changed, 389 insertions(+), 115 deletions(-)

-- 
2.25.1

[PATCH v2 01/11] common/cnxk: fix incorrect auth key length

[Tejasree Kondoj]

From: Gowrishankar Muthukrishnan <gmuthukrishn@marvell.com>

Auth key length is stored as 8 bit value in SE context. It should
be larger enough to accommodate supported auth key length of 1024
bytes maximum, as in HMAC.

Fixes: a45859312ff ("common/cnxk: add SE definitions for symmetric crypto")

Signed-off-by: Gowrishankar Muthukrishnan <gmuthukrishn@marvell.com>
---
 drivers/common/cnxk/roc_se.h | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h
index 6758142214..a0c97b26c5 100644
--- a/drivers/common/cnxk/roc_se.h
+++ b/drivers/common/cnxk/roc_se.h
@@ -288,16 +288,15 @@ struct roc_se_ctx {
 	uint64_t enc_cipher : 8;
 	uint64_t hash_type : 8;
 	uint64_t mac_len : 8;
-	uint64_t auth_key_len : 8;
+	uint64_t auth_key_len : 16;
 	uint64_t fc_type : 4;
 	uint64_t hmac : 1;
 	uint64_t zsk_flags : 3;
 	uint64_t k_ecb : 1;
 	uint64_t pdcp_ci_alg : 2;
 	uint64_t pdcp_auth_alg : 2;
-	uint16_t ciph_then_auth : 1;
-	uint16_t auth_then_ciph : 1;
-	uint64_t rsvd : 17;
+	uint64_t ciph_then_auth : 1;
+	uint64_t auth_then_ciph : 1;
 	union cpt_inst_w4 template_w4;
 	/* Below fields are accessed by hardware */
 	struct se_ctx_s {
-- 
2.25.1

[PATCH v2 02/11] crypto/cnxk: make sg version check const

[Tejasree Kondoj]

Remove sg_ver2 from burst structure and make it as
const argument for compiler optimized code.

Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 26 +++++++++++------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 4ee6b944c4..92f7002db9 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -38,7 +38,6 @@ struct ops_burst {
 	struct cn10k_sso_hws *ws;
 	struct cnxk_cpt_qp *qp;
 	uint16_t nb_ops;
-	bool is_sg_ver2;
 };
 
 /* Holds information required to send vector of operations */
@@ -489,7 +488,8 @@ cn10k_cpt_vec_submit(struct vec_request vec_tbl[], uint16_t vec_tbl_len, struct
 }
 
 static inline int
-ca_lmtst_vec_submit(struct ops_burst *burst, struct vec_request vec_tbl[], uint16_t *vec_tbl_len)
+ca_lmtst_vec_submit(struct ops_burst *burst, struct vec_request vec_tbl[], uint16_t *vec_tbl_len,
+		    const bool is_sg_ver2)
 {
 	struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP];
 	uint64_t lmt_base, lmt_arg, io_addr;
@@ -537,7 +537,7 @@ ca_lmtst_vec_submit(struct ops_burst *burst, struct vec_request vec_tbl[], uint1
 		infl_req = infl_reqs[i];
 		infl_req->op_flags = 0;
 
-		ret = cn10k_cpt_fill_inst(qp, &burst->op[i], inst, infl_req, burst->is_sg_ver2);
+		ret = cn10k_cpt_fill_inst(qp, &burst->op[i], inst, infl_req, is_sg_ver2);
 		if (unlikely(ret != 1)) {
 			plt_cpt_dbg("Could not process op: %p", burst->op[i]);
 			if (i != 0)
@@ -620,7 +620,7 @@ next_op:;
 }
 
 static inline uint16_t
-ca_lmtst_burst_submit(struct ops_burst *burst)
+ca_lmtst_burst_submit(struct ops_burst *burst, const bool is_sg_ver2)
 {
 	struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP];
 	uint64_t lmt_base, lmt_arg, io_addr;
@@ -660,7 +660,7 @@ ca_lmtst_burst_submit(struct ops_burst *burst)
 		infl_req = infl_reqs[i];
 		infl_req->op_flags = 0;
 
-		ret = cn10k_cpt_fill_inst(qp, &burst->op[i], inst, infl_req, burst->is_sg_ver2);
+		ret = cn10k_cpt_fill_inst(qp, &burst->op[i], inst, infl_req, is_sg_ver2);
 		if (unlikely(ret != 1)) {
 			plt_dp_dbg("Could not process op: %p", burst->op[i]);
 			if (i != 0)
@@ -729,7 +729,6 @@ cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_ev
 	burst.ws = ws;
 	burst.qp = NULL;
 	burst.nb_ops = 0;
-	burst.is_sg_ver2 = is_sg_ver2;
 
 	for (i = 0; i < nb_events; i++) {
 		op = ev[i].event_ptr;
@@ -743,8 +742,8 @@ cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_ev
 		if (qp != burst.qp) {
 			if (burst.nb_ops) {
 				if (is_vector) {
-					submitted =
-						ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len);
+					submitted = ca_lmtst_vec_submit(&burst, vec_tbl,
+									&vec_tbl_len, is_sg_ver2);
 					/*
 					 * Vector submission is required on qp change, but not in
 					 * other cases, since we could send several vectors per
@@ -753,7 +752,7 @@ cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_ev
 					cn10k_cpt_vec_submit(vec_tbl, vec_tbl_len, burst.qp);
 					vec_tbl_len = 0;
 				} else {
-					submitted = ca_lmtst_burst_submit(&burst);
+					submitted = ca_lmtst_burst_submit(&burst, is_sg_ver2);
 				}
 				count += submitted;
 				if (unlikely(submitted != burst.nb_ops))
@@ -769,9 +768,10 @@ cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_ev
 		/* Max nb_ops per burst check */
 		if (++burst.nb_ops == PKTS_PER_LOOP) {
 			if (is_vector)
-				submitted = ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len);
+				submitted = ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len,
+								is_sg_ver2);
 			else
-				submitted = ca_lmtst_burst_submit(&burst);
+				submitted = ca_lmtst_burst_submit(&burst, is_sg_ver2);
 			count += submitted;
 			if (unlikely(submitted != burst.nb_ops))
 				goto vec_submit;
@@ -781,9 +781,9 @@ cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_ev
 	/* Submit the rest of crypto operations */
 	if (burst.nb_ops) {
 		if (is_vector)
-			count += ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len);
+			count += ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len, is_sg_ver2);
 		else
-			count += ca_lmtst_burst_submit(&burst);
+			count += ca_lmtst_burst_submit(&burst, is_sg_ver2);
 	}
 
 vec_submit:
-- 
2.25.1

[PATCH v2 03/11] crypto/cnxk: use version field directly

[Tejasree Kondoj]

From: Anoob Joseph <anoobj@marvell.com>

As version field is available in rte_ip_hdr, use it directly instead of
masking version_ihl.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
 drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 4 ++--
 drivers/crypto/cnxk/cn9k_ipsec_la_ops.h  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 3a07842e4b..11541b6ab9 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -546,10 +546,10 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,
 			}
 		}
 
-		if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == IPVERSION) {
+		if (ip->version == IPVERSION) {
 			m_len = rte_be_to_cpu_16(ip->total_length);
 		} else {
-			PLT_ASSERT(((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == 6);
+			PLT_ASSERT((ip->version == 6));
 			ip6 = (struct rte_ipv6_hdr *)ip;
 			m_len = rte_be_to_cpu_16(ip6->payload_len) + sizeof(struct rte_ipv6_hdr);
 		}
diff --git a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
index 9df41bf65d..85aacb803f 100644
--- a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
@@ -28,13 +28,13 @@ ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_m
 		uintptr_t data = (uintptr_t)m_src->buf_addr + m_src->data_off;
 		struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)data;
 
-		if (unlikely(((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) != IPVERSION)) {
+		if (unlikely(ip->version != IPVERSION)) {
 			struct rte_ipv6_hdr *ip6 = (struct rte_ipv6_hdr *)ip;
 			uint8_t *nxt_hdr = (uint8_t *)ip6;
 			uint8_t dest_op_cnt = 0;
 			int nh = ip6->proto;
 
-			PLT_ASSERT(((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == 6);
+			PLT_ASSERT(ip->version == 6);
 
 			adj_len = ROC_CPT_TUNNEL_IPV6_HDR_LEN;
 			nxt_hdr += ROC_CPT_TUNNEL_IPV6_HDR_LEN;
-- 
2.25.1

[PATCH v2 04/11] crypto/cnxk: use direct mode for zero aad length

[Tejasree Kondoj]

Using direct mode if aad length is zero.

Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 drivers/crypto/cnxk/cnxk_se.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index c16027ec75..69cd343eea 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -2258,9 +2258,9 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 
 		aad_data = sym_op->aead.aad.data;
 		aad_len = sess->aad_length;
-		if (likely((aad_data + aad_len) ==
-			   rte_pktmbuf_mtod_offset(m_src, uint8_t *,
-						   sym_op->aead.data.offset))) {
+		if (likely((aad_len == 0) ||
+			   ((aad_data + aad_len) ==
+			    rte_pktmbuf_mtod_offset(m_src, uint8_t *, sym_op->aead.data.offset)))) {
 			d_offs = (d_offs - aad_len) | (d_offs << 16);
 			d_lens = (d_lens + aad_len) | (d_lens << 32);
 		} else {
-- 
2.25.1

[PATCH v2 05/11] crypto/cnxk: set ctx for AE

[Tejasree Kondoj]

Set ctx_val to 1 for asymmetric ops.

Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 18 ++++---------
 drivers/crypto/cnxk/cn9k_cryptodev_ops.c  | 16 +++--------
 drivers/crypto/cnxk/cnxk_ae.h             | 21 +++++++++++++++
 drivers/crypto/cnxk/cnxk_cryptodev_ops.c  | 33 ++++++++++++++++-------
 4 files changed, 53 insertions(+), 35 deletions(-)

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 92f7002db9..d1a43eaf13 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -158,10 +158,8 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[], struct
 
 		if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 			asym_op = op->asym;
-			ae_sess = (struct cnxk_ae_sess *)
-					asym_op->session->sess_private_data;
-			ret = cnxk_ae_enqueue(qp, op, infl_req, &inst[0],
-					      ae_sess);
+			ae_sess = (struct cnxk_ae_sess *)asym_op->session;
+			ret = cnxk_ae_enqueue(qp, op, infl_req, &inst[0], ae_sess);
 			if (unlikely(ret))
 				return 0;
 			w7 = ae_sess->cpt_inst_w7;
@@ -330,10 +328,9 @@ cn10k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused, vo
 			return -EINVAL;
 	} else if (op_type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
 		if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
-			struct rte_cryptodev_asym_session *asym_sess = sess;
 			struct cnxk_ae_sess *priv;
 
-			priv = (struct cnxk_ae_sess *)asym_sess->sess_private_data;
+			priv = (struct cnxk_ae_sess *)sess;
 			priv->qp = qp;
 			priv->cpt_inst_w2 = w2;
 		} else
@@ -381,11 +378,9 @@ cn10k_ca_meta_info_extract(struct rte_crypto_op *op, struct cnxk_cpt_qp **qp, ui
 		}
 	} else if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
 		if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
-			struct rte_cryptodev_asym_session *asym_sess;
 			struct cnxk_ae_sess *priv;
 
-			asym_sess = op->asym->session;
-			priv = (struct cnxk_ae_sess *)asym_sess->sess_private_data;
+			priv = (struct cnxk_ae_sess *)op->asym->session;
 			*qp = priv->qp;
 			*w2 = priv->cpt_inst_w2;
 		} else
@@ -890,10 +885,7 @@ cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
 		} else if (cop->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
 			struct rte_crypto_asym_op *op = cop->asym;
 			uintptr_t *mdata = infl_req->mdata;
-			struct cnxk_ae_sess *sess;
-
-			sess = (struct cnxk_ae_sess *)
-					op->session->sess_private_data;
+			struct cnxk_ae_sess *sess = (struct cnxk_ae_sess *)op->session;
 
 			cnxk_ae_post_process(cop, sess, (uint8_t *)mdata[0]);
 		}
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 11541b6ab9..34d40b07d4 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -105,13 +105,10 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 			inst->w7.u64 = sess->cpt_inst_w7;
 		}
 	} else if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
-		struct rte_crypto_asym_op *asym_op;
 		struct cnxk_ae_sess *sess;
 
 		if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
-			asym_op = op->asym;
-			sess = (struct cnxk_ae_sess *)
-					asym_op->session->sess_private_data;
+			sess = (struct cnxk_ae_sess *)op->asym->session;
 			ret = cnxk_ae_enqueue(qp, op, infl_req, inst, sess);
 			inst->w7.u64 = sess->cpt_inst_w7;
 		} else {
@@ -345,7 +342,7 @@ cn9k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused,
 			struct rte_cryptodev_asym_session *asym_sess = sess;
 			struct cnxk_ae_sess *priv;
 
-			priv = (struct cnxk_ae_sess *)asym_sess->sess_private_data;
+			priv = (struct cnxk_ae_sess *)asym_sess;
 			priv->qp = qp;
 			priv->cpt_inst_w2 = w2;
 		} else
@@ -393,11 +390,9 @@ cn9k_ca_meta_info_extract(struct rte_crypto_op *op,
 		}
 	} else if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
 		if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
-			struct rte_cryptodev_asym_session *asym_sess;
 			struct cnxk_ae_sess *priv;
 
-			asym_sess = op->asym->session;
-			priv = (struct cnxk_ae_sess *)asym_sess->sess_private_data;
+			priv = (struct cnxk_ae_sess *)op->asym->session;
 			*qp = priv->qp;
 			inst->w2.u64 = priv->cpt_inst_w2;
 		} else
@@ -609,10 +604,7 @@ cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop,
 		} else if (cop->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
 			struct rte_crypto_asym_op *op = cop->asym;
 			uintptr_t *mdata = infl_req->mdata;
-			struct cnxk_ae_sess *sess;
-
-			sess = (struct cnxk_ae_sess *)
-					op->session->sess_private_data;
+			struct cnxk_ae_sess *sess = (struct cnxk_ae_sess *)op->session;
 
 			cnxk_ae_post_process(cop, sess, (uint8_t *)mdata[0]);
 		}
diff --git a/drivers/crypto/cnxk/cnxk_ae.h b/drivers/crypto/cnxk/cnxk_ae.h
index 698c10129e..b7c13a9e01 100644
--- a/drivers/crypto/cnxk/cnxk_ae.h
+++ b/drivers/crypto/cnxk/cnxk_ae.h
@@ -13,7 +13,10 @@
 
 #include "cnxk_cryptodev_ops.h"
 
+#define ASYM_SESS_SIZE sizeof(struct rte_cryptodev_asym_session)
+
 struct cnxk_ae_sess {
+	uint8_t rte_sess[ASYM_SESS_SIZE];
 	enum rte_crypto_asym_xform_type xfrm_type;
 	union {
 		struct rte_crypto_rsa_xform rsa_ctx;
@@ -25,6 +28,24 @@ struct cnxk_ae_sess {
 	uint64_t cpt_inst_w7;
 	uint64_t cpt_inst_w2;
 	struct cnxk_cpt_qp *qp;
+	struct roc_cpt_lf *lf;
+	struct hw_ctx_s {
+		union {
+			struct {
+				uint64_t rsvd : 48;
+
+				uint64_t ctx_push_size : 7;
+				uint64_t rsvd1 : 1;
+
+				uint64_t ctx_hdr_size : 2;
+				uint64_t aop_valid : 1;
+				uint64_t rsvd2 : 1;
+				uint64_t ctx_size : 4;
+			} s;
+			uint64_t u64;
+		} w0;
+		uint8_t rsvd[256];
+	} hw_ctx __plt_aligned(ROC_ALIGN);
 };
 
 static __rte_always_inline void
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index 27f2846f74..f03646fe1a 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -760,14 +760,14 @@ cnxk_ae_session_size_get(struct rte_cryptodev *dev __rte_unused)
 }
 
 void
-cnxk_ae_session_clear(struct rte_cryptodev *dev,
-		      struct rte_cryptodev_asym_session *sess)
+cnxk_ae_session_clear(struct rte_cryptodev *dev, struct rte_cryptodev_asym_session *sess)
 {
-	struct cnxk_ae_sess *priv;
+	struct cnxk_ae_sess *priv = (struct cnxk_ae_sess *)sess;
 
-	priv = (struct cnxk_ae_sess *) sess->sess_private_data;
-	if (priv == NULL)
-		return;
+	/* Trigger CTX flush + invalidate to remove from CTX_CACHE */
+	roc_cpt_lf_ctx_flush(priv->lf, &priv->hw_ctx, true);
+
+	plt_delay_ms(1);
 
 	/* Free resources allocated in session_cfg */
 	cnxk_ae_free_session_parameters(priv);
@@ -777,23 +777,36 @@ cnxk_ae_session_clear(struct rte_cryptodev *dev,
 }
 
 int
-cnxk_ae_session_cfg(struct rte_cryptodev *dev,
-		    struct rte_crypto_asym_xform *xform,
+cnxk_ae_session_cfg(struct rte_cryptodev *dev, struct rte_crypto_asym_xform *xform,
 		    struct rte_cryptodev_asym_session *sess)
 {
-	struct cnxk_ae_sess *priv =
-			(struct cnxk_ae_sess *) sess->sess_private_data;
+	struct cnxk_ae_sess *priv = (struct cnxk_ae_sess *)sess;
 	struct cnxk_cpt_vf *vf = dev->data->dev_private;
 	struct roc_cpt *roc_cpt = &vf->cpt;
 	union cpt_inst_w7 w7;
+	struct hw_ctx_s *hwc;
 	int ret;
 
 	ret = cnxk_ae_fill_session_parameters(priv, xform);
 	if (ret)
 		return ret;
 
+	priv->lf = roc_cpt->lf[0];
+
 	w7.u64 = 0;
 	w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_AE];
+
+	if (vf->cpt.cpt_revision == ROC_CPT_REVISION_ID_106XX) {
+		hwc = &priv->hw_ctx;
+		hwc->w0.s.aop_valid = 1;
+		hwc->w0.s.ctx_hdr_size = 0;
+		hwc->w0.s.ctx_size = 1;
+		hwc->w0.s.ctx_push_size = 1;
+
+		w7.s.cptr = (uint64_t)hwc;
+		w7.s.ctx_val = 1;
+	}
+
 	priv->cpt_inst_w7 = w7.u64;
 	priv->cnxk_fpm_iova = vf->cnxk_fpm_iova;
 	priv->ec_grp = vf->ec_grp;
-- 
2.25.1

[PATCH v2 06/11] common/cnxk: ensure flush inval completion with CSR read

[Tejasree Kondoj]

From: Anoob Joseph <anoobj@marvell.com>

If a CSR read is issued after a write, the read would block till the
write operation is complete. This would help in determining when the
FLUSH+INVALIDATE operation is complete.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
 drivers/common/cnxk/hw/cpt.h             | 11 +++++++++++
 drivers/common/cnxk/roc_cpt.c            | 16 ++++++++++++++++
 drivers/crypto/cnxk/cnxk_cryptodev_ops.c |  4 ----
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/drivers/common/cnxk/hw/cpt.h b/drivers/common/cnxk/hw/cpt.h
index d378a4eadd..44ff8b08b2 100644
--- a/drivers/common/cnxk/hw/cpt.h
+++ b/drivers/common/cnxk/hw/cpt.h
@@ -100,6 +100,17 @@ union cpt_lf_ctx_flush {
 	} s;
 };
 
+union cpt_lf_ctx_err {
+	uint64_t u;
+	struct {
+		uint64_t flush_st_flt : 1;
+		uint64_t busy_flr : 1;
+		uint64_t busy_sw_flush : 1;
+		uint64_t reload_faulted : 1;
+		uint64_t reserved_4_63 : 1;
+	} s;
+};
+
 union cpt_lf_ctx_reload {
 	uint64_t u;
 	struct {
diff --git a/drivers/common/cnxk/roc_cpt.c b/drivers/common/cnxk/roc_cpt.c
index cf514be69f..dff2fbf2a4 100644
--- a/drivers/common/cnxk/roc_cpt.c
+++ b/drivers/common/cnxk/roc_cpt.c
@@ -783,6 +783,7 @@ int
 roc_cpt_lf_ctx_flush(struct roc_cpt_lf *lf, void *cptr, bool inval)
 {
 	union cpt_lf_ctx_flush reg;
+	union cpt_lf_ctx_err err;
 
 	if (lf == NULL) {
 		plt_err("Could not trigger CTX flush");
@@ -795,6 +796,21 @@ roc_cpt_lf_ctx_flush(struct roc_cpt_lf *lf, void *cptr, bool inval)
 
 	plt_write64(reg.u, lf->rbase + CPT_LF_CTX_FLUSH);
 
+	plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
+
+	/* Read a CSR to ensure that the FLUSH operation is complete */
+	err.u = plt_read64(lf->rbase + CPT_LF_CTX_ERR);
+
+	if (err.s.busy_sw_flush && inval) {
+		plt_err("CTX entry could not be invalidated due to active usage.");
+		return -EAGAIN;
+	}
+
+	if (err.s.flush_st_flt) {
+		plt_err("CTX flush could not complete due to store fault");
+		abort();
+	}
+
 	return 0;
 }
 
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index f03646fe1a..67bd7e3243 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -737,8 +737,6 @@ sym_session_clear(struct rte_cryptodev_sym_session *sess, bool is_session_less)
 	/* Trigger CTX flush + invalidate to remove from CTX_CACHE */
 	roc_cpt_lf_ctx_flush(sess_priv->lf, &sess_priv->roc_se_ctx.se_ctx, true);
 
-	plt_delay_ms(1);
-
 	if (sess_priv->roc_se_ctx.auth_key != NULL)
 		plt_free(sess_priv->roc_se_ctx.auth_key);
 
@@ -767,8 +765,6 @@ cnxk_ae_session_clear(struct rte_cryptodev *dev, struct rte_cryptodev_asym_sessi
 	/* Trigger CTX flush + invalidate to remove from CTX_CACHE */
 	roc_cpt_lf_ctx_flush(priv->lf, &priv->hw_ctx, true);
 
-	plt_delay_ms(1);
-
 	/* Free resources allocated in session_cfg */
 	cnxk_ae_free_session_parameters(priv);
 
-- 
2.25.1

[PATCH v2 07/11] common/cnxk: add errata function for CPT set ctx

[Tejasree Kondoj]

Adding function in errata header file for CPT ctx_val
and replace CPT revision_id checks with function call.

Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 drivers/common/cnxk/roc_errata.h         |  9 +++++++++
 drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 13 ++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/drivers/common/cnxk/roc_errata.h b/drivers/common/cnxk/roc_errata.h
index 36e6db467a..9954b7da32 100644
--- a/drivers/common/cnxk/roc_errata.h
+++ b/drivers/common/cnxk/roc_errata.h
@@ -4,6 +4,8 @@
 #ifndef _ROC_ERRATA_H_
 #define _ROC_ERRATA_H_
 
+#include "roc_model.h"
+
 /* Errata IPBUNIXRX-40129 */
 static inline bool
 roc_errata_nix_has_no_drop_re(void)
@@ -98,4 +100,11 @@ roc_errata_nix_sdp_send_has_mtu_size_16k(void)
 		roc_model_is_cn96_a0() || roc_model_is_cn96_b0());
 }
 
+/* Errata IPBUCPT-38753 */
+static inline bool
+roc_errata_cpt_hang_on_mixed_ctx_val(void)
+{
+	return roc_model_is_cn10ka_a0() || roc_model_is_cn10ka_a1();
+}
+
 #endif /* _ROC_ERRATA_H_ */
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index 67bd7e3243..adc1c7652b 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -8,6 +8,7 @@
 
 #include "roc_ae_fpm_tables.h"
 #include "roc_cpt.h"
+#include "roc_errata.h"
 #include "roc_ie_on.h"
 
 #include "cnxk_ae.h"
@@ -636,7 +637,7 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
 
 	inst_w7.s.cptr = (uint64_t)&sess->roc_se_ctx.se_ctx;
 
-	if (roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_106XX)
+	if (roc_errata_cpt_hang_on_mixed_ctx_val())
 		inst_w7.s.ctx_val = 1;
 	else
 		inst_w7.s.cptr += 8;
@@ -709,7 +710,7 @@ sym_session_configure(struct roc_cpt *roc_cpt, struct rte_crypto_sym_xform *xfor
 
 	sess_priv->cpt_inst_w7 = cnxk_cpt_inst_w7_get(sess_priv, roc_cpt);
 
-	if (roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_106XX)
+	if (roc_errata_cpt_hang_on_mixed_ctx_val())
 		roc_se_ctx_init(&sess_priv->roc_se_ctx);
 
 	return 0;
@@ -735,7 +736,8 @@ sym_session_clear(struct rte_cryptodev_sym_session *sess, bool is_session_less)
 	struct cnxk_se_sess *sess_priv = (struct cnxk_se_sess *)sess;
 
 	/* Trigger CTX flush + invalidate to remove from CTX_CACHE */
-	roc_cpt_lf_ctx_flush(sess_priv->lf, &sess_priv->roc_se_ctx.se_ctx, true);
+	if (roc_errata_cpt_hang_on_mixed_ctx_val())
+		roc_cpt_lf_ctx_flush(sess_priv->lf, &sess_priv->roc_se_ctx.se_ctx, true);
 
 	if (sess_priv->roc_se_ctx.auth_key != NULL)
 		plt_free(sess_priv->roc_se_ctx.auth_key);
@@ -763,7 +765,8 @@ cnxk_ae_session_clear(struct rte_cryptodev *dev, struct rte_cryptodev_asym_sessi
 	struct cnxk_ae_sess *priv = (struct cnxk_ae_sess *)sess;
 
 	/* Trigger CTX flush + invalidate to remove from CTX_CACHE */
-	roc_cpt_lf_ctx_flush(priv->lf, &priv->hw_ctx, true);
+	if (roc_errata_cpt_hang_on_mixed_ctx_val())
+		roc_cpt_lf_ctx_flush(priv->lf, &priv->hw_ctx, true);
 
 	/* Free resources allocated in session_cfg */
 	cnxk_ae_free_session_parameters(priv);
@@ -792,7 +795,7 @@ cnxk_ae_session_cfg(struct rte_cryptodev *dev, struct rte_crypto_asym_xform *xfo
 	w7.u64 = 0;
 	w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_AE];
 
-	if (vf->cpt.cpt_revision == ROC_CPT_REVISION_ID_106XX) {
+	if (roc_errata_cpt_hang_on_mixed_ctx_val()) {
 		hwc = &priv->hw_ctx;
 		hwc->w0.s.aop_valid = 1;
 		hwc->w0.s.ctx_hdr_size = 0;
-- 
2.25.1

[PATCH v2 08/11] common/cnxk: replace CPT revision check with caps

[Tejasree Kondoj]

Replace SG version revision check with capabilities
populated from microcode.

Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 drivers/common/cnxk/hw/cpt.h              | 3 ++-
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 2 +-
 drivers/event/cnxk/cn10k_eventdev.c       | 3 ++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/drivers/common/cnxk/hw/cpt.h b/drivers/common/cnxk/hw/cpt.h
index 44ff8b08b2..82ea076e4c 100644
--- a/drivers/common/cnxk/hw/cpt.h
+++ b/drivers/common/cnxk/hw/cpt.h
@@ -75,7 +75,8 @@ union cpt_eng_caps {
 		uint64_t __io mmul : 1;
 		uint64_t __io reserved_15_33 : 19;
 		uint64_t __io pdcp_chain : 1;
-		uint64_t __io reserved_35_63 : 29;
+		uint64_t __io sg_ver2 : 1;
+		uint64_t __io reserved_36_63 : 28;
 	};
 };
 
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index d1a43eaf13..9f6fd4e411 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -1045,7 +1045,7 @@ cn10k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 void
 cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev, struct cnxk_cpt_vf *vf)
 {
-	if (vf->cpt.cpt_revision > ROC_CPT_REVISION_ID_106XX)
+	if (vf->cpt.hw_caps[CPT_ENG_TYPE_SE].sg_ver2 && vf->cpt.hw_caps[CPT_ENG_TYPE_IE].sg_ver2)
 		dev->enqueue_burst = cn10k_cpt_sg_ver2_enqueue_burst;
 	else
 		dev->enqueue_burst = cn10k_cpt_sg_ver1_enqueue_burst;
diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c
index 8e74edff55..ee0428adc8 100644
--- a/drivers/event/cnxk/cn10k_eventdev.c
+++ b/drivers/event/cnxk/cn10k_eventdev.c
@@ -602,7 +602,8 @@ cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
 		}
 	}
 
-	if ((cpt != NULL) && (cpt->cpt_revision > ROC_CPT_REVISION_ID_106XX))
+	if ((cpt != NULL) && cpt->hw_caps[CPT_ENG_TYPE_SE].sg_ver2 &&
+	    cpt->hw_caps[CPT_ENG_TYPE_IE].sg_ver2)
 		event_dev->ca_enqueue = cn10k_cpt_sg_ver2_crypto_adapter_enqueue;
 	else
 		event_dev->ca_enqueue = cn10k_cpt_sg_ver1_crypto_adapter_enqueue;
-- 
2.25.1

[PATCH v2 09/11] crypto/cnxk: support cn10k IPsec SG mode

[Tejasree Kondoj]

Adding support for scatter-gather mode in 103XX and
106XX lookaside IPsec.

Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c |  21 +-
 drivers/crypto/cnxk/cn10k_ipsec_la_ops.h  | 222 ++++++++++++++++++++--
 drivers/crypto/cnxk/cnxk_sg.h             |  23 +++
 3 files changed, 239 insertions(+), 27 deletions(-)

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 9f6fd4e411..e405a2ad9f 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -77,8 +77,8 @@ cn10k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
 }
 
 static __rte_always_inline int __rte_hot
-cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
-		  struct cn10k_sec_session *sess, struct cpt_inst_s *inst)
+cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, struct cn10k_sec_session *sess,
+		  struct cpt_inst_s *inst, struct cpt_inflight_req *infl_req, const bool is_sg_ver2)
 {
 	struct rte_crypto_sym_op *sym_op = op->sym;
 	int ret;
@@ -88,15 +88,11 @@ cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 		return -ENOTSUP;
 	}
 
-	if (unlikely(!rte_pktmbuf_is_contiguous(sym_op->m_src))) {
-		plt_dp_err("Scatter Gather mode is not supported");
-		return -ENOTSUP;
-	}
-
 	if (sess->is_outbound)
-		ret = process_outb_sa(&qp->lf, op, sess, inst);
+		ret = process_outb_sa(&qp->lf, op, sess, &qp->meta_info, infl_req, inst,
+				      is_sg_ver2);
 	else
-		ret = process_inb_sa(op, sess, inst);
+		ret = process_inb_sa(op, sess, inst, &qp->meta_info, infl_req, is_sg_ver2);
 
 	return ret;
 }
@@ -129,7 +125,7 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[], struct
 	if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 		if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 			sec_sess = (struct cn10k_sec_session *)sym_op->session;
-			ret = cpt_sec_inst_fill(qp, op, sec_sess, &inst[0]);
+			ret = cpt_sec_inst_fill(qp, op, sec_sess, &inst[0], infl_req, is_sg_ver2);
 			if (unlikely(ret))
 				return 0;
 			w7 = sec_sess->inst.w7;
@@ -827,7 +823,10 @@ cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *re
 		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 		return;
 	}
-	mbuf->data_len = m_len;
+
+	if (mbuf->next == NULL)
+		mbuf->data_len = m_len;
+
 	mbuf->pkt_len = m_len;
 }
 
diff --git a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
index f2761a55a5..8e208eb2ca 100644
--- a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
@@ -8,9 +8,13 @@
 #include <rte_crypto_sym.h>
 #include <rte_security.h>
 
+#include "roc_ie.h"
+
 #include "cn10k_cryptodev.h"
 #include "cn10k_ipsec.h"
 #include "cnxk_cryptodev.h"
+#include "cnxk_cryptodev_ops.h"
+#include "cnxk_sg.h"
 
 static inline void
 ipsec_po_sa_iv_set(struct cn10k_sec_session *sess, struct rte_crypto_op *cop)
@@ -44,18 +48,14 @@ ipsec_po_sa_aes_gcm_iv_set(struct cn10k_sec_session *sess, struct rte_crypto_op
 
 static __rte_always_inline int
 process_outb_sa(struct roc_cpt_lf *lf, struct rte_crypto_op *cop, struct cn10k_sec_session *sess,
-		struct cpt_inst_s *inst)
+		struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
+		struct cpt_inst_s *inst, const bool is_sg_ver2)
 {
 	struct rte_crypto_sym_op *sym_op = cop->sym;
 	struct rte_mbuf *m_src = sym_op->m_src;
 	uint64_t inst_w4_u64 = sess->inst.w4;
 	uint64_t dptr;
 
-	if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) {
-		plt_dp_err("Not enough tail room");
-		return -ENOMEM;
-	}
-
 	RTE_SET_USED(lf);
 
 #ifdef LA_IPSEC_DEBUG
@@ -79,27 +79,217 @@ process_outb_sa(struct roc_cpt_lf *lf, struct rte_crypto_op *cop, struct cn10k_s
 	if (m_src->ol_flags & RTE_MBUF_F_TX_L4_MASK)
 		inst_w4_u64 &= ~BIT_ULL(32);
 
-	/* Prepare CPT instruction */
-	inst->w4.u64 = inst_w4_u64 | rte_pktmbuf_pkt_len(m_src);
-	dptr = rte_pktmbuf_mtod(m_src, uint64_t);
-	inst->dptr = dptr;
+	if (likely(m_src->next == NULL)) {
+		if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) {
+			plt_dp_err("Not enough tail room");
+			return -ENOMEM;
+		}
+
+		/* Prepare CPT instruction */
+		inst->w4.u64 = inst_w4_u64 | rte_pktmbuf_pkt_len(m_src);
+		dptr = rte_pktmbuf_mtod(m_src, uint64_t);
+		inst->dptr = dptr;
+	} else if (is_sg_ver2 == false) {
+		struct roc_sglist_comp *scatter_comp, *gather_comp;
+		uint32_t g_size_bytes, s_size_bytes;
+		struct rte_mbuf *last_seg;
+		uint8_t *in_buffer;
+		uint32_t dlen;
+		void *m_data;
+		int i;
+
+		last_seg = rte_pktmbuf_lastseg(m_src);
+
+		if (unlikely(rte_pktmbuf_tailroom(last_seg) < sess->max_extended_len)) {
+			plt_dp_err("Not enough tail room (required: %d, available: %d)",
+				   sess->max_extended_len, rte_pktmbuf_tailroom(last_seg));
+			return -ENOMEM;
+		}
+
+		m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+		if (unlikely(m_data == NULL)) {
+			plt_dp_err("Error allocating meta buffer for request");
+			return -ENOMEM;
+		}
+
+		in_buffer = m_data;
+
+		((uint16_t *)in_buffer)[0] = 0;
+		((uint16_t *)in_buffer)[1] = 0;
+
+		/* Input Gather List */
+		i = 0;
+		gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
+
+		i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);
+		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+		g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+		/* Output Scatter List */
+		last_seg->data_len += sess->max_extended_len;
+
+		i = 0;
+		scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+		i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);
+		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+
+		s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+		dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+		inst->dptr = (uint64_t)in_buffer;
+
+		inst->w4.u64 = sess->inst.w4 | dlen;
+		inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+	} else {
+		struct roc_sg2list_comp *scatter_comp, *gather_comp;
+		union cpt_inst_w5 cpt_inst_w5;
+		union cpt_inst_w6 cpt_inst_w6;
+		struct rte_mbuf *last_seg;
+		uint32_t g_size_bytes;
+		void *m_data;
+		int i;
+
+		last_seg = rte_pktmbuf_lastseg(m_src);
+
+		if (unlikely(rte_pktmbuf_tailroom(last_seg) < sess->max_extended_len)) {
+			plt_dp_err("Not enough tail room (required: %d, available: %d)",
+				   sess->max_extended_len, rte_pktmbuf_tailroom(last_seg));
+			return -ENOMEM;
+		}
+
+		m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+		if (unlikely(m_data == NULL)) {
+			plt_dp_err("Error allocating meta buffer for request");
+			return -ENOMEM;
+		}
+
+		/* Input Gather List */
+		i = 0;
+		gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_data);
+
+		i = fill_ipsec_sg2_comp_from_pkt(gather_comp, i, m_src);
+
+		cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
+		g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
+
+		/* Output Scatter List */
+		last_seg->data_len += sess->max_extended_len;
+
+		i = 0;
+		scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+		i = fill_ipsec_sg2_comp_from_pkt(scatter_comp, i, m_src);
+
+		cpt_inst_w6.s.scatter_sz = ((i + 2) / 3);
+
+		cpt_inst_w5.s.dptr = (uint64_t)gather_comp;
+		cpt_inst_w6.s.rptr = (uint64_t)scatter_comp;
+
+		inst->w5.u64 = cpt_inst_w5.u64;
+		inst->w6.u64 = cpt_inst_w6.u64;
+		inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+		inst->w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT));
+	}
 
 	return 0;
 }
 
 static __rte_always_inline int
-process_inb_sa(struct rte_crypto_op *cop, struct cn10k_sec_session *sess, struct cpt_inst_s *inst)
+process_inb_sa(struct rte_crypto_op *cop, struct cn10k_sec_session *sess, struct cpt_inst_s *inst,
+	       struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
+	       const bool is_sg_ver2)
 {
 	struct rte_crypto_sym_op *sym_op = cop->sym;
 	struct rte_mbuf *m_src = sym_op->m_src;
 	uint64_t dptr;
 
-	/* Prepare CPT instruction */
-	inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
-	dptr = rte_pktmbuf_mtod(m_src, uint64_t);
-	inst->dptr = dptr;
-	m_src->ol_flags |= (uint64_t)sess->ip_csum;
+	if (likely(m_src->next == NULL)) {
+		/* Prepare CPT instruction */
+		inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+		dptr = rte_pktmbuf_mtod(m_src, uint64_t);
+		inst->dptr = dptr;
+		m_src->ol_flags |= (uint64_t)sess->ip_csum;
+	} else if (is_sg_ver2 == false) {
+		struct roc_sglist_comp *scatter_comp, *gather_comp;
+		uint32_t g_size_bytes, s_size_bytes;
+		uint8_t *in_buffer;
+		uint32_t dlen;
+		void *m_data;
+		int i;
+
+		m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+		if (unlikely(m_data == NULL)) {
+			plt_dp_err("Error allocating meta buffer for request");
+			return -ENOMEM;
+		}
+
+		in_buffer = m_data;
+
+		((uint16_t *)in_buffer)[0] = 0;
+		((uint16_t *)in_buffer)[1] = 0;
+
+		/* Input Gather List */
+		i = 0;
+		gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
+		i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);
+		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+		g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
 
+		/* Output Scatter List */
+		i = 0;
+		scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+		i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);
+		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+
+		s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+		dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+		inst->dptr = (uint64_t)in_buffer;
+		inst->w4.u64 = sess->inst.w4 | dlen;
+		inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+	} else {
+		struct roc_sg2list_comp *scatter_comp, *gather_comp;
+		union cpt_inst_w5 cpt_inst_w5;
+		union cpt_inst_w6 cpt_inst_w6;
+		uint32_t g_size_bytes;
+		void *m_data;
+		int i;
+
+		m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+		if (unlikely(m_data == NULL)) {
+			plt_dp_err("Error allocating meta buffer for request");
+			return -ENOMEM;
+		}
+
+		/* Input Gather List */
+		i = 0;
+		gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_data);
+
+		i = fill_ipsec_sg2_comp_from_pkt(gather_comp, i, m_src);
+
+		cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
+		g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
+
+		/* Output Scatter List */
+		i = 0;
+		scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
+		i = fill_ipsec_sg2_comp_from_pkt(scatter_comp, i, m_src);
+
+		cpt_inst_w6.s.scatter_sz = ((i + 2) / 3);
+
+		cpt_inst_w5.s.dptr = (uint64_t)gather_comp;
+		cpt_inst_w6.s.rptr = (uint64_t)scatter_comp;
+
+		inst->w5.u64 = cpt_inst_w5.u64;
+		inst->w6.u64 = cpt_inst_w6.u64;
+		inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+		inst->w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT));
+	}
 	return 0;
 }
 
diff --git a/drivers/crypto/cnxk/cnxk_sg.h b/drivers/crypto/cnxk/cnxk_sg.h
index ead2886e99..65244199bd 100644
--- a/drivers/crypto/cnxk/cnxk_sg.h
+++ b/drivers/crypto/cnxk/cnxk_sg.h
@@ -6,6 +6,7 @@
 #define _CNXK_SG_H_
 
 #include "roc_cpt_sg.h"
+#include "roc_se.h"
 
 static __rte_always_inline uint32_t
 fill_sg_comp(struct roc_sglist_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
@@ -148,6 +149,28 @@ fill_ipsec_sg_comp_from_pkt(struct roc_sglist_comp *list, uint32_t i, struct rte
 	return i;
 }
 
+static __rte_always_inline uint32_t
+fill_ipsec_sg2_comp_from_pkt(struct roc_sg2list_comp *list, uint32_t i, struct rte_mbuf *pkt)
+{
+	uint32_t buf_sz;
+	void *vaddr;
+
+	while (unlikely(pkt != NULL)) {
+		struct roc_sg2list_comp *to = &list[i / 3];
+		buf_sz = pkt->data_len;
+		vaddr = rte_pktmbuf_mtod(pkt, void *);
+
+		to->u.s.len[i % 3] = buf_sz;
+		to->ptr[i % 3] = (uint64_t)vaddr;
+		to->u.s.valid_segs = (i % 3) + 1;
+
+		pkt = pkt->next;
+		i++;
+	}
+
+	return i;
+}
+
 static __rte_always_inline uint32_t
 fill_sg2_comp(struct roc_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
 {
-- 
2.25.1

[PATCH v2 10/11] crypto/cnxk: fix order of ECFPM params

[Tejasree Kondoj]

From: Gowrishankar Muthukrishnan <gmuthukrishn@marvell.com>

Fix the order of ECFPM parameters according to target board.

Fixes: 8e39b133235 ("crypto/cnxk: support fixed point multiplication")

Signed-off-by: Gowrishankar Muthukrishnan <gmuthukrishn@marvell.com>
---
 drivers/crypto/cnxk/cnxk_ae.h | 48 ++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/drivers/crypto/cnxk/cnxk_ae.h b/drivers/crypto/cnxk/cnxk_ae.h
index b7c13a9e01..47f000dd5e 100644
--- a/drivers/crypto/cnxk/cnxk_ae.h
+++ b/drivers/crypto/cnxk/cnxk_ae.h
@@ -700,7 +700,7 @@ static __rte_always_inline int
 cnxk_ae_ecfpm_prep(struct rte_crypto_ecpm_op_param *ecpm,
 		   struct roc_ae_buf_ptr *meta_buf, uint64_t *fpm_iova,
 		   struct roc_ae_ec_group *ec_grp, uint8_t curveid,
-		   struct cpt_inst_s *inst)
+		   struct cpt_inst_s *inst, int cpt_ver)
 {
 	uint16_t scalar_align, p_align;
 	uint16_t dlen, prime_len;
@@ -719,26 +719,33 @@ cnxk_ae_ecfpm_prep(struct rte_crypto_ecpm_op_param *ecpm,
 	scalar_align = RTE_ALIGN_CEIL(ecpm->scalar.length, 8);
 
 	/*
-	 * Set dlen = sum(ROUNDUP8(input point(x and y coordinates), prime,
-	 * scalar length),
+	 * Set dlen = sum(prime, scalar length, table address and
+	 * optionally ROUNDUP8(input point(x and y coordinates)).
 	 * Please note point length is equivalent to prime of the curve
 	 */
-	dlen = sizeof(fpm_table_iova) + 3 * p_align + scalar_align;
-
-	memset(dptr, 0, dlen);
-
-	*(uint64_t *)dptr = fpm_table_iova;
-	dptr += sizeof(fpm_table_iova);
-
-	/* Copy scalar, prime */
-	memcpy(dptr, ecpm->scalar.data, ecpm->scalar.length);
-	dptr += scalar_align;
-	memcpy(dptr, ec_grp->prime.data, ec_grp->prime.length);
-	dptr += p_align;
-	memcpy(dptr, ec_grp->consta.data, ec_grp->consta.length);
-	dptr += p_align;
-	memcpy(dptr, ec_grp->constb.data, ec_grp->constb.length);
-	dptr += p_align;
+	if (cpt_ver == ROC_CPT_REVISION_ID_96XX_C0) {
+		dlen = sizeof(fpm_table_iova) + 3 * p_align + scalar_align;
+		memset(dptr, 0, dlen);
+		*(uint64_t *)dptr = fpm_table_iova;
+		dptr += sizeof(fpm_table_iova);
+		memcpy(dptr, ecpm->scalar.data, ecpm->scalar.length);
+		dptr += scalar_align;
+		memcpy(dptr, ec_grp->prime.data, ec_grp->prime.length);
+		dptr += p_align;
+		memcpy(dptr, ec_grp->consta.data, ec_grp->consta.length);
+		dptr += p_align;
+		memcpy(dptr, ec_grp->constb.data, ec_grp->constb.length);
+		dptr += p_align;
+	} else {
+		dlen = sizeof(fpm_table_iova) + p_align + scalar_align;
+		memset(dptr, 0, dlen);
+		memcpy(dptr, ecpm->scalar.data, ecpm->scalar.length);
+		dptr += scalar_align;
+		memcpy(dptr, ec_grp->prime.data, ec_grp->prime.length);
+		dptr += p_align;
+		*(uint64_t *)dptr = fpm_table_iova;
+		dptr += sizeof(fpm_table_iova);
+	}
 
 	/* Setup opcodes */
 	w4.s.opcode_major = ROC_AE_MAJOR_OP_ECC;
@@ -969,7 +976,8 @@ cnxk_ae_enqueue(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 		ret = cnxk_ae_ecfpm_prep(&asym_op->ecpm, &meta_buf,
 					 sess->cnxk_fpm_iova,
 					 sess->ec_grp[sess->ec_ctx.curveid],
-					 sess->ec_ctx.curveid, inst);
+					 sess->ec_ctx.curveid, inst,
+					 sess->lf->roc_cpt->cpt_revision);
 		if (unlikely(ret))
 			goto req_fail;
 		break;
-- 
2.25.1

[PATCH v2 11/11] crypto/cnxk: add model check for pdcp chain

[Tejasree Kondoj]

Adding cn9k model check for pdcp_chain as
it is not supported in cn10k.

Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index adc1c7652b..86efe75cc3 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -479,7 +479,7 @@ cnxk_sess_fill(struct roc_cpt *roc_cpt, struct rte_crypto_sym_xform *xform,
 	bool pdcp_chain_supported = false;
 	bool ciph_then_auth = false;
 
-	if (roc_cpt->hw_caps[CPT_ENG_TYPE_SE].pdcp_chain)
+	if (roc_model_is_cn9k() && (roc_cpt->hw_caps[CPT_ENG_TYPE_SE].pdcp_chain))
 		pdcp_chain_supported = true;
 
 	if (xform == NULL)
-- 
2.25.1

RE: [PATCH v2 05/11] crypto/cnxk: set ctx for AE

[Akhil Goyal]

> Subject: [PATCH v2 05/11] crypto/cnxk: set ctx for AE
> 
> Set ctx_val to 1 for asymmetric ops.
> 

Please justify patch with proper description.

> Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>

RE: [PATCH v2 05/11] crypto/cnxk: set ctx for AE

[Tejasree Kondoj]

Hi Akhil,

Patch sets ctx_val to 1 for asymmetric due to cn10k A0 hardware errata and structures are redesigned to satisfy hardware alignment requirements.

Thanks
Tejasree

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Monday, February 27, 2023 11:09 PM
> To: Tejasree Kondoj <ktejasree@marvell.com>
> Cc: Anoob Joseph <anoobj@marvell.com>; Gowrishankar Muthukrishnan
> <gmuthukrishn@marvell.com>; dev@dpdk.org
> Subject: RE: [PATCH v2 05/11] crypto/cnxk: set ctx for AE
> 
> > Subject: [PATCH v2 05/11] crypto/cnxk: set ctx for AE
> >
> > Set ctx_val to 1 for asymmetric ops.
> >
> 
> Please justify patch with proper description.
> 
> > Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>

RE: [PATCH v2 00/11] fixes and improvements to cnxk crypto PMD

[Akhil Goyal]

> Subject: [PATCH v2 00/11] fixes and improvements to cnxk crypto PMD
> 
> This series adds cn10k scatter gather support and improvements
> to cnxk crypto PMD.
> 
> v2:
> * Fixed clang build failure
Series applied to dpdk-next-crypto
Fixed few patch title and descriptions

Thanks.