* [dpdk-dev] [PATCH 1/7] common/cnxk: update to v1.13 ZUC API
2021-09-02 12:22 [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Anoob Joseph
@ 2021-09-02 12:22 ` Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 2/7] common/cnxk: update to v1.16 ucc codes Anoob Joseph
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anoob Joseph @ 2021-09-02 12:22 UTC (permalink / raw)
To: Akhil Goyal, Jerin Jacob
Cc: Vidya Sagar Velumuri, Archana Muniganti, Tejasree Kondoj, dev,
Anoob Joseph
From: Vidya Sagar Velumuri <vvelumuri@marvell.com>
Add support for ZUC API change in ucode 1.13
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Vidya Sagar Velumuri <vvelumuri@marvell.com>
---
drivers/common/cnxk/roc_se.c | 150 ++++++++++++++++++++++++++++++++++++----
drivers/common/cnxk/roc_se.h | 37 +++++++++-
drivers/common/cnxk/version.map | 1 +
3 files changed, 172 insertions(+), 16 deletions(-)
diff --git a/drivers/common/cnxk/roc_se.c b/drivers/common/cnxk/roc_se.c
index b04de79..03fbc5f 100644
--- a/drivers/common/cnxk/roc_se.c
+++ b/drivers/common/cnxk/roc_se.c
@@ -72,6 +72,11 @@ cpt_ciph_type_set(roc_se_cipher_type type, struct roc_se_ctx *ctx,
fc_type = ROC_SE_FC_GEN;
break;
case ROC_SE_ZUC_EEA3:
+ /* No support for chained operations */
+ if (unlikely(ctx->hash_type))
+ return -1;
+ fc_type = ROC_SE_PDCP;
+ break;
case ROC_SE_SNOW3G_UEA2:
if (unlikely(key_len != 16))
return -1;
@@ -123,6 +128,56 @@ cpt_ciph_aes_key_type_set(struct roc_se_context *fctx, uint16_t key_len)
fctx->enc.aes_key = aes_key_type;
}
+static int
+cpt_pdcp_key_type_set(struct roc_se_zuc_snow3g_ctx *zs_ctx, uint16_t key_len)
+{
+ roc_se_aes_type key_type = 0;
+
+ if (roc_model_is_cn9k()) {
+ if (key_len != 16) {
+ plt_err("Only key len 16 is supported on cn9k");
+ return -ENOTSUP;
+ }
+ }
+
+ switch (key_len) {
+ case 16:
+ key_type = ROC_SE_AES_128_BIT;
+ break;
+ case 32:
+ key_type = ROC_SE_AES_256_BIT;
+ break;
+ default:
+ plt_err("Invalid AES key len");
+ return -ENOTSUP;
+ }
+ zs_ctx->zuc.otk_ctx.w0.s.key_len = key_type;
+ return 0;
+}
+
+static int
+cpt_pdcp_mac_len_set(struct roc_se_zuc_snow3g_ctx *zs_ctx, uint16_t mac_len)
+{
+ roc_se_pdcp_mac_len_type mac_type = 0;
+
+ switch (mac_len) {
+ case 4:
+ mac_type = ROC_SE_PDCP_MAC_LEN_32_BIT;
+ break;
+ case 8:
+ mac_type = ROC_SE_PDCP_MAC_LEN_64_BIT;
+ break;
+ case 16:
+ mac_type = ROC_SE_PDCP_MAC_LEN_128_BIT;
+ break;
+ default:
+ plt_err("Invalid ZUC MAC len");
+ return -ENOTSUP;
+ }
+ zs_ctx->zuc.otk_ctx.w0.s.mac_len = mac_type;
+ return 0;
+}
+
int
roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
const uint8_t *key, uint16_t key_len, uint16_t mac_len)
@@ -130,6 +185,7 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
struct roc_se_zuc_snow3g_ctx *zs_ctx;
struct roc_se_kasumi_ctx *k_ctx;
struct roc_se_context *fctx;
+ int ret;
if (se_ctx == NULL)
return -1;
@@ -139,32 +195,57 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
fctx = &se_ctx->se_ctx.fctx;
if ((type >= ROC_SE_ZUC_EIA3) && (type <= ROC_SE_KASUMI_F9_ECB)) {
+ uint8_t *zuc_const;
uint32_t keyx[4];
+ uint8_t *ci_key;
- if (key_len != 16)
+ if (!key_len)
return -1;
- /* No support for AEAD yet */
+
+ /* No support for chained operations yet */
if (se_ctx->enc_cipher)
return -1;
+
+ if (roc_model_is_cn9k()) {
+ ci_key = zs_ctx->zuc.onk_ctx.ci_key;
+ zuc_const = zs_ctx->zuc.onk_ctx.zuc_const;
+ } else {
+ ci_key = zs_ctx->zuc.otk_ctx.ci_key;
+ zuc_const = zs_ctx->zuc.otk_ctx.zuc_const;
+ }
+
/* For ZUC/SNOW3G/Kasumi */
switch (type) {
case ROC_SE_SNOW3G_UIA2:
+ zs_ctx->zuc.otk_ctx.w0.s.alg_type =
+ ROC_SE_PDCP_ALG_TYPE_SNOW3G;
+ zs_ctx->zuc.otk_ctx.w0.s.mac_len =
+ ROC_SE_PDCP_MAC_LEN_32_BIT;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
cpt_snow3g_key_gen(key, keyx);
- memcpy(zs_ctx->ci_key, keyx, key_len);
+ memcpy(ci_key, keyx, key_len);
se_ctx->fc_type = ROC_SE_PDCP;
se_ctx->zsk_flags = 0x1;
break;
case ROC_SE_ZUC_EIA3:
+ zs_ctx->zuc.otk_ctx.w0.s.alg_type =
+ ROC_SE_PDCP_ALG_TYPE_ZUC;
+ ret = cpt_pdcp_mac_len_set(zs_ctx, mac_len);
+ if (ret)
+ return ret;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
- memcpy(zs_ctx->ci_key, key, key_len);
- memcpy(zs_ctx->zuc_const, zuc_d, 32);
+ memcpy(ci_key, key, key_len);
+ memcpy(zuc_const, zuc_d, 32);
se_ctx->fc_type = ROC_SE_PDCP;
se_ctx->zsk_flags = 0x1;
break;
case ROC_SE_AES_CMAC_EIA2:
+ zs_ctx->zuc.otk_ctx.w0.s.alg_type =
+ ROC_SE_PDCP_ALG_TYPE_AES_CTR;
+ zs_ctx->zuc.otk_ctx.w0.s.mac_len =
+ ROC_SE_PDCP_MAC_LEN_32_BIT;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_AES_CTR;
- memcpy(zs_ctx->ci_key, key, key_len);
+ memcpy(ci_key, key, key_len);
se_ctx->fc_type = ROC_SE_PDCP;
se_ctx->zsk_flags = 0x1;
break;
@@ -183,8 +264,13 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
default:
return -1;
}
- se_ctx->mac_len = 4;
+ se_ctx->mac_len = mac_len;
se_ctx->hash_type = type;
+ if (roc_model_is_cn9k())
+ se_ctx->template_w4.s.opcode_minor =
+ ((1 << 7) | (se_ctx->pdcp_alg_type << 5) | 1);
+ else
+ se_ctx->template_w4.s.opcode_minor = ((1 << 4) | 1);
return 0;
}
@@ -227,11 +313,21 @@ int
roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
const uint8_t *key, uint16_t key_len, uint8_t *salt)
{
+ struct roc_se_zuc_snow3g_ctx *zs_ctx = &se_ctx->se_ctx.zs_ctx;
struct roc_se_context *fctx = &se_ctx->se_ctx.fctx;
- struct roc_se_zuc_snow3g_ctx *zs_ctx;
+ uint8_t *zuc_const;
uint32_t keyx[4];
+ uint8_t *ci_key;
int ret;
+ if (roc_model_is_cn9k()) {
+ ci_key = zs_ctx->zuc.onk_ctx.ci_key;
+ zuc_const = zs_ctx->zuc.onk_ctx.zuc_const;
+ } else {
+ ci_key = zs_ctx->zuc.otk_ctx.ci_key;
+ zuc_const = zs_ctx->zuc.otk_ctx.zuc_const;
+ }
+
/* For AES-GCM, salt is taken from ctx even if IV source
* is from DPTR
*/
@@ -301,21 +397,29 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
memcpy(fctx->hmac.ipad, &key[key_len], key_len);
break;
case ROC_SE_SNOW3G_UEA2:
+ zs_ctx->zuc.otk_ctx.w0.s.key_len = ROC_SE_AES_128_BIT;
+ zs_ctx->zuc.otk_ctx.w0.s.alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
cpt_snow3g_key_gen(key, keyx);
- memcpy(se_ctx->se_ctx.zs_ctx.ci_key, keyx, key_len);
+ memcpy(ci_key, keyx, key_len);
se_ctx->zsk_flags = 0;
goto success;
case ROC_SE_ZUC_EEA3:
- zs_ctx = &se_ctx->se_ctx.zs_ctx;
+ ret = cpt_pdcp_key_type_set(zs_ctx, key_len);
+ if (ret)
+ return ret;
+ zs_ctx->zuc.otk_ctx.w0.s.alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
- memcpy(zs_ctx->ci_key, key, key_len);
- memcpy(zs_ctx->zuc_const, zuc_d, 32);
+ memcpy(ci_key, key, key_len);
+ memcpy(zuc_const, zuc_d, 32);
se_ctx->zsk_flags = 0;
goto success;
case ROC_SE_AES_CTR_EEA2:
+ zs_ctx->zuc.otk_ctx.w0.s.key_len = ROC_SE_AES_128_BIT;
+ zs_ctx->zuc.otk_ctx.w0.s.alg_type =
+ ROC_SE_PDCP_ALG_TYPE_AES_CTR;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_AES_CTR;
- memcpy(se_ctx->se_ctx.zs_ctx.ci_key, key, key_len);
+ memcpy(ci_key, key, key_len);
se_ctx->zsk_flags = 0;
goto success;
case ROC_SE_KASUMI_F8_ECB:
@@ -341,6 +445,24 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
success:
se_ctx->enc_cipher = type;
-
+ if (se_ctx->fc_type == ROC_SE_PDCP) {
+ if (roc_model_is_cn9k())
+ se_ctx->template_w4.s.opcode_minor =
+ ((1 << 7) | (se_ctx->pdcp_alg_type << 5) |
+ (se_ctx->zsk_flags & 0x7));
+ else
+ se_ctx->template_w4.s.opcode_minor = ((1 << 4));
+ }
return 0;
}
+
+void
+roc_se_ctx_swap(struct roc_se_ctx *se_ctx)
+{
+ struct roc_se_zuc_snow3g_ctx *zs_ctx = &se_ctx->se_ctx.zs_ctx;
+
+ if (roc_model_is_cn9k())
+ return;
+
+ zs_ctx->zuc.otk_ctx.w0.u64 = htobe64(zs_ctx->zuc.otk_ctx.w0.u64);
+}
diff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h
index 5c7e2ca..051d496 100644
--- a/drivers/common/cnxk/roc_se.h
+++ b/drivers/common/cnxk/roc_se.h
@@ -158,6 +158,12 @@ typedef enum {
ROC_SE_AES_256_BIT = 0x3
} roc_se_aes_type;
+typedef enum {
+ ROC_SE_PDCP_MAC_LEN_32_BIT = 0x1,
+ ROC_SE_PDCP_MAC_LEN_64_BIT = 0x2,
+ ROC_SE_PDCP_MAC_LEN_128_BIT = 0x3
+} roc_se_pdcp_mac_len_type;
+
struct roc_se_sglist_comp {
union {
uint64_t len;
@@ -192,12 +198,35 @@ struct roc_se_context {
struct roc_se_hmac_context hmac;
};
-struct roc_se_zuc_snow3g_ctx {
+struct roc_se_otk_zuc_ctx {
+ union {
+ uint64_t u64;
+ struct {
+ uint64_t rsvd_56 : 57;
+ uint64_t mac_len : 2;
+ uint64_t key_len : 2;
+ uint64_t lfsr_state : 1;
+ uint64_t alg_type : 2;
+ } s;
+ } w0;
+ uint8_t ci_key[32];
+ uint8_t encr_auth_iv[24];
+ uint8_t zuc_const[32];
+};
+
+struct roc_se_onk_zuc_ctx {
uint8_t encr_auth_iv[16];
uint8_t ci_key[16];
uint8_t zuc_const[32];
};
+struct roc_se_zuc_snow3g_ctx {
+ union {
+ struct roc_se_onk_zuc_ctx onk_ctx;
+ struct roc_se_otk_zuc_ctx otk_ctx;
+ } zuc;
+};
+
struct roc_se_kasumi_ctx {
uint8_t reg_A[8];
uint8_t ci_key[16];
@@ -229,7 +258,10 @@ struct roc_se_fc_params {
void *auth_iv_buf;
struct roc_se_buf_ptr meta_buf;
struct roc_se_buf_ptr ctx_buf;
- uint64_t rsvd2;
+ uint32_t rsvd2;
+ uint16_t rsvd3;
+ uint8_t cipher_iv_len;
+ uint8_t auth_iv_len;
/* 1st cache line */
struct roc_se_buf_ptr aad_buf __plt_cache_aligned;
@@ -272,4 +304,5 @@ int __roc_api roc_se_ciph_key_set(struct roc_se_ctx *se_ctx,
roc_se_cipher_type type, const uint8_t *key,
uint16_t key_len, uint8_t *salt);
+void __roc_api roc_se_ctx_swap(struct roc_se_ctx *se_ctx);
#endif /* __ROC_SE_H__ */
diff --git a/drivers/common/cnxk/version.map b/drivers/common/cnxk/version.map
index 34a844b..5865fc6 100644
--- a/drivers/common/cnxk/version.map
+++ b/drivers/common/cnxk/version.map
@@ -267,6 +267,7 @@ INTERNAL {
roc_tim_lf_disable;
roc_tim_lf_enable;
roc_tim_lf_free;
+ roc_se_ctx_swap;
local: *;
};
--
2.7.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [dpdk-dev] [PATCH 2/7] common/cnxk: update to v1.16 ucc codes
2021-09-02 12:22 [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 1/7] common/cnxk: update to v1.13 ZUC API Anoob Joseph
@ 2021-09-02 12:22 ` Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 3/7] crypto/cnxk: remove redundant snow3g dec Anoob Joseph
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anoob Joseph @ 2021-09-02 12:22 UTC (permalink / raw)
To: Akhil Goyal, Jerin Jacob
Cc: Anoob Joseph, Archana Muniganti, Tejasree Kondoj, dev
Update to v1.16 microcode completion codes.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/common/cnxk/roc_ie_ot.h | 65 +++++++++++++++++++++--------------------
1 file changed, 34 insertions(+), 31 deletions(-)
diff --git a/drivers/common/cnxk/roc_ie_ot.h b/drivers/common/cnxk/roc_ie_ot.h
index 3987a08..1ff4688 100644
--- a/drivers/common/cnxk/roc_ie_ot.h
+++ b/drivers/common/cnxk/roc_ie_ot.h
@@ -14,37 +14,40 @@
enum roc_ie_ot_ucc_ipsec {
ROC_IE_OT_UCC_SUCCESS = 0x00,
- ROC_IE_OT_UCC_SUCCESS_PKT_IP_GOODCSUM = 0x02,
- ROC_IE_OT_UCC_ERR_SA_INVAL = 0x03,
- ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM = 0x04,
- ROC_IE_OT_UCC_ERR_SA_EXPIRED = 0x05,
- ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM = 0x06,
- ROC_IE_OT_UCC_ERR_SA_OVERFLOW = 0x07,
- ROC_IE_OT_UCC_SUCCESS_PKT_L4_BADCSUM = 0x08,
- ROC_IE_OT_UCC_ERR_SA_ESP_BAD_ALGO = 0x09,
- ROC_IE_OT_UCC_SUCCESS_PKT_UDPESP_NZCSUM = 0x0a,
- ROC_IE_OT_UCC_ERR_SA_ESP_BAD_KEYS = 0x0b,
- ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST = 0x0c,
- ROC_IE_OT_UCC_ERR_SA_AH_BAD_ALGO = 0x0d,
- ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_AGAIN = 0x0e,
- ROC_IE_OT_UCC_ERR_SA_AH_BAD_KEYS = 0x0f,
- ROC_IE_OT_UCC_ERR_SA_BAD_IP = 0x11,
- ROC_IE_OT_UCC_ERR_SA_BAD_CTX = 0x13,
- ROC_IE_OT_UCC_ERR_AOP_IPSEC = 0x17,
- ROC_IE_OT_UCC_ERR_PKT_IP = 0x23,
- ROC_IE_OT_UCC_ERR_PKT_IP6_BAD_EXT = 0x25,
- ROC_IE_OT_UCC_ERR_PKT_IP6_HBH = 0x27,
- ROC_IE_OT_UCC_ERR_PKT_IP6_BIGEXT = 0x29,
- ROC_IE_OT_UCC_ERR_PKT_IP_FRAG = 0x2b,
- ROC_IE_OT_UCC_ERR_PKT_IP_ULP = 0x2d,
- ROC_IE_OT_UCC_ERR_PKT_SA_MISMATCH = 0x2f,
- ROC_IE_OT_UCC_ERR_PKT_SPI_MISMATCH = 0x31,
- ROC_IE_OT_UCC_ERR_PKT_ESP_BADPAD = 0x33,
- ROC_IE_OT_UCC_ERR_PKT_BADICV = 0x35,
- ROC_IE_OT_UCC_ERR_PKT_REPLAY_SEQ = 0x37,
- ROC_IE_OT_UCC_ERR_PKT_REPLAY_WINDOW = 0x39,
- ROC_IE_OT_UCC_ERR_PKT_BADNH = 0x3b,
- ROC_IE_OT_UCC_ERR_PKT_SA_PORT_MISMATCH = 0x3d,
+ ROC_IE_OT_UCC_ERR_SA_INVAL = 0xb0,
+ ROC_IE_OT_UCC_ERR_SA_EXPIRED = 0xb1,
+ ROC_IE_OT_UCC_ERR_SA_OVERFLOW = 0xb2,
+ ROC_IE_OT_UCC_ERR_SA_ESP_BAD_ALGO = 0xb3,
+ ROC_IE_OT_UCC_ERR_SA_AH_BAD_ALGO = 0xb4,
+ ROC_IE_OT_UCC_ERR_SA_BAD_CTX = 0xb5,
+ ROC_IE_OT_UCC_SA_CTX_FLAG_MISMATCH = 0xb6,
+ ROC_IE_OT_UCC_ERR_AOP_IPSEC = 0xb7,
+ ROC_IE_OT_UCC_ERR_PKT_IP = 0xb8,
+ ROC_IE_OT_UCC_ERR_PKT_IP6_BAD_EXT = 0xb9,
+ ROC_IE_OT_UCC_ERR_PKT_IP6_HBH = 0xba,
+ ROC_IE_OT_UCC_ERR_PKT_IP6_BIGEXT = 0xbb,
+ ROC_IE_OT_UCC_ERR_PKT_IP_ULP = 0xbc,
+ ROC_IE_OT_UCC_ERR_PKT_SA_MISMATCH = 0xbd,
+ ROC_IE_OT_UCC_ERR_PKT_SPI_MISMATCH = 0xbe,
+ ROC_IE_OT_UCC_ERR_PKT_ESP_BADPAD = 0xbf,
+ ROC_IE_OT_UCC_ERR_PKT_BADICV = 0xc0,
+ ROC_IE_OT_UCC_ERR_PKT_REPLAY_SEQ = 0xc1,
+ ROC_IE_OT_UCC_ERR_PKT_BADNH = 0xc2,
+ ROC_IE_OT_UCC_ERR_PKT_SA_PORT_MISMATCH = 0xc3,
+ ROC_IE_OT_UCC_ERR_PKT_BAD_DLEN = 0xc4,
+ ROC_IE_OT_UCC_ERR_SA_ESP_BAD_KEYS = 0xc5,
+ ROC_IE_OT_UCC_ERR_SA_AH_BAD_KEYS = 0xc6,
+ ROC_IE_OT_UCC_ERR_SA_BAD_IP = 0xc7,
+ ROC_IE_OT_UCC_ERR_PKT_REPLAY_WINDOW = 0xc8,
+ ROC_IE_OT_UCC_ERR_PKT_IP_FRAG = 0xc9,
+ ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST = 0xf0,
+ ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM = 0xf1,
+ ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_AGAIN = 0xf2,
+ ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM = 0xf3,
+ ROC_IE_OT_UCC_SUCCESS_PKT_L4_BADCSUM = 0xf4,
+ ROC_IE_OT_UCC_SUCCESS_PKT_UDPESP_NZCSUM = 0xf5,
+ ROC_IE_OT_UCC_SUCCESS_PKT_UDP_ZEROCSUM = 0xf6,
+ ROC_IE_OT_UCC_SUCCESS_PKT_IP_GOODCSUM = 0xf7,
};
enum {
--
2.7.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [dpdk-dev] [PATCH 3/7] crypto/cnxk: remove redundant snow3g dec
2021-09-02 12:22 [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 1/7] common/cnxk: update to v1.13 ZUC API Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 2/7] common/cnxk: update to v1.16 ucc codes Anoob Joseph
@ 2021-09-02 12:22 ` Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 4/7] crypto/cnxk: remove redundant memcpy of IV for ZUC Anoob Joseph
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anoob Joseph @ 2021-09-02 12:22 UTC (permalink / raw)
To: Akhil Goyal, Jerin Jacob
Cc: Anoob Joseph, Archana Muniganti, Tejasree Kondoj, dev
The opcode for encryption & decryption is the same and single routine
would be able to handle both encryption and decryption operations.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/crypto/cnxk/cnxk_se.h | 232 ++++--------------------------------------
1 file changed, 17 insertions(+), 215 deletions(-)
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 3ed6b90..9d1ce09 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -947,17 +947,16 @@ cpt_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
}
static __rte_always_inline int
-cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
- struct roc_se_fc_params *params,
- struct cpt_inst_s *inst)
+cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
+ struct roc_se_fc_params *params, struct cpt_inst_s *inst)
{
uint32_t size;
int32_t inputlen, outputlen;
struct roc_se_ctx *se_ctx;
uint32_t mac_len = 0;
uint8_t pdcp_alg_type, j;
- uint32_t encr_offset = 0, auth_offset = 0;
- uint32_t encr_data_len = 0, auth_data_len = 0;
+ uint32_t encr_offset, auth_offset;
+ uint32_t encr_data_len, auth_data_len;
int flags, iv_len = 16;
uint64_t offset_ctrl;
uint64_t *offset_vaddr;
@@ -995,6 +994,10 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset);
+ encr_data_len = 0;
+ encr_offset = 0;
+
+ iv_s = params->auth_iv_buf;
} else {
/* EEA3 or UEA2 */
/*
@@ -1013,6 +1016,11 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
/* iv offset is 0 */
offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+
+ auth_data_len = 0;
+ auth_offset = 0;
+
+ iv_s = params->iv_buf;
}
if (unlikely((encr_offset >> 16) || (auth_offset >> 8))) {
@@ -1022,9 +1030,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
return -1;
}
- /* IV */
- iv_s = (flags == 0x1) ? params->auth_iv_buf : params->iv_buf;
-
if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
/*
* DPDK seems to provide it in form of IV3 IV2 IV1 IV0
@@ -1209,209 +1214,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
}
static __rte_always_inline int
-cpt_zuc_snow3g_dec_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
- struct roc_se_fc_params *params,
- struct cpt_inst_s *inst)
-{
- uint32_t size;
- int32_t inputlen = 0, outputlen;
- struct roc_se_ctx *se_ctx;
- uint8_t pdcp_alg_type, iv_len = 16;
- uint32_t encr_offset;
- uint32_t encr_data_len;
- int flags;
- uint64_t *offset_vaddr;
- uint32_t *iv_s, iv[4], j;
- union cpt_inst_w4 cpt_inst_w4;
-
- /*
- * Microcode expects offsets in bytes
- * TODO: Rounding off
- */
- encr_offset = ROC_SE_ENCR_OFFSET(d_offs) / 8;
- encr_data_len = ROC_SE_ENCR_DLEN(d_lens);
-
- se_ctx = params->ctx_buf.vaddr;
- flags = se_ctx->zsk_flags;
- pdcp_alg_type = se_ctx->pdcp_alg_type;
-
- cpt_inst_w4.u64 = 0;
- cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_ZUC_SNOW3G;
-
- /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
-
- cpt_inst_w4.s.opcode_minor = ((1 << 7) | (pdcp_alg_type << 5) |
- (0 << 4) | (0 << 3) | (flags & 0x7));
-
- /* consider iv len */
- encr_offset += iv_len;
-
- inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
- outputlen = inputlen;
-
- /* IV */
- iv_s = params->iv_buf;
- if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
- /*
- * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
- * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
- */
-
- for (j = 0; j < 4; j++)
- iv[j] = iv_s[3 - j];
- } else {
- /* ZUC doesn't need a swap */
- for (j = 0; j < 4; j++)
- iv[j] = iv_s[j];
- }
-
- /*
- * GP op header, lengths are expected in bits.
- */
- cpt_inst_w4.s.param1 = encr_data_len;
-
- /*
- * In cn9k, cn10k since we have a limitation of
- * IV & Offset control word not part of instruction
- * and need to be part of Data Buffer, we check if
- * head room is there and then only do the Direct mode processing
- */
- if (likely((req_flags & ROC_SE_SINGLE_BUF_INPLACE) &&
- (req_flags & ROC_SE_SINGLE_BUF_HEADROOM))) {
- void *dm_vaddr = params->bufs[0].vaddr;
-
- /* Use Direct mode */
-
- offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
- ROC_SE_OFF_CTRL_LEN - iv_len);
-
- /* DPTR */
- inst->dptr = (uint64_t)offset_vaddr;
-
- /* RPTR should just exclude offset control word */
- inst->rptr = (uint64_t)dm_vaddr - iv_len;
-
- cpt_inst_w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN;
-
- if (likely(iv_len)) {
- uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr +
- ROC_SE_OFF_CTRL_LEN);
- memcpy(iv_d, iv, 16);
- }
-
- /* iv offset is 0 */
- *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
- } else {
- void *m_vaddr = params->meta_buf.vaddr;
- uint32_t i, g_size_bytes, s_size_bytes;
- struct roc_se_sglist_comp *gather_comp;
- struct roc_se_sglist_comp *scatter_comp;
- uint8_t *in_buffer;
- uint32_t *iv_d;
-
- /* save space for offset and iv... */
- offset_vaddr = m_vaddr;
-
- m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + iv_len;
-
- cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;
-
- /* DPTR has SG list */
- in_buffer = m_vaddr;
-
- ((uint16_t *)in_buffer)[0] = 0;
- ((uint16_t *)in_buffer)[1] = 0;
-
- /* TODO Add error check if space will be sufficient */
- gather_comp =
- (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);
-
- /*
- * Input Gather List
- */
- i = 0;
-
- /* Offset control word */
-
- /* iv offset is 0 */
- *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
-
- i = fill_sg_comp(gather_comp, i, (uint64_t)offset_vaddr,
- ROC_SE_OFF_CTRL_LEN + iv_len);
-
- iv_d = (uint32_t *)((uint8_t *)offset_vaddr +
- ROC_SE_OFF_CTRL_LEN);
- memcpy(iv_d, iv, 16);
-
- /* Add input data */
- size = inputlen - iv_len;
- if (size) {
- i = fill_sg_comp_from_iov(gather_comp, i,
- params->src_iov, 0, &size,
- NULL, 0);
- if (unlikely(size)) {
- plt_dp_err("Insufficient buffer space,"
- " size %d needed",
- size);
- return -1;
- }
- }
- ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
- g_size_bytes =
- ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
-
- /*
- * Output Scatter List
- */
-
- i = 0;
- scatter_comp =
- (struct roc_se_sglist_comp *)((uint8_t *)gather_comp +
- g_size_bytes);
-
- /* IV */
- i = fill_sg_comp(scatter_comp, i,
- (uint64_t)offset_vaddr + ROC_SE_OFF_CTRL_LEN,
- iv_len);
-
- /* Add output data */
- size = outputlen - iv_len;
- if (size) {
- i = fill_sg_comp_from_iov(scatter_comp, i,
- params->dst_iov, 0, &size,
- NULL, 0);
-
- if (unlikely(size)) {
- plt_dp_err("Insufficient buffer space,"
- " size %d needed",
- size);
- return -1;
- }
- }
- ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- s_size_bytes =
- ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
-
- size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;
-
- /* This is DPTR len in case of SG mode */
- cpt_inst_w4.s.dlen = size;
-
- inst->dptr = (uint64_t)in_buffer;
- }
-
- if (unlikely((encr_offset >> 16))) {
- plt_dp_err("Offset not supported");
- plt_dp_err("enc_offset: %d", encr_offset);
- return -1;
- }
-
- inst->w4.u64 = cpt_inst_w4.u64;
-
- return 0;
-}
-
-static __rte_always_inline int
cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
struct roc_se_fc_params *params, struct cpt_inst_s *inst)
{
@@ -1749,8 +1551,8 @@ cpt_fc_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
if (likely(fc_type == ROC_SE_FC_GEN)) {
ret = cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, inst);
} else if (fc_type == ROC_SE_PDCP) {
- ret = cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params,
- inst);
+ ret = cpt_zuc_snow3g_prep(flags, d_offs, d_lens, fc_params,
+ inst);
} else if (fc_type == ROC_SE_KASUMI) {
ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, inst);
}
@@ -1778,8 +1580,8 @@ cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
if (likely(fc_type == ROC_SE_FC_GEN)) {
ret = cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, inst);
} else if (fc_type == ROC_SE_PDCP) {
- ret = cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params,
- inst);
+ ret = cpt_zuc_snow3g_prep(flags, d_offs, d_lens, fc_params,
+ inst);
} else if (fc_type == ROC_SE_KASUMI) {
ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params,
inst);
--
2.7.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [dpdk-dev] [PATCH 4/7] crypto/cnxk: remove redundant memcpy of IV for ZUC
2021-09-02 12:22 [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Anoob Joseph
` (2 preceding siblings ...)
2021-09-02 12:22 ` [dpdk-dev] [PATCH 3/7] crypto/cnxk: remove redundant snow3g dec Anoob Joseph
@ 2021-09-02 12:22 ` Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 5/7] crypto/cnxk: remove redundant assignment Anoob Joseph
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anoob Joseph @ 2021-09-02 12:22 UTC (permalink / raw)
To: Akhil Goyal, Jerin Jacob
Cc: Anoob Joseph, Archana Muniganti, Tejasree Kondoj, dev
Swap is not required for ZUC. Update IV updation sequence to remove the
redundant swap in case of ZUC.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/crypto/cnxk/cnxk_se.h | 55 +++++++++++++++++++++++--------------------
1 file changed, 30 insertions(+), 25 deletions(-)
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 9d1ce09..bbad289 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -36,6 +36,29 @@ struct cnxk_se_sess {
struct roc_se_ctx roc_se_ctx;
} __rte_cache_aligned;
+static inline void
+pdcp_iv_copy(uint8_t *iv_d, uint8_t *iv_s, const uint8_t pdcp_alg_type)
+{
+ uint32_t *iv_s_temp, iv_temp[4];
+ int j;
+
+ if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
+ /*
+ * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
+ * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
+ */
+
+ iv_s_temp = (uint32_t *)iv_s;
+
+ for (j = 0; j < 4; j++)
+ iv_temp[j] = iv_s_temp[3 - j];
+ memcpy(iv_d, iv_temp, 16);
+ } else {
+ /* ZUC doesn't need a swap */
+ memcpy(iv_d, iv_s, 16);
+ }
+}
+
static __rte_always_inline int
cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
{
@@ -954,13 +977,13 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
int32_t inputlen, outputlen;
struct roc_se_ctx *se_ctx;
uint32_t mac_len = 0;
- uint8_t pdcp_alg_type, j;
+ uint8_t pdcp_alg_type;
uint32_t encr_offset, auth_offset;
uint32_t encr_data_len, auth_data_len;
int flags, iv_len = 16;
uint64_t offset_ctrl;
uint64_t *offset_vaddr;
- uint32_t *iv_s, iv[4];
+ uint8_t *iv_s;
union cpt_inst_w4 cpt_inst_w4;
se_ctx = params->ctx_buf.vaddr;
@@ -1030,20 +1053,6 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
return -1;
}
- if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
- /*
- * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
- * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
- */
-
- for (j = 0; j < 4; j++)
- iv[j] = iv_s[3 - j];
- } else {
- /* ZUC doesn't need a swap */
- for (j = 0; j < 4; j++)
- iv[j] = iv_s[j];
- }
-
/*
* GP op header, lengths are expected in bits.
*/
@@ -1072,11 +1081,8 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
cpt_inst_w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN;
- if (likely(iv_len)) {
- uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr +
- ROC_SE_OFF_CTRL_LEN);
- memcpy(iv_d, iv, 16);
- }
+ uint8_t *iv_d = ((uint8_t *)offset_vaddr + ROC_SE_OFF_CTRL_LEN);
+ pdcp_iv_copy(iv_d, iv_s, pdcp_alg_type);
*offset_vaddr = offset_ctrl;
} else {
@@ -1085,7 +1091,7 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
struct roc_se_sglist_comp *gather_comp;
struct roc_se_sglist_comp *scatter_comp;
uint8_t *in_buffer;
- uint32_t *iv_d;
+ uint8_t *iv_d;
/* save space for iv */
offset_vaddr = m_vaddr;
@@ -1117,9 +1123,8 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
/* iv offset is 0 */
*offset_vaddr = offset_ctrl;
- iv_d = (uint32_t *)((uint8_t *)offset_vaddr +
- ROC_SE_OFF_CTRL_LEN);
- memcpy(iv_d, iv, 16);
+ iv_d = ((uint8_t *)offset_vaddr + ROC_SE_OFF_CTRL_LEN);
+ pdcp_iv_copy(iv_d, iv_s, pdcp_alg_type);
/* input data */
size = inputlen - iv_len;
--
2.7.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [dpdk-dev] [PATCH 5/7] crypto/cnxk: remove redundant assignment
2021-09-02 12:22 [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Anoob Joseph
` (3 preceding siblings ...)
2021-09-02 12:22 ` [dpdk-dev] [PATCH 4/7] crypto/cnxk: remove redundant memcpy of IV for ZUC Anoob Joseph
@ 2021-09-02 12:22 ` Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 6/7] crypto/cnxk: support for ucode API change Anoob Joseph
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anoob Joseph @ 2021-09-02 12:22 UTC (permalink / raw)
To: Akhil Goyal, Jerin Jacob
Cc: Anoob Joseph, Archana Muniganti, Tejasree Kondoj, dev
The assignment to -1 is not required. Remove the same.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 2 +-
drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 28055ac..cccca77 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -84,7 +84,7 @@ cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
struct cpt_inst_s *inst)
{
uint64_t cpt_op;
- int ret = -1;
+ int ret;
cpt_op = sess->cpt_op;
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 08f08c8..4c0eb12 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -20,7 +20,7 @@ cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
struct cpt_inst_s *inst)
{
uint64_t cpt_op;
- int ret = -1;
+ int ret;
cpt_op = sess->cpt_op;
--
2.7.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [dpdk-dev] [PATCH 6/7] crypto/cnxk: support for ucode API change
2021-09-02 12:22 [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Anoob Joseph
` (4 preceding siblings ...)
2021-09-02 12:22 ` [dpdk-dev] [PATCH 5/7] crypto/cnxk: remove redundant assignment Anoob Joseph
@ 2021-09-02 12:22 ` Anoob Joseph
2021-09-02 12:22 ` [dpdk-dev] [PATCH 7/7] crypto/cnxk: add dual submission in crypto_cn9k Anoob Joseph
2021-09-06 19:15 ` [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Akhil Goyal
7 siblings, 0 replies; 9+ messages in thread
From: Anoob Joseph @ 2021-09-02 12:22 UTC (permalink / raw)
To: Akhil Goyal, Jerin Jacob
Cc: Vidya Sagar Velumuri, Archana Muniganti, Tejasree Kondoj, dev
From: Vidya Sagar Velumuri <vvelumuri@marvell.com>
Add support for API changes in ucode 1.13
Signed-off-by: Vidya Sagar Velumuri <vvelumuri@marvell.com>
---
drivers/crypto/cnxk/cnxk_se.h | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index bbad289..aedc4bc 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -993,10 +993,7 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_ZUC_SNOW3G;
- /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
-
- cpt_inst_w4.s.opcode_minor = ((1 << 7) | (pdcp_alg_type << 5) |
- (0 << 4) | (0 << 3) | (flags & 0x7));
+ cpt_inst_w4.s.opcode_minor = se_ctx->template_w4.s.opcode_minor;
if (flags == 0x1) {
/*
@@ -1766,6 +1763,8 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
NULL)))
return -1;
+ if ((enc_type >= ROC_SE_ZUC_EEA3) && (enc_type <= ROC_SE_AES_CTR_EEA2))
+ roc_se_ctx_swap(&sess->roc_se_ctx);
return 0;
}
@@ -1869,6 +1868,10 @@ fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
a_form->digest_length)))
return -1;
+ if ((auth_type >= ROC_SE_ZUC_EIA3) &&
+ (auth_type <= ROC_SE_AES_CMAC_EIA2))
+ roc_se_ctx_swap(&sess->roc_se_ctx);
+
return 0;
}
--
2.7.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [dpdk-dev] [PATCH 7/7] crypto/cnxk: add dual submission in crypto_cn9k
2021-09-02 12:22 [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Anoob Joseph
` (5 preceding siblings ...)
2021-09-02 12:22 ` [dpdk-dev] [PATCH 6/7] crypto/cnxk: support for ucode API change Anoob Joseph
@ 2021-09-02 12:22 ` Anoob Joseph
2021-09-06 19:15 ` [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Akhil Goyal
7 siblings, 0 replies; 9+ messages in thread
From: Anoob Joseph @ 2021-09-02 12:22 UTC (permalink / raw)
To: Akhil Goyal, Jerin Jacob
Cc: Anoob Joseph, Archana Muniganti, Tejasree Kondoj, dev
Submit two instructions with one LMTST operation. Also updated dequeue
path to have local var for constants.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 155 +++++++++++++++++++++++++------
1 file changed, 127 insertions(+), 28 deletions(-)
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 4c0eb12..8ade197 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -5,6 +5,7 @@
#include <rte_cryptodev.h>
#include <rte_cryptodev_pmd.h>
#include <rte_event_crypto_adapter.h>
+#include <rte_vect.h>
#include "cn9k_cryptodev.h"
#include "cn9k_cryptodev_ops.h"
@@ -64,9 +65,8 @@ cn9k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
}
static inline int
-cn9k_cpt_prepare_instruction(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
- struct cpt_inflight_req *infl_req,
- struct cpt_inst_s *inst)
+cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
+ struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst)
{
int ret;
@@ -118,8 +118,8 @@ cn9k_cpt_prepare_instruction(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
}
static inline void
-cn9k_cpt_submit_instruction(struct cpt_inst_s *inst, uint64_t lmtline,
- uint64_t io_addr)
+cn9k_cpt_inst_submit(struct cpt_inst_s *inst, uint64_t lmtline,
+ uint64_t io_addr)
{
uint64_t lmt_status;
@@ -138,46 +138,144 @@ cn9k_cpt_submit_instruction(struct cpt_inst_s *inst, uint64_t lmtline,
} while (lmt_status == 0);
}
+static __plt_always_inline void
+cn9k_cpt_inst_submit_dual(struct cpt_inst_s *inst, uint64_t lmtline,
+ uint64_t io_addr)
+{
+ uint64_t lmt_status;
+
+ do {
+ /* Copy 2 CPT inst_s to LMTLINE */
+#if defined(RTE_ARCH_ARM64)
+ uint64_t *s = (uint64_t *)inst;
+ uint64_t *d = (uint64_t *)lmtline;
+
+ vst1q_u64(&d[0], vld1q_u64(&s[0]));
+ vst1q_u64(&d[2], vld1q_u64(&s[2]));
+ vst1q_u64(&d[4], vld1q_u64(&s[4]));
+ vst1q_u64(&d[6], vld1q_u64(&s[6]));
+ vst1q_u64(&d[8], vld1q_u64(&s[8]));
+ vst1q_u64(&d[10], vld1q_u64(&s[10]));
+ vst1q_u64(&d[12], vld1q_u64(&s[12]));
+ vst1q_u64(&d[14], vld1q_u64(&s[14]));
+#else
+ roc_lmt_mov_seg((void *)lmtline, inst, 8);
+#endif
+
+ /*
+ * Make sure compiler does not reorder memcpy and ldeor.
+ * LMTST transactions are always flushed from the write
+ * buffer immediately, a DMB is not required to push out
+ * LMTSTs.
+ */
+ rte_io_wmb();
+ lmt_status = roc_lmt_submit_ldeor(io_addr);
+ } while (lmt_status == 0);
+}
+
static uint16_t
cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
{
- struct cpt_inflight_req *infl_req;
+ struct cpt_inflight_req *infl_req_1, *infl_req_2;
+ struct cpt_inst_s inst[2] __rte_cache_aligned;
+ struct rte_crypto_op *op_1, *op_2;
uint16_t nb_allowed, count = 0;
struct cnxk_cpt_qp *qp = qptr;
struct pending_queue *pend_q;
- struct rte_crypto_op *op;
- struct cpt_inst_s inst;
+ uint64_t enq_tail;
int ret;
+ const uint32_t nb_desc = qp->lf.nb_desc;
+ const uint64_t lmt_base = qp->lf.lmt_base;
+ const uint64_t io_addr = qp->lf.io_addr;
+
pend_q = &qp->pend_q;
- inst.w0.u64 = 0;
- inst.w2.u64 = 0;
- inst.w3.u64 = 0;
+ /* Clear w0, w2, w3 of both inst */
+
+ inst[0].w0.u64 = 0;
+ inst[0].w2.u64 = 0;
+ inst[0].w3.u64 = 0;
+ inst[1].w0.u64 = 0;
+ inst[1].w2.u64 = 0;
+ inst[1].w3.u64 = 0;
nb_allowed = qp->lf.nb_desc - pend_q->pending_count;
nb_ops = RTE_MIN(nb_ops, nb_allowed);
- for (count = 0; count < nb_ops; count++) {
- op = ops[count];
- infl_req = &pend_q->req_queue[pend_q->enq_tail];
- infl_req->op_flags = 0;
+ enq_tail = pend_q->enq_tail;
+
+ if (unlikely(nb_ops & 1)) {
+ op_1 = ops[0];
+ infl_req_1 = &pend_q->req_queue[enq_tail];
+ infl_req_1->op_flags = 0;
- ret = cn9k_cpt_prepare_instruction(qp, op, infl_req, &inst);
+ ret = cn9k_cpt_inst_prep(qp, op_1, infl_req_1, &inst[0]);
if (unlikely(ret)) {
- plt_dp_err("Could not process op: %p", op);
+ plt_dp_err("Could not process op: %p", op_1);
+ return 0;
+ }
+
+ infl_req_1->cop = op_1;
+ infl_req_1->res.cn9k.compcode = CPT_COMP_NOT_DONE;
+ inst[0].res_addr = (uint64_t)&infl_req_1->res;
+
+ cn9k_cpt_inst_submit(&inst[0], lmt_base, io_addr);
+ MOD_INC(enq_tail, nb_desc);
+ count++;
+ }
+
+ while (count < nb_ops) {
+ op_1 = ops[count];
+ op_2 = ops[count + 1];
+
+ infl_req_1 = &pend_q->req_queue[enq_tail];
+ MOD_INC(enq_tail, nb_desc);
+ infl_req_2 = &pend_q->req_queue[enq_tail];
+ MOD_INC(enq_tail, nb_desc);
+
+ infl_req_1->cop = op_1;
+ infl_req_2->cop = op_2;
+ infl_req_1->op_flags = 0;
+ infl_req_2->op_flags = 0;
+
+ infl_req_1->res.cn9k.compcode = CPT_COMP_NOT_DONE;
+ inst[0].res_addr = (uint64_t)&infl_req_1->res;
+
+ infl_req_2->res.cn9k.compcode = CPT_COMP_NOT_DONE;
+ inst[1].res_addr = (uint64_t)&infl_req_2->res;
+
+ ret = cn9k_cpt_inst_prep(qp, op_1, infl_req_1, &inst[0]);
+ if (unlikely(ret)) {
+ plt_dp_err("Could not process op: %p", op_1);
+ if (enq_tail == 0)
+ enq_tail = nb_desc - 2;
+ else if (enq_tail == 1)
+ enq_tail = nb_desc - 1;
+ else
+ enq_tail--;
+ break;
+ }
+
+ ret = cn9k_cpt_inst_prep(qp, op_2, infl_req_2, &inst[1]);
+ if (unlikely(ret)) {
+ plt_dp_err("Could not process op: %p", op_2);
+ if (enq_tail == 0)
+ enq_tail = nb_desc - 1;
+ else
+ enq_tail--;
+
+ cn9k_cpt_inst_submit(&inst[0], lmt_base, io_addr);
+ count++;
break;
}
- infl_req->cop = op;
- infl_req->res.cn9k.compcode = CPT_COMP_NOT_DONE;
- inst.res_addr = (uint64_t)&infl_req->res;
+ cn9k_cpt_inst_submit_dual(&inst[0], lmt_base, io_addr);
- cn9k_cpt_submit_instruction(&inst, qp->lmtline.lmt_base,
- qp->lmtline.io_addr);
- MOD_INC(pend_q->enq_tail, qp->lf.nb_desc);
+ count += 2;
}
+ pend_q->enq_tail = enq_tail;
pend_q->pending_count += count;
pend_q->time_out = rte_get_timer_cycles() +
DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
@@ -219,7 +317,7 @@ cn9k_cpt_crypto_adapter_enqueue(uintptr_t tag_op, struct rte_crypto_op *op)
}
infl_req->op_flags = 0;
- ret = cn9k_cpt_prepare_instruction(qp, op, infl_req, &inst);
+ ret = cn9k_cpt_inst_prep(qp, op, infl_req, &inst);
if (unlikely(ret)) {
plt_dp_err("Could not process op: %p", op);
rte_mempool_put(qp->ca.req_mp, infl_req);
@@ -245,8 +343,7 @@ cn9k_cpt_crypto_adapter_enqueue(uintptr_t tag_op, struct rte_crypto_op *op)
if (!rsp_info->sched_type)
roc_sso_hws_head_wait(tag_op);
- cn9k_cpt_submit_instruction(&inst, qp->lmtline.lmt_base,
- qp->lmtline.io_addr);
+ cn9k_cpt_inst_submit(&inst, qp->lmtline.lmt_base, qp->lmtline.io_addr);
return 1;
}
@@ -347,14 +444,16 @@ cn9k_cpt_crypto_adapter_dequeue(uintptr_t get_work1)
static uint16_t
cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
{
+ struct cpt_inflight_req *infl_req;
struct cnxk_cpt_qp *qp = qptr;
struct pending_queue *pend_q;
- struct cpt_inflight_req *infl_req;
struct cpt_cn9k_res_s *res;
struct rte_crypto_op *cop;
uint32_t pq_deq_head;
int i;
+ const uint32_t nb_desc = qp->lf.nb_desc;
+
pend_q = &qp->pend_q;
nb_ops = RTE_MIN(nb_ops, pend_q->pending_count);
@@ -377,7 +476,7 @@ cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
break;
}
- MOD_INC(pq_deq_head, qp->lf.nb_desc);
+ MOD_INC(pq_deq_head, nb_desc);
cop = infl_req->cop;
--
2.7.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* Re: [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs
2021-09-02 12:22 [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs Anoob Joseph
` (6 preceding siblings ...)
2021-09-02 12:22 ` [dpdk-dev] [PATCH 7/7] crypto/cnxk: add dual submission in crypto_cn9k Anoob Joseph
@ 2021-09-06 19:15 ` Akhil Goyal
7 siblings, 0 replies; 9+ messages in thread
From: Akhil Goyal @ 2021-09-06 19:15 UTC (permalink / raw)
To: Anoob Joseph, Jerin Jacob Kollanukkaran
Cc: Anoob Joseph, Archana Muniganti, Tejasree Kondoj, dev
> Improvements and fixes to crypto_cn9k & crypto_cn10k PMDs.
> - Updated IPsec completion codes to match latest firmware
> - Updated ZUC API to match latest firmware
> - Added dual submission in crypto_cn9k PMD
>
> Depends on:
> http://patches.dpdk.org/project/dpdk/list/?series=18627
>
> Anoob Joseph (5):
> common/cnxk: update to v1.16 ucc codes
> crypto/cnxk: remove redundant snow3g dec
> crypto/cnxk: remove redundant memcpy of IV for ZUC
> crypto/cnxk: remove redundant assignment
> crypto/cnxk: add dual submission
>
> Vidya Sagar Velumuri (2):
> common/cnxk: update to v1.13 ZUC API
> crypto/cnxk: support for ucode API change
>
> drivers/common/cnxk/roc_ie_ot.h | 65 +++----
> drivers/common/cnxk/roc_se.c | 150 +++++++++++++--
> drivers/common/cnxk/roc_se.h | 37 +++-
> drivers/common/cnxk/version.map | 1 +
> drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 2 +-
> drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 157 +++++++++++++---
> drivers/crypto/cnxk/cnxk_se.h | 298 ++++++------------------------
> 7 files changed, 389 insertions(+), 321 deletions(-)
>
Series Acked-by: Akhil Goyal <gakhil@marvell.com>
Applied to dpdk-next-crypto
Thanks.
^ permalink raw reply [flat|nested] 9+ messages in thread