* [PATCH 0/10] crypto: Driver conversions for DMA alignment
@ 2022-12-02 9:19 Herbert Xu
2022-12-02 9:20 ` [PATCH 1/10] crypto: cavium - Set DMA alignment explicitly Herbert Xu
` (11 more replies)
0 siblings, 12 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:19 UTC (permalink / raw)
To: Catalin Marinas
Cc: Ard Biesheuvel, Will Deacon, Marc Zyngier, Arnd Bergmann,
Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM, LinuxKernelMailingList,
linux-kernel, David S. Miller, Linux Crypto Mailing List
These are the rest of the driver conversions in order for arm64
to safely lower the kmalloc alignment below that required for DMA.
My criteria for inclusion are:
1) the driver can be built on arm64.
2) the driver may perform DMA on the context structure.
I have worked through all the drivers in crypto but if you think
I've missed something please let me know.
Thanks,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH 1/10] crypto: cavium - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
@ 2022-12-02 9:20 ` Herbert Xu
2022-12-02 9:20 ` [PATCH 2/10] crypto: ccp " Herbert Xu
` (10 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:20 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/cavium/cpt/cptvf_algs.c | 10 +++++-----
drivers/crypto/cavium/nitrox/nitrox_aead.c | 12 ++++++------
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c
index ce3b91c612f0..9eca0c302186 100644
--- a/drivers/crypto/cavium/cpt/cptvf_algs.c
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.c
@@ -97,7 +97,7 @@ static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct cvm_req_ctx *rctx = skcipher_request_ctx(req);
+ struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct fc_context *fctx = &rctx->fctx;
u32 enc_iv_len = crypto_skcipher_ivsize(tfm);
struct cpt_request_info *req_info = &rctx->cpt_req;
@@ -151,7 +151,7 @@ static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
static inline u32 create_input_list(struct skcipher_request *req, u32 enc,
u32 enc_iv_len)
{
- struct cvm_req_ctx *rctx = skcipher_request_ctx(req);
+ struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct cpt_request_info *req_info = &rctx->cpt_req;
u32 argcnt = 0;
@@ -173,7 +173,7 @@ static inline void store_cb_info(struct skcipher_request *req,
static inline void create_output_list(struct skcipher_request *req,
u32 enc_iv_len)
{
- struct cvm_req_ctx *rctx = skcipher_request_ctx(req);
+ struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct cpt_request_info *req_info = &rctx->cpt_req;
u32 argcnt = 0;
@@ -193,7 +193,7 @@ static inline void create_output_list(struct skcipher_request *req,
static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct cvm_req_ctx *rctx = skcipher_request_ctx(req);
+ struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
u32 enc_iv_len = crypto_skcipher_ivsize(tfm);
struct fc_context *fctx = &rctx->fctx;
struct cpt_request_info *req_info = &rctx->cpt_req;
@@ -335,7 +335,7 @@ static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
static int cvm_enc_dec_init(struct crypto_skcipher *tfm)
{
- crypto_skcipher_set_reqsize(tfm, sizeof(struct cvm_req_ctx));
+ crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct cvm_req_ctx));
return 0;
}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c
index c93c4e41d267..0653484df23f 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_aead.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c
@@ -392,7 +392,7 @@ static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
{
- struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
+ struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
struct nitrox_aead_rctx *aead_rctx = &rctx->base;
unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
struct scatterlist *sg;
@@ -424,7 +424,7 @@ static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
static void nitrox_rfc4106_callback(void *arg, int err)
{
struct aead_request *areq = arg;
- struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
+ struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
free_src_sglist(nkreq);
@@ -441,7 +441,7 @@ static int nitrox_rfc4106_enc(struct aead_request *areq)
{
struct crypto_aead *aead = crypto_aead_reqtfm(areq);
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
- struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
+ struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
struct nitrox_aead_rctx *aead_rctx = &rctx->base;
struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
int ret;
@@ -472,7 +472,7 @@ static int nitrox_rfc4106_enc(struct aead_request *areq)
static int nitrox_rfc4106_dec(struct aead_request *areq)
{
struct crypto_aead *aead = crypto_aead_reqtfm(areq);
- struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
+ struct nitrox_crypto_ctx *nctx = crypto_aead_ctx_dma(aead);
struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
struct nitrox_aead_rctx *aead_rctx = &rctx->base;
struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
@@ -510,8 +510,8 @@ static int nitrox_rfc4106_init(struct crypto_aead *aead)
if (ret)
return ret;
- crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
- sizeof(struct nitrox_rfc4106_rctx));
+ crypto_aead_set_reqsize_dma(aead, sizeof(struct aead_request) +
+ sizeof(struct nitrox_rfc4106_rctx));
return 0;
}
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 2/10] crypto: ccp - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
2022-12-02 9:20 ` [PATCH 1/10] crypto: cavium - Set DMA alignment explicitly Herbert Xu
@ 2022-12-02 9:20 ` Herbert Xu
2022-12-02 9:20 ` [PATCH 3/10] crypto: ccree " Herbert Xu
` (9 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:20 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/ccp/ccp-crypto-aes-cmac.c | 21 +++++++++++----------
drivers/crypto/ccp/ccp-crypto-aes-galois.c | 12 ++++++------
drivers/crypto/ccp/ccp-crypto-aes-xts.c | 20 +++++++++++---------
drivers/crypto/ccp/ccp-crypto-aes.c | 29 +++++++++++++++--------------
drivers/crypto/ccp/ccp-crypto-des3.c | 17 +++++++++--------
drivers/crypto/ccp/ccp-crypto-main.c | 4 ++--
drivers/crypto/ccp/ccp-crypto-rsa.c | 18 +++++++++---------
drivers/crypto/ccp/ccp-crypto-sha.c | 26 +++++++++++++-------------
8 files changed, 76 insertions(+), 71 deletions(-)
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
index 11a305fa19e6..d8426bdf3190 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
@@ -25,7 +25,7 @@ static int ccp_aes_cmac_complete(struct crypto_async_request *async_req,
{
struct ahash_request *req = ahash_request_cast(async_req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req);
unsigned int digest_size = crypto_ahash_digestsize(tfm);
if (ret)
@@ -56,8 +56,8 @@ static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes,
unsigned int final)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ccp_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm);
+ struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req);
struct scatterlist *sg, *cmac_key_sg = NULL;
unsigned int block_size =
crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
@@ -182,7 +182,7 @@ static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes,
static int ccp_aes_cmac_init(struct ahash_request *req)
{
- struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req);
memset(rctx, 0, sizeof(*rctx));
@@ -219,7 +219,7 @@ static int ccp_aes_cmac_digest(struct ahash_request *req)
static int ccp_aes_cmac_export(struct ahash_request *req, void *out)
{
- struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req);
struct ccp_aes_cmac_exp_ctx state;
/* Don't let anything leak to 'out' */
@@ -238,7 +238,7 @@ static int ccp_aes_cmac_export(struct ahash_request *req, void *out)
static int ccp_aes_cmac_import(struct ahash_request *req, const void *in)
{
- struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req);
struct ccp_aes_cmac_exp_ctx state;
/* 'in' may not be aligned so memcpy to local variable */
@@ -256,7 +256,7 @@ static int ccp_aes_cmac_import(struct ahash_request *req, const void *in)
static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int key_len)
{
- struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+ struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm);
struct ccp_crypto_ahash_alg *alg =
ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm));
u64 k0_hi, k0_lo, k1_hi, k1_lo, k2_hi, k2_lo;
@@ -334,13 +334,14 @@ static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
static int ccp_aes_cmac_cra_init(struct crypto_tfm *tfm)
{
- struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_tfm_ctx_dma(tfm);
struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
ctx->complete = ccp_aes_cmac_complete;
ctx->u.aes.key_len = 0;
- crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_aes_cmac_req_ctx));
+ crypto_ahash_set_reqsize_dma(ahash,
+ sizeof(struct ccp_aes_cmac_req_ctx));
return 0;
}
@@ -382,7 +383,7 @@ int ccp_register_aes_cmac_algs(struct list_head *head)
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
base->cra_blocksize = AES_BLOCK_SIZE;
- base->cra_ctxsize = sizeof(struct ccp_ctx);
+ base->cra_ctxsize = sizeof(struct ccp_ctx) + crypto_dma_padding();
base->cra_priority = CCP_CRA_PRIORITY;
base->cra_init = ccp_aes_cmac_cra_init;
base->cra_module = THIS_MODULE;
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
index 1c1c939f5c39..b1dbb8cea559 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
@@ -29,7 +29,7 @@ static int ccp_aes_gcm_complete(struct crypto_async_request *async_req, int ret)
static int ccp_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
unsigned int key_len)
{
- struct ccp_ctx *ctx = crypto_aead_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_aead_ctx_dma(tfm);
switch (key_len) {
case AES_KEYSIZE_128:
@@ -76,8 +76,8 @@ static int ccp_aes_gcm_setauthsize(struct crypto_aead *tfm,
static int ccp_aes_gcm_crypt(struct aead_request *req, bool encrypt)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct ccp_ctx *ctx = crypto_aead_ctx(tfm);
- struct ccp_aes_req_ctx *rctx = aead_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_aead_ctx_dma(tfm);
+ struct ccp_aes_req_ctx *rctx = aead_request_ctx_dma(req);
struct scatterlist *iv_sg = NULL;
unsigned int iv_len = 0;
int i;
@@ -148,12 +148,12 @@ static int ccp_aes_gcm_decrypt(struct aead_request *req)
static int ccp_aes_gcm_cra_init(struct crypto_aead *tfm)
{
- struct ccp_ctx *ctx = crypto_aead_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_aead_ctx_dma(tfm);
ctx->complete = ccp_aes_gcm_complete;
ctx->u.aes.key_len = 0;
- crypto_aead_set_reqsize(tfm, sizeof(struct ccp_aes_req_ctx));
+ crypto_aead_set_reqsize_dma(tfm, sizeof(struct ccp_aes_req_ctx));
return 0;
}
@@ -176,7 +176,7 @@ static struct aead_alg ccp_aes_gcm_defaults = {
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ccp_ctx),
+ .cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = CCP_CRA_PRIORITY,
.cra_exit = ccp_aes_gcm_cra_exit,
.cra_module = THIS_MODULE,
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
index 6849261ca47d..93f735d6b02b 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
@@ -62,7 +62,7 @@ static struct ccp_unit_size_map xts_unit_sizes[] = {
static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret)
{
struct skcipher_request *req = skcipher_request_cast(async_req);
- struct ccp_aes_req_ctx *rctx = skcipher_request_ctx(req);
+ struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req);
if (ret)
return ret;
@@ -75,7 +75,7 @@ static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret)
static int ccp_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
{
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
unsigned int ccpversion = ccp_version();
int ret;
@@ -105,8 +105,8 @@ static int ccp_aes_xts_crypt(struct skcipher_request *req,
unsigned int encrypt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct ccp_aes_req_ctx *rctx = skcipher_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
+ struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req);
unsigned int ccpversion = ccp_version();
unsigned int fallback = 0;
unsigned int unit;
@@ -196,7 +196,7 @@ static int ccp_aes_xts_decrypt(struct skcipher_request *req)
static int ccp_aes_xts_init_tfm(struct crypto_skcipher *tfm)
{
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
struct crypto_skcipher *fallback_tfm;
ctx->complete = ccp_aes_xts_complete;
@@ -210,15 +210,16 @@ static int ccp_aes_xts_init_tfm(struct crypto_skcipher *tfm)
}
ctx->u.aes.tfm_skcipher = fallback_tfm;
- crypto_skcipher_set_reqsize(tfm, sizeof(struct ccp_aes_req_ctx) +
- crypto_skcipher_reqsize(fallback_tfm));
+ crypto_skcipher_set_reqsize_dma(tfm,
+ sizeof(struct ccp_aes_req_ctx) +
+ crypto_skcipher_reqsize(fallback_tfm));
return 0;
}
static void ccp_aes_xts_exit_tfm(struct crypto_skcipher *tfm)
{
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
crypto_free_skcipher(ctx->u.aes.tfm_skcipher);
}
@@ -246,7 +247,8 @@ static int ccp_register_aes_xts_alg(struct list_head *head,
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
alg->base.cra_blocksize = AES_BLOCK_SIZE;
- alg->base.cra_ctxsize = sizeof(struct ccp_ctx);
+ alg->base.cra_ctxsize = sizeof(struct ccp_ctx) +
+ crypto_dma_padding();
alg->base.cra_priority = CCP_CRA_PRIORITY;
alg->base.cra_module = THIS_MODULE;
diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c
index bed331953ff9..918e223f21b6 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes.c
@@ -22,8 +22,9 @@
static int ccp_aes_complete(struct crypto_async_request *async_req, int ret)
{
struct skcipher_request *req = skcipher_request_cast(async_req);
- struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
- struct ccp_aes_req_ctx *rctx = skcipher_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(
+ crypto_skcipher_reqtfm(req));
+ struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req);
if (ret)
return ret;
@@ -38,7 +39,7 @@ static int ccp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
{
struct ccp_crypto_skcipher_alg *alg = ccp_crypto_skcipher_alg(tfm);
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
switch (key_len) {
case AES_KEYSIZE_128:
@@ -65,8 +66,8 @@ static int ccp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
static int ccp_aes_crypt(struct skcipher_request *req, bool encrypt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct ccp_aes_req_ctx *rctx = skcipher_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
+ struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct scatterlist *iv_sg = NULL;
unsigned int iv_len = 0;
@@ -118,7 +119,7 @@ static int ccp_aes_decrypt(struct skcipher_request *req)
static int ccp_aes_init_tfm(struct crypto_skcipher *tfm)
{
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
ctx->complete = ccp_aes_complete;
ctx->u.aes.key_len = 0;
@@ -132,7 +133,7 @@ static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req,
int ret)
{
struct skcipher_request *req = skcipher_request_cast(async_req);
- struct ccp_aes_req_ctx *rctx = skcipher_request_ctx(req);
+ struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req);
/* Restore the original pointer */
req->iv = rctx->rfc3686_info;
@@ -143,7 +144,7 @@ static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req,
static int ccp_aes_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
{
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
if (key_len < CTR_RFC3686_NONCE_SIZE)
return -EINVAL;
@@ -157,8 +158,8 @@ static int ccp_aes_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key,
static int ccp_aes_rfc3686_crypt(struct skcipher_request *req, bool encrypt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct ccp_aes_req_ctx *rctx = skcipher_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
+ struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req);
u8 *iv;
/* Initialize the CTR block */
@@ -190,12 +191,12 @@ static int ccp_aes_rfc3686_decrypt(struct skcipher_request *req)
static int ccp_aes_rfc3686_init_tfm(struct crypto_skcipher *tfm)
{
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
ctx->complete = ccp_aes_rfc3686_complete;
ctx->u.aes.key_len = 0;
- crypto_skcipher_set_reqsize(tfm, sizeof(struct ccp_aes_req_ctx));
+ crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct ccp_aes_req_ctx));
return 0;
}
@@ -213,7 +214,7 @@ static const struct skcipher_alg ccp_aes_defaults = {
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK,
.base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ccp_ctx),
+ .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING,
.base.cra_priority = CCP_CRA_PRIORITY,
.base.cra_module = THIS_MODULE,
};
@@ -231,7 +232,7 @@ static const struct skcipher_alg ccp_aes_rfc3686_defaults = {
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK,
.base.cra_blocksize = CTR_RFC3686_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ccp_ctx),
+ .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING,
.base.cra_priority = CCP_CRA_PRIORITY,
.base.cra_module = THIS_MODULE,
};
diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c
index 278636ed251a..afae30adb703 100644
--- a/drivers/crypto/ccp/ccp-crypto-des3.c
+++ b/drivers/crypto/ccp/ccp-crypto-des3.c
@@ -21,8 +21,9 @@
static int ccp_des3_complete(struct crypto_async_request *async_req, int ret)
{
struct skcipher_request *req = skcipher_request_cast(async_req);
- struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
- struct ccp_des3_req_ctx *rctx = skcipher_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(
+ crypto_skcipher_reqtfm(req));
+ struct ccp_des3_req_ctx *rctx = skcipher_request_ctx_dma(req);
if (ret)
return ret;
@@ -37,7 +38,7 @@ static int ccp_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
{
struct ccp_crypto_skcipher_alg *alg = ccp_crypto_skcipher_alg(tfm);
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
int err;
err = verify_skcipher_des3_key(tfm, key);
@@ -60,8 +61,8 @@ static int ccp_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
static int ccp_des3_crypt(struct skcipher_request *req, bool encrypt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct ccp_des3_req_ctx *rctx = skcipher_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
+ struct ccp_des3_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct scatterlist *iv_sg = NULL;
unsigned int iv_len = 0;
@@ -114,12 +115,12 @@ static int ccp_des3_decrypt(struct skcipher_request *req)
static int ccp_des3_init_tfm(struct crypto_skcipher *tfm)
{
- struct ccp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
ctx->complete = ccp_des3_complete;
ctx->u.des3.key_len = 0;
- crypto_skcipher_set_reqsize(tfm, sizeof(struct ccp_des3_req_ctx));
+ crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct ccp_des3_req_ctx));
return 0;
}
@@ -137,7 +138,7 @@ static const struct skcipher_alg ccp_des3_defaults = {
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK,
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ccp_ctx),
+ .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING,
.base.cra_priority = CCP_CRA_PRIORITY,
.base.cra_module = THIS_MODULE,
};
diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c
index dd86d2650bea..73442a382f68 100644
--- a/drivers/crypto/ccp/ccp-crypto-main.c
+++ b/drivers/crypto/ccp/ccp-crypto-main.c
@@ -139,7 +139,7 @@ static void ccp_crypto_complete(void *data, int err)
struct ccp_crypto_cmd *crypto_cmd = data;
struct ccp_crypto_cmd *held, *next, *backlog;
struct crypto_async_request *req = crypto_cmd->req;
- struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm);
+ struct ccp_ctx *ctx = crypto_tfm_ctx_dma(req->tfm);
int ret;
if (err == -EINPROGRESS) {
@@ -183,7 +183,7 @@ static void ccp_crypto_complete(void *data, int err)
break;
/* Error occurred, report it and get the next entry */
- ctx = crypto_tfm_ctx(held->req->tfm);
+ ctx = crypto_tfm_ctx_dma(held->req->tfm);
if (ctx->complete)
ret = ctx->complete(held->req, ret);
held->req->complete(held->req, ret);
diff --git a/drivers/crypto/ccp/ccp-crypto-rsa.c b/drivers/crypto/ccp/ccp-crypto-rsa.c
index 1223ac70aea2..a14f85512cf4 100644
--- a/drivers/crypto/ccp/ccp-crypto-rsa.c
+++ b/drivers/crypto/ccp/ccp-crypto-rsa.c
@@ -44,7 +44,7 @@ static inline int ccp_copy_and_save_keypart(u8 **kpbuf, unsigned int *kplen,
static int ccp_rsa_complete(struct crypto_async_request *async_req, int ret)
{
struct akcipher_request *req = akcipher_request_cast(async_req);
- struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx(req);
+ struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx_dma(req);
if (ret)
return ret;
@@ -56,7 +56,7 @@ static int ccp_rsa_complete(struct crypto_async_request *async_req, int ret)
static unsigned int ccp_rsa_maxsize(struct crypto_akcipher *tfm)
{
- struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
+ struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm);
return ctx->u.rsa.n_len;
}
@@ -64,8 +64,8 @@ static unsigned int ccp_rsa_maxsize(struct crypto_akcipher *tfm)
static int ccp_rsa_crypt(struct akcipher_request *req, bool encrypt)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
- struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
- struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx(req);
+ struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm);
+ struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx_dma(req);
int ret = 0;
memset(&rctx->cmd, 0, sizeof(rctx->cmd));
@@ -126,7 +126,7 @@ static void ccp_rsa_free_key_bufs(struct ccp_ctx *ctx)
static int ccp_rsa_setkey(struct crypto_akcipher *tfm, const void *key,
unsigned int keylen, bool private)
{
- struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
+ struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm);
struct rsa_key raw_key;
int ret;
@@ -192,9 +192,9 @@ static int ccp_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
static int ccp_rsa_init_tfm(struct crypto_akcipher *tfm)
{
- struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
+ struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm);
- akcipher_set_reqsize(tfm, sizeof(struct ccp_rsa_req_ctx));
+ akcipher_set_reqsize_dma(tfm, sizeof(struct ccp_rsa_req_ctx));
ctx->complete = ccp_rsa_complete;
return 0;
@@ -202,7 +202,7 @@ static int ccp_rsa_init_tfm(struct crypto_akcipher *tfm)
static void ccp_rsa_exit_tfm(struct crypto_akcipher *tfm)
{
- struct ccp_ctx *ctx = crypto_tfm_ctx(&tfm->base);
+ struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm);
ccp_rsa_free_key_bufs(ctx);
}
@@ -220,7 +220,7 @@ static struct akcipher_alg ccp_rsa_defaults = {
.cra_driver_name = "rsa-ccp",
.cra_priority = CCP_CRA_PRIORITY,
.cra_module = THIS_MODULE,
- .cra_ctxsize = 2 * sizeof(struct ccp_ctx),
+ .cra_ctxsize = 2 * sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING,
},
};
diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c
index 74fa5360e722..fa3ae8e78f6f 100644
--- a/drivers/crypto/ccp/ccp-crypto-sha.c
+++ b/drivers/crypto/ccp/ccp-crypto-sha.c
@@ -28,7 +28,7 @@ static int ccp_sha_complete(struct crypto_async_request *async_req, int ret)
{
struct ahash_request *req = ahash_request_cast(async_req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req);
unsigned int digest_size = crypto_ahash_digestsize(tfm);
if (ret)
@@ -59,8 +59,8 @@ static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes,
unsigned int final)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ccp_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm);
+ struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req);
struct scatterlist *sg;
unsigned int block_size =
crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
@@ -182,8 +182,8 @@ static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes,
static int ccp_sha_init(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ccp_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm);
+ struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req);
struct ccp_crypto_ahash_alg *alg =
ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm));
unsigned int block_size =
@@ -231,7 +231,7 @@ static int ccp_sha_digest(struct ahash_request *req)
static int ccp_sha_export(struct ahash_request *req, void *out)
{
- struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req);
struct ccp_sha_exp_ctx state;
/* Don't let anything leak to 'out' */
@@ -252,7 +252,7 @@ static int ccp_sha_export(struct ahash_request *req, void *out)
static int ccp_sha_import(struct ahash_request *req, const void *in)
{
- struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req);
+ struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req);
struct ccp_sha_exp_ctx state;
/* 'in' may not be aligned so memcpy to local variable */
@@ -272,7 +272,7 @@ static int ccp_sha_import(struct ahash_request *req, const void *in)
static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int key_len)
{
- struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+ struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm);
struct crypto_shash *shash = ctx->u.sha.hmac_tfm;
unsigned int block_size = crypto_shash_blocksize(shash);
unsigned int digest_size = crypto_shash_digestsize(shash);
@@ -313,13 +313,13 @@ static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
static int ccp_sha_cra_init(struct crypto_tfm *tfm)
{
- struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
+ struct ccp_ctx *ctx = crypto_ahash_ctx_dma(ahash);
ctx->complete = ccp_sha_complete;
ctx->u.sha.key_len = 0;
- crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_sha_req_ctx));
+ crypto_ahash_set_reqsize_dma(ahash, sizeof(struct ccp_sha_req_ctx));
return 0;
}
@@ -330,7 +330,7 @@ static void ccp_sha_cra_exit(struct crypto_tfm *tfm)
static int ccp_hmac_sha_cra_init(struct crypto_tfm *tfm)
{
- struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_tfm_ctx_dma(tfm);
struct ccp_crypto_ahash_alg *alg = ccp_crypto_ahash_alg(tfm);
struct crypto_shash *hmac_tfm;
@@ -348,7 +348,7 @@ static int ccp_hmac_sha_cra_init(struct crypto_tfm *tfm)
static void ccp_hmac_sha_cra_exit(struct crypto_tfm *tfm)
{
- struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct ccp_ctx *ctx = crypto_tfm_ctx_dma(tfm);
if (ctx->u.sha.hmac_tfm)
crypto_free_shash(ctx->u.sha.hmac_tfm);
@@ -492,7 +492,7 @@ static int ccp_register_sha_alg(struct list_head *head,
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
base->cra_blocksize = def->block_size;
- base->cra_ctxsize = sizeof(struct ccp_ctx);
+ base->cra_ctxsize = sizeof(struct ccp_ctx) + crypto_dma_padding();
base->cra_priority = CCP_CRA_PRIORITY;
base->cra_init = ccp_sha_cra_init;
base->cra_exit = ccp_sha_cra_exit;
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 3/10] crypto: ccree - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
2022-12-02 9:20 ` [PATCH 1/10] crypto: cavium - Set DMA alignment explicitly Herbert Xu
2022-12-02 9:20 ` [PATCH 2/10] crypto: ccp " Herbert Xu
@ 2022-12-02 9:20 ` Herbert Xu
2022-12-02 9:20 ` [PATCH 4/10] crypto: chelsio " Herbert Xu
` (8 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:20 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/ccree/cc_aead.c | 62 ++++++++++++-------------
drivers/crypto/ccree/cc_buffer_mgr.c | 18 +++----
drivers/crypto/ccree/cc_hash.c | 86 +++++++++++++++++------------------
3 files changed, 83 insertions(+), 83 deletions(-)
diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c
index 35794c7271fb..109ffb375fc6 100644
--- a/drivers/crypto/ccree/cc_aead.c
+++ b/drivers/crypto/ccree/cc_aead.c
@@ -138,7 +138,7 @@ static int cc_aead_init(struct crypto_aead *tfm)
ctx->flow_mode = cc_alg->flow_mode;
ctx->auth_mode = cc_alg->auth_mode;
ctx->drvdata = cc_alg->drvdata;
- crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx));
+ crypto_aead_set_reqsize_dma(tfm, sizeof(struct aead_req_ctx));
/* Allocate key buffer, cache line aligned */
ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
@@ -208,7 +208,7 @@ static int cc_aead_init(struct crypto_aead *tfm)
static void cc_aead_complete(struct device *dev, void *cc_req, int err)
{
struct aead_request *areq = (struct aead_request *)cc_req;
- struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);
struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
@@ -723,7 +723,7 @@ static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,
{
struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);
enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;
unsigned int idx = *seq_size;
struct device *dev = drvdata_to_dev(ctx->drvdata);
@@ -762,7 +762,7 @@ static void cc_proc_authen_desc(struct aead_request *areq,
struct cc_hw_desc desc[],
unsigned int *seq_size, int direct)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);
enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
unsigned int idx = *seq_size;
struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
@@ -827,7 +827,7 @@ static void cc_proc_cipher_desc(struct aead_request *areq,
unsigned int *seq_size)
{
unsigned int idx = *seq_size;
- struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);
enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
@@ -873,7 +873,7 @@ static void cc_proc_digest_desc(struct aead_request *req,
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
unsigned int idx = *seq_size;
unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
@@ -923,7 +923,7 @@ static void cc_set_cipher_desc(struct aead_request *req,
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
unsigned int hw_iv_size = req_ctx->hw_iv_size;
unsigned int idx = *seq_size;
int direct = req_ctx->gen_ctx.op_type;
@@ -965,7 +965,7 @@ static void cc_set_cipher_desc(struct aead_request *req,
static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],
unsigned int *seq_size, unsigned int data_flow_mode)
{
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
int direct = req_ctx->gen_ctx.op_type;
unsigned int idx = *seq_size;
@@ -1082,7 +1082,7 @@ static void cc_proc_header_desc(struct aead_request *req,
struct cc_hw_desc desc[],
unsigned int *seq_size)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
unsigned int idx = *seq_size;
/* Hash associated data */
@@ -1158,7 +1158,7 @@ static void cc_proc_scheme_desc(struct aead_request *req,
static void cc_mlli_to_sram(struct aead_request *req,
struct cc_hw_desc desc[], unsigned int *seq_size)
{
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
@@ -1212,7 +1212,7 @@ static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
int direct = req_ctx->gen_ctx.op_type;
unsigned int data_flow_mode =
cc_get_data_flow(direct, ctx->flow_mode,
@@ -1265,7 +1265,7 @@ cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
int direct = req_ctx->gen_ctx.op_type;
unsigned int data_flow_mode =
cc_get_data_flow(direct, ctx->flow_mode,
@@ -1312,7 +1312,7 @@ static int validate_data_size(struct cc_aead_ctx *ctx,
enum drv_crypto_direction direct,
struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
struct device *dev = drvdata_to_dev(ctx->drvdata);
unsigned int assoclen = areq_ctx->assoclen;
unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
@@ -1411,7 +1411,7 @@ static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
unsigned int idx = *seq_size;
unsigned int cipher_flow_mode;
dma_addr_t mac_result;
@@ -1533,7 +1533,7 @@ static int config_ccm_adata(struct aead_request *req)
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
//unsigned int size_of_a = 0, rem_a_size = 0;
unsigned int lp = req->iv[0];
/* Note: The code assume that req->iv[0] already contains the value
@@ -1591,7 +1591,7 @@ static void cc_proc_rfc4309_ccm(struct aead_request *req)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
/* L' */
memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);
@@ -1615,7 +1615,7 @@ static void cc_set_ghash_desc(struct aead_request *req,
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
unsigned int idx = *seq_size;
/* load key to AES*/
@@ -1693,7 +1693,7 @@ static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
unsigned int idx = *seq_size;
/* load key to AES*/
@@ -1730,7 +1730,7 @@ static void cc_proc_gcm_result(struct aead_request *req,
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
dma_addr_t mac_result;
unsigned int idx = *seq_size;
@@ -1792,7 +1792,7 @@ static void cc_proc_gcm_result(struct aead_request *req,
static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],
unsigned int *seq_size)
{
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
unsigned int cipher_flow_mode;
//in RFC4543 no data to encrypt. just copy data from src to dest.
@@ -1830,7 +1830,7 @@ static int config_gcm_context(struct aead_request *req)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *req_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
struct device *dev = drvdata_to_dev(ctx->drvdata);
unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
@@ -1879,7 +1879,7 @@ static void cc_proc_rfc4_gcm(struct aead_request *req)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,
ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);
@@ -1896,7 +1896,7 @@ static int cc_proc_aead(struct aead_request *req,
struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
struct device *dev = drvdata_to_dev(ctx->drvdata);
struct cc_crypto_req cc_req = {};
@@ -2019,7 +2019,7 @@ static int cc_proc_aead(struct aead_request *req,
static int cc_aead_encrypt(struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc;
memset(areq_ctx, 0, sizeof(*areq_ctx));
@@ -2039,7 +2039,7 @@ static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
{
/* Very similar to cc_aead_encrypt() above. */
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc;
rc = crypto_ipsec_check_assoclen(req->assoclen);
@@ -2063,7 +2063,7 @@ static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
static int cc_aead_decrypt(struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc;
memset(areq_ctx, 0, sizeof(*areq_ctx));
@@ -2081,7 +2081,7 @@ static int cc_aead_decrypt(struct aead_request *req)
static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc;
rc = crypto_ipsec_check_assoclen(req->assoclen);
@@ -2193,7 +2193,7 @@ static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc;
rc = crypto_ipsec_check_assoclen(req->assoclen);
@@ -2217,7 +2217,7 @@ static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc;
rc = crypto_ipsec_check_assoclen(req->assoclen);
@@ -2244,7 +2244,7 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc;
rc = crypto_ipsec_check_assoclen(req->assoclen);
@@ -2268,7 +2268,7 @@ static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc;
rc = crypto_ipsec_check_assoclen(req->assoclen);
diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c
index 9efd88f871d1..bcca55bff910 100644
--- a/drivers/crypto/ccree/cc_buffer_mgr.c
+++ b/drivers/crypto/ccree/cc_buffer_mgr.c
@@ -52,7 +52,7 @@ static inline char *cc_dma_buf_type(enum cc_req_dma_buf_type type)
static void cc_copy_mac(struct device *dev, struct aead_request *req,
enum cc_sg_cpy_direct dir)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
u32 skip = req->assoclen + req->cryptlen;
cc_copy_sg_portion(dev, areq_ctx->backup_mac, req->src,
@@ -456,7 +456,7 @@ int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx,
void cc_unmap_aead_request(struct device *dev, struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
unsigned int hw_iv_size = areq_ctx->hw_iv_size;
struct cc_drvdata *drvdata = dev_get_drvdata(dev);
int src_direction = (req->src != req->dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL);
@@ -546,7 +546,7 @@ static int cc_aead_chain_iv(struct cc_drvdata *drvdata,
struct buffer_array *sg_data,
bool is_last, bool do_chain)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
unsigned int hw_iv_size = areq_ctx->hw_iv_size;
struct device *dev = drvdata_to_dev(drvdata);
gfp_t flags = cc_gfp_flags(&req->base);
@@ -586,7 +586,7 @@ static int cc_aead_chain_assoc(struct cc_drvdata *drvdata,
struct buffer_array *sg_data,
bool is_last, bool do_chain)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
int rc = 0;
int mapped_nents = 0;
struct device *dev = drvdata_to_dev(drvdata);
@@ -652,7 +652,7 @@ static int cc_aead_chain_assoc(struct cc_drvdata *drvdata,
static void cc_prepare_aead_data_dlli(struct aead_request *req,
u32 *src_last_bytes, u32 *dst_last_bytes)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
unsigned int authsize = areq_ctx->req_authsize;
struct scatterlist *sg;
@@ -678,7 +678,7 @@ static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata,
u32 *src_last_bytes, u32 *dst_last_bytes,
bool is_last_table)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
unsigned int authsize = areq_ctx->req_authsize;
struct device *dev = drvdata_to_dev(drvdata);
@@ -790,7 +790,7 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata,
struct buffer_array *sg_data,
bool is_last_table, bool do_chain)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
struct device *dev = drvdata_to_dev(drvdata);
enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
unsigned int authsize = areq_ctx->req_authsize;
@@ -895,7 +895,7 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata,
static void cc_update_aead_mlli_nents(struct cc_drvdata *drvdata,
struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
u32 curr_mlli_size = 0;
if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) {
@@ -945,7 +945,7 @@ static void cc_update_aead_mlli_nents(struct cc_drvdata *drvdata,
int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req)
{
- struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
+ struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
struct mlli_params *mlli_params = &areq_ctx->mlli_params;
struct device *dev = drvdata_to_dev(drvdata);
struct buffer_array sg_data;
diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c
index 683c9a430e11..f418162932fe 100644
--- a/drivers/crypto/ccree/cc_hash.c
+++ b/drivers/crypto/ccree/cc_hash.c
@@ -283,9 +283,9 @@ static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state,
static void cc_update_complete(struct device *dev, void *cc_req, int err)
{
struct ahash_request *req = (struct ahash_request *)cc_req;
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
dev_dbg(dev, "req=%pK\n", req);
@@ -301,9 +301,9 @@ static void cc_update_complete(struct device *dev, void *cc_req, int err)
static void cc_digest_complete(struct device *dev, void *cc_req, int err)
{
struct ahash_request *req = (struct ahash_request *)cc_req;
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
u32 digestsize = crypto_ahash_digestsize(tfm);
dev_dbg(dev, "req=%pK\n", req);
@@ -321,9 +321,9 @@ static void cc_digest_complete(struct device *dev, void *cc_req, int err)
static void cc_hash_complete(struct device *dev, void *cc_req, int err)
{
struct ahash_request *req = (struct ahash_request *)cc_req;
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
u32 digestsize = crypto_ahash_digestsize(tfm);
dev_dbg(dev, "req=%pK\n", req);
@@ -341,9 +341,9 @@ static void cc_hash_complete(struct device *dev, void *cc_req, int err)
static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req,
int idx)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
u32 digestsize = crypto_ahash_digestsize(tfm);
/* Get final MAC result */
@@ -364,9 +364,9 @@ static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req,
static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req,
int idx)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
u32 digestsize = crypto_ahash_digestsize(tfm);
/* store the hash digest result in the context */
@@ -417,9 +417,9 @@ static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req,
static int cc_hash_digest(struct ahash_request *req)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
u32 digestsize = crypto_ahash_digestsize(tfm);
struct scatterlist *src = req->src;
unsigned int nbytes = req->nbytes;
@@ -555,9 +555,9 @@ static int cc_restore_hash(struct cc_hw_desc *desc, struct cc_hash_ctx *ctx,
static int cc_hash_update(struct ahash_request *req)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);
struct scatterlist *src = req->src;
unsigned int nbytes = req->nbytes;
@@ -631,9 +631,9 @@ static int cc_hash_update(struct ahash_request *req)
static int cc_do_finup(struct ahash_request *req, bool update)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
u32 digestsize = crypto_ahash_digestsize(tfm);
struct scatterlist *src = req->src;
unsigned int nbytes = req->nbytes;
@@ -711,9 +711,9 @@ static int cc_hash_final(struct ahash_request *req)
static int cc_hash_init(struct ahash_request *req)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
dev_dbg(dev, "===== init (%d) ====\n", req->nbytes);
@@ -736,7 +736,7 @@ static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key,
u32 larval_addr;
struct device *dev;
- ctx = crypto_ahash_ctx(ahash);
+ ctx = crypto_ahash_ctx_dma(ahash);
dev = drvdata_to_dev(ctx->drvdata);
dev_dbg(dev, "start keylen: %d", keylen);
@@ -922,7 +922,7 @@ static int cc_xcbc_setkey(struct crypto_ahash *ahash,
const u8 *key, unsigned int keylen)
{
struct cc_crypto_req cc_req = {};
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
struct device *dev = drvdata_to_dev(ctx->drvdata);
int rc = 0;
unsigned int idx = 0;
@@ -1007,7 +1007,7 @@ static int cc_xcbc_setkey(struct crypto_ahash *ahash,
static int cc_cmac_setkey(struct crypto_ahash *ahash,
const u8 *key, unsigned int keylen)
{
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
struct device *dev = drvdata_to_dev(ctx->drvdata);
dev_dbg(dev, "===== setkey (%d) ====\n", keylen);
@@ -1109,7 +1109,7 @@ static int cc_alloc_ctx(struct cc_hash_ctx *ctx)
static int cc_get_hash_len(struct crypto_tfm *tfm)
{
- struct cc_hash_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
if (ctx->hash_mode == DRV_HASH_SM3)
return CC_SM3_HASH_LEN_SIZE;
@@ -1119,7 +1119,7 @@ static int cc_get_hash_len(struct crypto_tfm *tfm)
static int cc_cra_init(struct crypto_tfm *tfm)
{
- struct cc_hash_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
struct hash_alg_common *hash_alg_common =
container_of(tfm->__crt_alg, struct hash_alg_common, base);
struct ahash_alg *ahash_alg =
@@ -1127,8 +1127,8 @@ static int cc_cra_init(struct crypto_tfm *tfm)
struct cc_hash_alg *cc_alg =
container_of(ahash_alg, struct cc_hash_alg, ahash_alg);
- crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
- sizeof(struct ahash_req_ctx));
+ crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
+ sizeof(struct ahash_req_ctx));
ctx->hash_mode = cc_alg->hash_mode;
ctx->hw_mode = cc_alg->hw_mode;
@@ -1140,7 +1140,7 @@ static int cc_cra_init(struct crypto_tfm *tfm)
static void cc_cra_exit(struct crypto_tfm *tfm)
{
- struct cc_hash_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
dev_dbg(dev, "cc_cra_exit");
@@ -1149,9 +1149,9 @@ static void cc_cra_exit(struct crypto_tfm *tfm)
static int cc_mac_update(struct ahash_request *req)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);
struct cc_crypto_req cc_req = {};
@@ -1217,9 +1217,9 @@ static int cc_mac_update(struct ahash_request *req)
static int cc_mac_final(struct ahash_request *req)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
struct cc_crypto_req cc_req = {};
struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
@@ -1338,9 +1338,9 @@ static int cc_mac_final(struct ahash_request *req)
static int cc_mac_finup(struct ahash_request *req)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
struct cc_crypto_req cc_req = {};
struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
@@ -1419,9 +1419,9 @@ static int cc_mac_finup(struct ahash_request *req)
static int cc_mac_digest(struct ahash_request *req)
{
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
u32 digestsize = crypto_ahash_digestsize(tfm);
struct cc_crypto_req cc_req = {};
@@ -1499,8 +1499,8 @@ static int cc_mac_digest(struct ahash_request *req)
static int cc_hash_export(struct ahash_request *req, void *out)
{
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
u8 *curr_buff = cc_hash_buf(state);
u32 curr_buff_cnt = *cc_hash_buf_cnt(state);
const u32 tmp = CC_EXPORT_MAGIC;
@@ -1525,9 +1525,9 @@ static int cc_hash_export(struct ahash_request *req, void *out)
static int cc_hash_import(struct ahash_request *req, const void *in)
{
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
struct device *dev = drvdata_to_dev(ctx->drvdata);
- struct ahash_req_ctx *state = ahash_request_ctx(req);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
u32 tmp;
memcpy(&tmp, in, sizeof(u32));
@@ -1846,7 +1846,7 @@ static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template,
template->driver_name);
}
alg->cra_module = THIS_MODULE;
- alg->cra_ctxsize = sizeof(struct cc_hash_ctx);
+ alg->cra_ctxsize = sizeof(struct cc_hash_ctx) + crypto_dma_padding();
alg->cra_priority = CC_CRA_PRIO;
alg->cra_blocksize = template->blocksize;
alg->cra_alignmask = 0;
@@ -2073,9 +2073,9 @@ static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],
unsigned int *seq_size)
{
unsigned int idx = *seq_size;
- struct ahash_req_ctx *state = ahash_request_ctx(areq);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(areq);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
/* Setup XCBC MAC K1 */
hw_desc_init(&desc[idx]);
@@ -2130,9 +2130,9 @@ static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],
unsigned int *seq_size)
{
unsigned int idx = *seq_size;
- struct ahash_req_ctx *state = ahash_request_ctx(areq);
+ struct ahash_req_ctx *state = ahash_request_ctx_dma(areq);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
- struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
/* Setup CMAC Key */
hw_desc_init(&desc[idx]);
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 4/10] crypto: chelsio - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (2 preceding siblings ...)
2022-12-02 9:20 ` [PATCH 3/10] crypto: ccree " Herbert Xu
@ 2022-12-02 9:20 ` Herbert Xu
2022-12-02 9:20 ` [PATCH 5/10] crypto: hisilicon/hpre " Herbert Xu
` (7 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:20 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/chelsio/chcr_algo.c | 43 ++++++++++++++++++-------------------
1 file changed, 22 insertions(+), 21 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 6933546f87b1..2158bc237dbb 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -210,7 +210,7 @@ static inline int chcr_handle_aead_resp(struct aead_request *req,
unsigned char *input,
int err)
{
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct chcr_dev *dev = a_ctx(tfm)->dev;
@@ -718,7 +718,7 @@ static inline int get_qidxs(struct crypto_async_request *req,
{
struct aead_request *aead_req =
container_of(req, struct aead_request, base);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(aead_req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(aead_req);
*txqidx = reqctx->txqidx;
*rxqidx = reqctx->rxqidx;
break;
@@ -2362,7 +2362,7 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
inline void chcr_aead_common_exit(struct aead_request *req)
{
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
@@ -2373,7 +2373,7 @@ static int chcr_aead_common_init(struct aead_request *req)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
unsigned int authsize = crypto_aead_authsize(tfm);
int error = -EINVAL;
@@ -2417,7 +2417,7 @@ static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
- struct aead_request *subreq = aead_request_ctx(req);
+ struct aead_request *subreq = aead_request_ctx_dma(req);
aead_request_set_tfm(subreq, aeadctx->sw_cipher);
aead_request_set_callback(subreq, req->base.flags,
@@ -2438,7 +2438,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
struct uld_ctx *u_ctx = ULD_CTX(ctx);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct sk_buff *skb = NULL;
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
@@ -2576,7 +2576,7 @@ int chcr_aead_dma_map(struct device *dev,
unsigned short op_type)
{
int error;
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
unsigned int authsize = crypto_aead_authsize(tfm);
int src_len, dst_len;
@@ -2637,7 +2637,7 @@ void chcr_aead_dma_unmap(struct device *dev,
struct aead_request *req,
unsigned short op_type)
{
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
unsigned int authsize = crypto_aead_authsize(tfm);
int src_len, dst_len;
@@ -2678,7 +2678,7 @@ void chcr_add_aead_src_ent(struct aead_request *req,
struct ulptx_sgl *ulptx)
{
struct ulptx_walk ulp_walk;
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
if (reqctx->imm) {
u8 *buf = (u8 *)ulptx;
@@ -2704,7 +2704,7 @@ void chcr_add_aead_dst_ent(struct aead_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
unsigned short qid)
{
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct dsgl_walk dsgl_walk;
unsigned int authsize = crypto_aead_authsize(tfm);
@@ -2894,7 +2894,7 @@ static int generate_b0(struct aead_request *req, u8 *ivptr,
unsigned int l, lp, m;
int rc;
struct crypto_aead *aead = crypto_aead_reqtfm(req);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
u8 *b0 = reqctx->scratch_pad;
m = crypto_aead_authsize(aead);
@@ -2932,7 +2932,7 @@ static int ccm_format_packet(struct aead_request *req,
unsigned short op_type,
unsigned int assoclen)
{
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
int rc = 0;
@@ -2963,7 +2963,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
struct chcr_context *ctx = a_ctx(tfm);
struct uld_ctx *u_ctx = ULD_CTX(ctx);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
@@ -3036,7 +3036,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct sk_buff *skb = NULL;
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
@@ -3135,7 +3135,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
struct chcr_context *ctx = a_ctx(tfm);
struct uld_ctx *u_ctx = ULD_CTX(ctx);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct sk_buff *skb = NULL;
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
@@ -3255,9 +3255,10 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm)
CRYPTO_ALG_ASYNC);
if (IS_ERR(aeadctx->sw_cipher))
return PTR_ERR(aeadctx->sw_cipher);
- crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
- sizeof(struct aead_request) +
- crypto_aead_reqsize(aeadctx->sw_cipher)));
+ crypto_aead_set_reqsize_dma(
+ tfm, max(sizeof(struct chcr_aead_reqctx),
+ sizeof(struct aead_request) +
+ crypto_aead_reqsize(aeadctx->sw_cipher)));
return chcr_device_init(a_ctx(tfm));
}
@@ -3735,7 +3736,7 @@ static int chcr_aead_op(struct aead_request *req,
create_wr_t create_wr_fn)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct chcr_context *ctx = a_ctx(tfm);
struct uld_ctx *u_ctx = ULD_CTX(ctx);
struct sk_buff *skb;
@@ -3785,7 +3786,7 @@ static int chcr_aead_op(struct aead_request *req,
static int chcr_aead_encrypt(struct aead_request *req)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
struct chcr_context *ctx = a_ctx(tfm);
unsigned int cpu;
@@ -3816,7 +3817,7 @@ static int chcr_aead_decrypt(struct aead_request *req)
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct chcr_context *ctx = a_ctx(tfm);
struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
int size;
unsigned int cpu;
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 5/10] crypto: hisilicon/hpre - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (3 preceding siblings ...)
2022-12-02 9:20 ` [PATCH 4/10] crypto: chelsio " Herbert Xu
@ 2022-12-02 9:20 ` Herbert Xu
2022-12-02 9:20 ` [PATCH 6/10] crypto: safexcel " Herbert Xu
` (6 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:20 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/hisilicon/hpre/hpre_crypto.c | 40 +++++++++++++++++-----------
1 file changed, 25 insertions(+), 15 deletions(-)
diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
index 5f6d363c9435..8ede77310dc5 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
@@ -147,6 +147,16 @@ struct hpre_asym_request {
struct timespec64 req_time;
};
+static inline unsigned int hpre_align_sz(void)
+{
+ return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1;
+}
+
+static inline unsigned int hpre_align_pd(void)
+{
+ return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1);
+}
+
static int hpre_alloc_req_id(struct hpre_ctx *ctx)
{
unsigned long flags;
@@ -517,7 +527,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
}
tmp = akcipher_request_ctx(akreq);
- h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ h_req = PTR_ALIGN(tmp, hpre_align_sz());
h_req->cb = hpre_rsa_cb;
h_req->areq.rsa = akreq;
msg = &h_req->req;
@@ -531,7 +541,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
}
tmp = kpp_request_ctx(kreq);
- h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ h_req = PTR_ALIGN(tmp, hpre_align_sz());
h_req->cb = hpre_dh_cb;
h_req->areq.dh = kreq;
msg = &h_req->req;
@@ -582,7 +592,7 @@ static int hpre_dh_compute_value(struct kpp_request *req)
struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
void *tmp = kpp_request_ctx(req);
- struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
struct hpre_sqe *msg = &hpre_req->req;
int ret;
@@ -740,7 +750,7 @@ static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
{
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
- kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ);
+ kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
}
@@ -785,7 +795,7 @@ static int hpre_rsa_enc(struct akcipher_request *req)
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
void *tmp = akcipher_request_ctx(req);
- struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
struct hpre_sqe *msg = &hpre_req->req;
int ret;
@@ -833,7 +843,7 @@ static int hpre_rsa_dec(struct akcipher_request *req)
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
void *tmp = akcipher_request_ctx(req);
- struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
struct hpre_sqe *msg = &hpre_req->req;
int ret;
@@ -1168,7 +1178,7 @@ static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
}
akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) +
- HPRE_ALIGN_SZ);
+ hpre_align_pd());
ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
if (ret)
@@ -1490,7 +1500,7 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
}
tmp = kpp_request_ctx(req);
- h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ h_req = PTR_ALIGN(tmp, hpre_align_sz());
h_req->cb = hpre_ecdh_cb;
h_req->areq.ecdh = req;
msg = &h_req->req;
@@ -1571,7 +1581,7 @@ static int hpre_ecdh_compute_value(struct kpp_request *req)
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
struct device *dev = ctx->dev;
void *tmp = kpp_request_ctx(req);
- struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
struct hpre_sqe *msg = &hpre_req->req;
int ret;
@@ -1622,7 +1632,7 @@ static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
ctx->curve_id = ECC_CURVE_NIST_P192;
- kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ);
+ kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
}
@@ -1633,7 +1643,7 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
ctx->curve_id = ECC_CURVE_NIST_P256;
- kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ);
+ kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
}
@@ -1644,7 +1654,7 @@ static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
ctx->curve_id = ECC_CURVE_NIST_P384;
- kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ);
+ kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
}
@@ -1802,7 +1812,7 @@ static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
}
tmp = kpp_request_ctx(req);
- h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ h_req = PTR_ALIGN(tmp, hpre_align_sz());
h_req->cb = hpre_curve25519_cb;
h_req->areq.curve25519 = req;
msg = &h_req->req;
@@ -1923,7 +1933,7 @@ static int hpre_curve25519_compute_value(struct kpp_request *req)
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
struct device *dev = ctx->dev;
void *tmp = kpp_request_ctx(req);
- struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
struct hpre_sqe *msg = &hpre_req->req;
int ret;
@@ -1972,7 +1982,7 @@ static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)
{
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
- kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ);
+ kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
}
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 6/10] crypto: safexcel - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (4 preceding siblings ...)
2022-12-02 9:20 ` [PATCH 5/10] crypto: hisilicon/hpre " Herbert Xu
@ 2022-12-02 9:20 ` Herbert Xu
2022-12-02 9:20 ` [PATCH 7/10] crypto: keembay " Herbert Xu
` (5 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:20 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/inside-secure/safexcel_hash.c | 99 +++++++++++++--------------
1 file changed, 50 insertions(+), 49 deletions(-)
diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c
index 103fc551d2af..ca46328472d4 100644
--- a/drivers/crypto/inside-secure/safexcel_hash.c
+++ b/drivers/crypto/inside-secure/safexcel_hash.c
@@ -231,7 +231,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
struct safexcel_result_desc *rdesc;
struct ahash_request *areq = ahash_request_cast(async);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
- struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *sreq = ahash_request_ctx_dma(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
u64 cache_len;
@@ -312,7 +312,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
int *commands, int *results)
{
struct ahash_request *areq = ahash_request_cast(async);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_crypto_priv *priv = ctx->base.priv;
struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
@@ -569,7 +569,7 @@ static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
bool *should_complete, int *ret)
{
struct ahash_request *areq = ahash_request_cast(async);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
int err;
BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
@@ -608,7 +608,7 @@ static int safexcel_ahash_send(struct crypto_async_request *async,
int ring, int *commands, int *results)
{
struct ahash_request *areq = ahash_request_cast(async);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
int ret;
if (req->needs_inv)
@@ -624,7 +624,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
struct safexcel_crypto_priv *priv = ctx->base.priv;
EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
- struct safexcel_ahash_req *rctx = ahash_request_ctx(req);
+ struct safexcel_ahash_req *rctx = ahash_request_ctx_dma(req);
struct safexcel_inv_result result = {};
int ring = ctx->base.ring;
@@ -663,7 +663,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
*/
static int safexcel_ahash_cache(struct ahash_request *areq)
{
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
u64 cache_len;
/* cache_len: everything accepted by the driver but not sent yet,
@@ -689,7 +689,7 @@ static int safexcel_ahash_cache(struct ahash_request *areq)
static int safexcel_ahash_enqueue(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret, ring;
@@ -741,7 +741,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)
static int safexcel_ahash_update(struct ahash_request *areq)
{
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
int ret;
/* If the request is 0 length, do nothing */
@@ -766,7 +766,7 @@ static int safexcel_ahash_update(struct ahash_request *areq)
static int safexcel_ahash_final(struct ahash_request *areq)
{
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
req->finish = true;
@@ -870,7 +870,7 @@ static int safexcel_ahash_final(struct ahash_request *areq)
static int safexcel_ahash_finup(struct ahash_request *areq)
{
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
req->finish = true;
@@ -880,7 +880,7 @@ static int safexcel_ahash_finup(struct ahash_request *areq)
static int safexcel_ahash_export(struct ahash_request *areq, void *out)
{
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
struct safexcel_ahash_export_state *export = out;
export->len = req->len;
@@ -896,7 +896,7 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out)
static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
{
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
const struct safexcel_ahash_export_state *export = in;
int ret;
@@ -927,15 +927,15 @@ static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
ctx->base.handle_result = safexcel_handle_result;
ctx->fb_do_setkey = false;
- crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
- sizeof(struct safexcel_ahash_req));
+ crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
+ sizeof(struct safexcel_ahash_req));
return 0;
}
static int safexcel_sha1_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1012,7 +1012,7 @@ struct safexcel_alg_template safexcel_alg_sha1 = {
static int safexcel_hmac_sha1_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1124,7 +1124,7 @@ static int safexcel_hmac_init_iv(struct ahash_request *areq,
if (ret)
return ret;
- req = ahash_request_ctx(areq);
+ req = ahash_request_ctx_dma(areq);
req->hmac = true;
req->last_req = true;
@@ -1264,7 +1264,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
static int safexcel_sha256_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1321,7 +1321,7 @@ struct safexcel_alg_template safexcel_alg_sha256 = {
static int safexcel_sha224_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1385,7 +1385,7 @@ static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sha224_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1457,7 +1457,7 @@ static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sha256_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1522,7 +1522,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
static int safexcel_sha512_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1579,7 +1579,7 @@ struct safexcel_alg_template safexcel_alg_sha512 = {
static int safexcel_sha384_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1643,7 +1643,7 @@ static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sha512_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1715,7 +1715,7 @@ static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sha384_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1780,7 +1780,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
static int safexcel_md5_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1837,7 +1837,7 @@ struct safexcel_alg_template safexcel_alg_md5 = {
static int safexcel_hmac_md5_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1920,7 +1920,7 @@ static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
static int safexcel_crc32_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -1992,7 +1992,7 @@ struct safexcel_alg_template safexcel_alg_crc32 = {
static int safexcel_cbcmac_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2252,7 +2252,7 @@ struct safexcel_alg_template safexcel_alg_cmac = {
static int safexcel_sm3_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2316,7 +2316,7 @@ static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sm3_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2382,7 +2382,7 @@ static int safexcel_sha3_224_init(struct ahash_request *areq)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2400,7 +2400,7 @@ static int safexcel_sha3_fbcheck(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ahash_request *subreq = ahash_request_ctx(req);
+ struct ahash_request *subreq = ahash_request_ctx_dma(req);
int ret = 0;
if (ctx->do_fallback) {
@@ -2437,7 +2437,7 @@ static int safexcel_sha3_update(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ahash_request *subreq = ahash_request_ctx(req);
+ struct ahash_request *subreq = ahash_request_ctx_dma(req);
ctx->do_fallback = true;
return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
@@ -2447,7 +2447,7 @@ static int safexcel_sha3_final(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ahash_request *subreq = ahash_request_ctx(req);
+ struct ahash_request *subreq = ahash_request_ctx_dma(req);
ctx->do_fallback = true;
return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
@@ -2457,7 +2457,7 @@ static int safexcel_sha3_finup(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ahash_request *subreq = ahash_request_ctx(req);
+ struct ahash_request *subreq = ahash_request_ctx_dma(req);
ctx->do_fallback |= !req->nbytes;
if (ctx->do_fallback)
@@ -2472,7 +2472,7 @@ static int safexcel_sha3_digest_fallback(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ahash_request *subreq = ahash_request_ctx(req);
+ struct ahash_request *subreq = ahash_request_ctx_dma(req);
ctx->do_fallback = true;
ctx->fb_init_done = false;
@@ -2492,7 +2492,7 @@ static int safexcel_sha3_export(struct ahash_request *req, void *out)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ahash_request *subreq = ahash_request_ctx(req);
+ struct ahash_request *subreq = ahash_request_ctx_dma(req);
ctx->do_fallback = true;
return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
@@ -2502,7 +2502,7 @@ static int safexcel_sha3_import(struct ahash_request *req, const void *in)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct ahash_request *subreq = ahash_request_ctx(req);
+ struct ahash_request *subreq = ahash_request_ctx_dma(req);
ctx->do_fallback = true;
return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
@@ -2526,9 +2526,10 @@ static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
/* Update statesize from fallback algorithm! */
crypto_hash_alg_common(ahash)->statesize =
crypto_ahash_statesize(ctx->fback);
- crypto_ahash_set_reqsize(ahash, max(sizeof(struct safexcel_ahash_req),
- sizeof(struct ahash_request) +
- crypto_ahash_reqsize(ctx->fback)));
+ crypto_ahash_set_reqsize_dma(
+ ahash, max(sizeof(struct safexcel_ahash_req),
+ sizeof(struct ahash_request) +
+ crypto_ahash_reqsize(ctx->fback)));
return 0;
}
@@ -2575,7 +2576,7 @@ static int safexcel_sha3_256_init(struct ahash_request *areq)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2633,7 +2634,7 @@ static int safexcel_sha3_384_init(struct ahash_request *areq)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2691,7 +2692,7 @@ static int safexcel_sha3_512_init(struct ahash_request *areq)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2841,7 +2842,7 @@ static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2912,7 +2913,7 @@ static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -2983,7 +2984,7 @@ static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
@@ -3054,7 +3055,7 @@ static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+ struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 7/10] crypto: keembay - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (5 preceding siblings ...)
2022-12-02 9:20 ` [PATCH 6/10] crypto: safexcel " Herbert Xu
@ 2022-12-02 9:20 ` Herbert Xu
2022-12-02 9:21 ` [PATCH 8/10] crypto: octeontx " Herbert Xu
` (4 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:20 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/keembay/keembay-ocs-hcu-core.c | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/drivers/crypto/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/keembay/keembay-ocs-hcu-core.c
index 0379dbf32a4c..d4bcbed1f546 100644
--- a/drivers/crypto/keembay/keembay-ocs-hcu-core.c
+++ b/drivers/crypto/keembay/keembay-ocs-hcu-core.c
@@ -226,7 +226,7 @@ static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req,
*/
static int kmb_ocs_dma_prepare(struct ahash_request *req)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
struct device *dev = rctx->hcu_dev->dev;
unsigned int remainder = 0;
unsigned int total;
@@ -356,7 +356,7 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req)
static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
/* Clear buffer of any data. */
memzero_explicit(rctx->buffer, sizeof(rctx->buffer));
@@ -374,7 +374,7 @@ static int kmb_ocs_hcu_handle_queue(struct ahash_request *req)
static int prepare_ipad(struct ahash_request *req)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
int i;
@@ -414,7 +414,7 @@ static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq)
base);
struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm);
int rc;
int i;
@@ -561,7 +561,7 @@ static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq)
static int kmb_ocs_hcu_init(struct ahash_request *req)
{
struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
@@ -614,7 +614,7 @@ static int kmb_ocs_hcu_init(struct ahash_request *req)
static int kmb_ocs_hcu_update(struct ahash_request *req)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
int rc;
if (!req->nbytes)
@@ -650,7 +650,7 @@ static int kmb_ocs_hcu_update(struct ahash_request *req)
/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */
static int kmb_ocs_hcu_fin_common(struct ahash_request *req)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
int rc;
@@ -687,7 +687,7 @@ static int kmb_ocs_hcu_fin_common(struct ahash_request *req)
static int kmb_ocs_hcu_final(struct ahash_request *req)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
rctx->sg_data_total = 0;
rctx->sg_data_offset = 0;
@@ -698,7 +698,7 @@ static int kmb_ocs_hcu_final(struct ahash_request *req)
static int kmb_ocs_hcu_finup(struct ahash_request *req)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
rctx->sg_data_total = req->nbytes;
rctx->sg_data_offset = 0;
@@ -726,7 +726,7 @@ static int kmb_ocs_hcu_digest(struct ahash_request *req)
static int kmb_ocs_hcu_export(struct ahash_request *req, void *out)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
/* Intermediate data is always stored and applied per request. */
memcpy(out, rctx, sizeof(*rctx));
@@ -736,7 +736,7 @@ static int kmb_ocs_hcu_export(struct ahash_request *req, void *out)
static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in)
{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
/* Intermediate data is always stored and applied per request. */
memcpy(rctx, in, sizeof(*rctx));
@@ -822,8 +822,8 @@ static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key,
/* Set request size and initialize tfm context. */
static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx)
{
- crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
- sizeof(struct ocs_hcu_rctx));
+ crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
+ sizeof(struct ocs_hcu_rctx));
/* Init context to 0. */
memzero_explicit(ctx, sizeof(*ctx));
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 8/10] crypto: octeontx - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (6 preceding siblings ...)
2022-12-02 9:20 ` [PATCH 7/10] crypto: keembay " Herbert Xu
@ 2022-12-02 9:21 ` Herbert Xu
2022-12-02 9:21 ` [PATCH 9/10] crypto: octeontx2 " Herbert Xu
` (3 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:21 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/marvell/octeontx/otx_cptvf_algs.c | 69 +++++++++++------------
1 file changed, 35 insertions(+), 34 deletions(-)
diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c
index 01c48ddc4eeb..80ba77c793a7 100644
--- a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c
+++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c
@@ -103,7 +103,7 @@ static inline int validate_hmac_cipher_null(struct otx_cpt_req_info *cpt_req)
req = container_of(cpt_req->areq, struct aead_request, base);
tfm = crypto_aead_reqtfm(req);
- rctx = aead_request_ctx(req);
+ rctx = aead_request_ctx_dma(req);
if (memcmp(rctx->fctx.hmac.s.hmac_calc,
rctx->fctx.hmac.s.hmac_recv,
crypto_aead_authsize(tfm)) != 0)
@@ -155,7 +155,7 @@ static void output_iv_copyback(struct crypto_async_request *areq)
ctx = crypto_skcipher_ctx(stfm);
if (ctx->cipher_type == OTX_CPT_AES_CBC ||
ctx->cipher_type == OTX_CPT_DES3_CBC) {
- rctx = skcipher_request_ctx(sreq);
+ rctx = skcipher_request_ctx_dma(sreq);
req_info = &rctx->cpt_req;
ivsize = crypto_skcipher_ivsize(stfm);
start = sreq->cryptlen - ivsize;
@@ -233,7 +233,7 @@ static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
u32 *argcnt)
{
struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
- struct otx_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
struct otx_cpt_enc_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -303,7 +303,7 @@ static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
static inline u32 create_input_list(struct skcipher_request *req, u32 enc,
u32 enc_iv_len)
{
- struct otx_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
u32 argcnt = 0;
int ret;
@@ -321,7 +321,7 @@ static inline u32 create_input_list(struct skcipher_request *req, u32 enc,
static inline void create_output_list(struct skcipher_request *req,
u32 enc_iv_len)
{
- struct otx_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
u32 argcnt = 0;
@@ -340,7 +340,7 @@ static inline void create_output_list(struct skcipher_request *req,
static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
{
struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
- struct otx_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
struct pci_dev *pdev;
@@ -501,15 +501,16 @@ static int otx_cpt_enc_dec_init(struct crypto_skcipher *tfm)
* allocated since the cryptd daemon uses
* this memory for request_ctx information
*/
- crypto_skcipher_set_reqsize(tfm, sizeof(struct otx_cpt_req_ctx) +
- sizeof(struct skcipher_request));
+ crypto_skcipher_set_reqsize_dma(
+ tfm, sizeof(struct otx_cpt_req_ctx) +
+ sizeof(struct skcipher_request));
return 0;
}
static int cpt_aead_init(struct crypto_aead *tfm, u8 cipher_type, u8 mac_type)
{
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
ctx->cipher_type = cipher_type;
ctx->mac_type = mac_type;
@@ -551,7 +552,7 @@ static int cpt_aead_init(struct crypto_aead *tfm, u8 cipher_type, u8 mac_type)
}
}
- crypto_aead_set_reqsize(tfm, sizeof(struct otx_cpt_req_ctx));
+ crypto_aead_set_reqsize_dma(tfm, sizeof(struct otx_cpt_req_ctx));
return 0;
}
@@ -603,7 +604,7 @@ static int otx_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
static void otx_cpt_aead_exit(struct crypto_aead *tfm)
{
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
kfree(ctx->ipad);
kfree(ctx->opad);
@@ -619,7 +620,7 @@ static void otx_cpt_aead_exit(struct crypto_aead *tfm)
static int otx_cpt_aead_set_authsize(struct crypto_aead *tfm,
unsigned int authsize)
{
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
switch (ctx->mac_type) {
case OTX_CPT_SHA1:
@@ -739,7 +740,7 @@ static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
static int aead_hmac_init(struct crypto_aead *cipher)
{
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
int state_size = crypto_shash_statesize(ctx->hashalg);
int ds = crypto_shash_digestsize(ctx->hashalg);
int bs = crypto_shash_blocksize(ctx->hashalg);
@@ -837,7 +838,7 @@ static int otx_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
const unsigned char *key,
unsigned int keylen)
{
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
struct crypto_authenc_key_param *param;
int enckeylen = 0, authkeylen = 0;
struct rtattr *rta = (void *)key;
@@ -896,7 +897,7 @@ static int otx_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
const unsigned char *key,
unsigned int keylen)
{
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
struct crypto_authenc_key_param *param;
struct rtattr *rta = (void *)key;
int enckeylen = 0;
@@ -932,7 +933,7 @@ static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
const unsigned char *key,
unsigned int keylen)
{
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
/*
* For aes gcm we expect to get encryption key (16, 24, 32 bytes)
@@ -965,9 +966,9 @@ static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
static inline u32 create_aead_ctx_hdr(struct aead_request *req, u32 enc,
u32 *argcnt)
{
- struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
struct otx_cpt_fc_ctx *fctx = &rctx->fctx;
int mac_len = crypto_aead_authsize(tfm);
@@ -1050,9 +1051,9 @@ static inline u32 create_aead_ctx_hdr(struct aead_request *req, u32 enc,
static inline u32 create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
u32 enc)
{
- struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;
@@ -1076,7 +1077,7 @@ static inline u32 create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
static inline u32 create_aead_input_list(struct aead_request *req, u32 enc)
{
- struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
u32 inputlen = req->cryptlen + req->assoclen;
u32 status, argcnt = 0;
@@ -1093,7 +1094,7 @@ static inline u32 create_aead_input_list(struct aead_request *req, u32 enc)
static inline u32 create_aead_output_list(struct aead_request *req, u32 enc,
u32 mac_len)
{
- struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
u32 argcnt = 0, outputlen = 0;
@@ -1111,7 +1112,7 @@ static inline u32 create_aead_output_list(struct aead_request *req, u32 enc,
static inline u32 create_aead_null_input_list(struct aead_request *req,
u32 enc, u32 mac_len)
{
- struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
u32 inputlen, argcnt = 0;
@@ -1130,7 +1131,7 @@ static inline u32 create_aead_null_input_list(struct aead_request *req,
static inline u32 create_aead_null_output_list(struct aead_request *req,
u32 enc, u32 mac_len)
{
- struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
struct scatterlist *dst;
u8 *ptr = NULL;
@@ -1217,7 +1218,7 @@ static inline u32 create_aead_null_output_list(struct aead_request *req,
static u32 cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
{
- struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx_cpt_req_info *req_info = &rctx->cpt_req;
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct pci_dev *pdev;
@@ -1409,7 +1410,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha1_cbc_aes",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1428,7 +1429,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha256_cbc_aes",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1447,7 +1448,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha384_cbc_aes",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1466,7 +1467,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha512_cbc_aes",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1485,7 +1486,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha1_ecb_null",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1504,7 +1505,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha256_ecb_null",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1523,7 +1524,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha384_ecb_null",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1542,7 +1543,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha512_ecb_null",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1561,7 +1562,7 @@ static struct aead_alg otx_cpt_aeads[] = { {
.cra_driver_name = "cpt_rfc4106_gcm_aes",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
- .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 9/10] crypto: octeontx2 - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (7 preceding siblings ...)
2022-12-02 9:21 ` [PATCH 8/10] crypto: octeontx " Herbert Xu
@ 2022-12-02 9:21 ` Herbert Xu
2022-12-02 9:21 ` [PATCH 10/10] crypto: qce " Herbert Xu
` (2 subsequent siblings)
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:21 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c | 79 ++++++++++-----------
1 file changed, 40 insertions(+), 39 deletions(-)
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
index 67530e90bbfe..30b423605c9c 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
@@ -87,7 +87,7 @@ static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
req = container_of(cpt_req->areq, struct aead_request, base);
tfm = crypto_aead_reqtfm(req);
- rctx = aead_request_ctx(req);
+ rctx = aead_request_ctx_dma(req);
if (memcmp(rctx->fctx.hmac.s.hmac_calc,
rctx->fctx.hmac.s.hmac_recv,
crypto_aead_authsize(tfm)) != 0)
@@ -137,7 +137,7 @@ static void output_iv_copyback(struct crypto_async_request *areq)
ctx = crypto_skcipher_ctx(stfm);
if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
ctx->cipher_type == OTX2_CPT_DES3_CBC) {
- rctx = skcipher_request_ctx(sreq);
+ rctx = skcipher_request_ctx_dma(sreq);
req_info = &rctx->cpt_req;
ivsize = crypto_skcipher_ivsize(stfm);
start = sreq->cryptlen - ivsize;
@@ -219,7 +219,7 @@ static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
u32 *argcnt)
{
struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
- struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
@@ -288,7 +288,7 @@ static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
static inline int create_input_list(struct skcipher_request *req, u32 enc,
u32 enc_iv_len)
{
- struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
u32 argcnt = 0;
int ret;
@@ -306,7 +306,7 @@ static inline int create_input_list(struct skcipher_request *req, u32 enc,
static inline void create_output_list(struct skcipher_request *req,
u32 enc_iv_len)
{
- struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
u32 argcnt = 0;
@@ -325,7 +325,7 @@ static inline void create_output_list(struct skcipher_request *req,
static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
{
struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
- struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
int ret;
@@ -348,7 +348,7 @@ static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
{
struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
- struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
@@ -537,8 +537,9 @@ static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
* allocated since the cryptd daemon uses
* this memory for request_ctx information
*/
- crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) +
- sizeof(struct skcipher_request));
+ crypto_skcipher_set_reqsize_dma(
+ stfm, sizeof(struct otx2_cpt_req_ctx) +
+ sizeof(struct skcipher_request));
return cpt_skcipher_fallback_init(ctx, alg);
}
@@ -572,7 +573,7 @@ static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
{
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm);
struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
struct crypto_alg *alg = tfm->__crt_alg;
@@ -629,7 +630,7 @@ static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
ctx->enc_align_len = 1;
break;
}
- crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx));
+ crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx));
return cpt_aead_fallback_init(ctx, alg);
}
@@ -681,7 +682,7 @@ static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
{
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
kfree(ctx->ipad);
kfree(ctx->opad);
@@ -698,7 +699,7 @@ static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
unsigned int authsize)
{
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
if (crypto_rfc4106_check_authsize(authsize))
return -EINVAL;
@@ -722,7 +723,7 @@ static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
unsigned int authsize)
{
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
ctx->is_trunc_hmac = true;
tfm->authsize = authsize;
@@ -794,7 +795,7 @@ static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
static int aead_hmac_init(struct crypto_aead *cipher)
{
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
int state_size = crypto_shash_statesize(ctx->hashalg);
int ds = crypto_shash_digestsize(ctx->hashalg);
int bs = crypto_shash_blocksize(ctx->hashalg);
@@ -892,7 +893,7 @@ static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
const unsigned char *key,
unsigned int keylen)
{
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
struct crypto_authenc_key_param *param;
int enckeylen = 0, authkeylen = 0;
struct rtattr *rta = (void *)key;
@@ -944,7 +945,7 @@ static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
const unsigned char *key,
unsigned int keylen)
{
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
struct crypto_authenc_key_param *param;
struct rtattr *rta = (void *)key;
int enckeylen = 0;
@@ -979,7 +980,7 @@ static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
const unsigned char *key,
unsigned int keylen)
{
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
/*
* For aes gcm we expect to get encryption key (16, 24, 32 bytes)
@@ -1012,9 +1013,9 @@ static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
u32 *argcnt)
{
- struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
int mac_len = crypto_aead_authsize(tfm);
@@ -1103,9 +1104,9 @@ static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
u32 enc)
{
- struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
@@ -1127,7 +1128,7 @@ static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
static inline int create_aead_input_list(struct aead_request *req, u32 enc)
{
- struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
u32 inputlen = req->cryptlen + req->assoclen;
u32 status, argcnt = 0;
@@ -1144,7 +1145,7 @@ static inline int create_aead_input_list(struct aead_request *req, u32 enc)
static inline void create_aead_output_list(struct aead_request *req, u32 enc,
u32 mac_len)
{
- struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
u32 argcnt = 0, outputlen = 0;
@@ -1160,7 +1161,7 @@ static inline void create_aead_output_list(struct aead_request *req, u32 enc,
static inline void create_aead_null_input_list(struct aead_request *req,
u32 enc, u32 mac_len)
{
- struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
u32 inputlen, argcnt = 0;
@@ -1177,7 +1178,7 @@ static inline void create_aead_null_input_list(struct aead_request *req,
static inline int create_aead_null_output_list(struct aead_request *req,
u32 enc, u32 mac_len)
{
- struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
struct scatterlist *dst;
u8 *ptr = NULL;
@@ -1257,9 +1258,9 @@ static inline int create_aead_null_output_list(struct aead_request *req,
static int aead_do_fallback(struct aead_request *req, bool is_enc)
{
- struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct crypto_aead *aead = crypto_aead_reqtfm(req);
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(aead);
int ret;
if (ctx->fbk_cipher) {
@@ -1281,10 +1282,10 @@ static int aead_do_fallback(struct aead_request *req, bool is_enc)
static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
{
- struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
+ struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
struct pci_dev *pdev;
int status, cpu_num;
@@ -1458,7 +1459,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha1_cbc_aes",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1477,7 +1478,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha256_cbc_aes",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1496,7 +1497,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha384_cbc_aes",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1515,7 +1516,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha512_cbc_aes",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1534,7 +1535,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha1_ecb_null",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1553,7 +1554,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha256_ecb_null",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1572,7 +1573,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha384_ecb_null",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1591,7 +1592,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_hmac_sha512_ecb_null",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
@@ -1610,7 +1611,7 @@ static struct aead_alg otx2_cpt_aeads[] = { {
.cra_driver_name = "cpt_rfc4106_gcm_aes",
.cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
- .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
+ .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
.cra_priority = 4001,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 10/10] crypto: qce - Set DMA alignment explicitly
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (8 preceding siblings ...)
2022-12-02 9:21 ` [PATCH 9/10] crypto: octeontx2 " Herbert Xu
@ 2022-12-02 9:21 ` Herbert Xu
2022-12-02 9:49 ` [PATCH 0/10] crypto: Driver conversions for DMA alignment liulongfang
2022-12-04 9:32 ` Christoph Hellwig
11 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:21 UTC (permalink / raw)
To: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
This driver has been implicitly relying on kmalloc alignment
to be sufficient for DMA. This may no longer be the case with
upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from
the Crypto API.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
drivers/crypto/qce/aead.c | 22 +++++++++++-----------
drivers/crypto/qce/common.c | 5 +++--
drivers/crypto/qce/sha.c | 18 +++++++++---------
3 files changed, 23 insertions(+), 22 deletions(-)
diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c
index 6eb4d2e35629..7d811728f047 100644
--- a/drivers/crypto/qce/aead.c
+++ b/drivers/crypto/qce/aead.c
@@ -24,7 +24,7 @@ static void qce_aead_done(void *data)
{
struct crypto_async_request *async_req = data;
struct aead_request *req = aead_request_cast(async_req);
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
@@ -92,7 +92,7 @@ static void qce_aead_done(void *data)
static struct scatterlist *
qce_aead_prepare_result_buf(struct sg_table *tbl, struct aead_request *req)
{
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
@@ -103,7 +103,7 @@ qce_aead_prepare_result_buf(struct sg_table *tbl, struct aead_request *req)
static struct scatterlist *
qce_aead_prepare_ccm_result_buf(struct sg_table *tbl, struct aead_request *req)
{
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
sg_init_one(&rctx->result_sg, rctx->ccmresult_buf, QCE_BAM_BURST_SIZE);
return qce_sgtable_add(tbl, &rctx->result_sg, QCE_BAM_BURST_SIZE);
@@ -112,7 +112,7 @@ qce_aead_prepare_ccm_result_buf(struct sg_table *tbl, struct aead_request *req)
static struct scatterlist *
qce_aead_prepare_dst_buf(struct aead_request *req)
{
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
struct scatterlist *sg, *msg_sg, __sg[2];
@@ -186,7 +186,7 @@ qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req)
{
struct scatterlist *sg, *msg_sg, __sg[2];
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
unsigned int assoclen = rctx->assoclen;
unsigned int adata_header_len, cryptlen, totallen;
@@ -300,7 +300,7 @@ qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req)
static int qce_aead_prepare_buf(struct aead_request *req)
{
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
struct scatterlist *sg;
@@ -328,7 +328,7 @@ static int qce_aead_prepare_buf(struct aead_request *req)
static int qce_aead_ccm_prepare_buf(struct aead_request *req)
{
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct scatterlist *sg;
@@ -408,7 +408,7 @@ static int
qce_aead_async_req_handle(struct crypto_async_request *async_req)
{
struct aead_request *req = aead_request_cast(async_req);
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
@@ -502,7 +502,7 @@ qce_aead_async_req_handle(struct crypto_async_request *async_req)
static int qce_aead_crypt(struct aead_request *req, int encrypt)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct qce_alg_template *tmpl = to_aead_tmpl(tfm);
unsigned int blocksize = crypto_aead_blocksize(tfm);
@@ -675,8 +675,8 @@ static int qce_aead_init(struct crypto_aead *tfm)
if (IS_ERR(ctx->fallback))
return PTR_ERR(ctx->fallback);
- crypto_aead_set_reqsize(tfm, sizeof(struct qce_aead_reqctx) +
- crypto_aead_reqsize(ctx->fallback));
+ crypto_aead_set_reqsize_dma(tfm, sizeof(struct qce_aead_reqctx) +
+ crypto_aead_reqsize(ctx->fallback));
return 0;
}
diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c
index 7c612ba5068f..04253a8d3340 100644
--- a/drivers/crypto/qce/common.c
+++ b/drivers/crypto/qce/common.c
@@ -3,6 +3,7 @@
* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved.
*/
+#include <crypto/internal/hash.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/types.h>
@@ -147,7 +148,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req)
{
struct ahash_request *req = ahash_request_cast(async_req);
struct crypto_ahash *ahash = __crypto_ahash_cast(async_req->tfm);
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm);
struct qce_device *qce = tmpl->qce;
unsigned int digestsize = crypto_ahash_digestsize(ahash);
@@ -419,7 +420,7 @@ static unsigned int qce_be32_to_cpu_array(u32 *dst, const u8 *src, unsigned int
static int qce_setup_regs_aead(struct crypto_async_request *async_req)
{
struct aead_request *req = aead_request_cast(async_req);
- struct qce_aead_reqctx *rctx = aead_request_ctx(req);
+ struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req);
struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c
index 37bafd7aeb79..fc72af8aa9a7 100644
--- a/drivers/crypto/qce/sha.c
+++ b/drivers/crypto/qce/sha.c
@@ -38,7 +38,7 @@ static void qce_ahash_done(void *data)
struct crypto_async_request *async_req = data;
struct ahash_request *req = ahash_request_cast(async_req);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm);
struct qce_device *qce = tmpl->qce;
struct qce_result_dump *result = qce->dma.result_buf;
@@ -75,7 +75,7 @@ static void qce_ahash_done(void *data)
static int qce_ahash_async_req_handle(struct crypto_async_request *async_req)
{
struct ahash_request *req = ahash_request_cast(async_req);
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
struct qce_sha_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm);
struct qce_device *qce = tmpl->qce;
@@ -132,7 +132,7 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req)
static int qce_ahash_init(struct ahash_request *req)
{
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
const u32 *std_iv = tmpl->std_iv;
@@ -147,7 +147,7 @@ static int qce_ahash_init(struct ahash_request *req)
static int qce_ahash_export(struct ahash_request *req, void *out)
{
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
struct qce_sha_saved_state *export_state = out;
memcpy(export_state->pending_buf, rctx->buf, rctx->buflen);
@@ -164,7 +164,7 @@ static int qce_ahash_export(struct ahash_request *req, void *out)
static int qce_ahash_import(struct ahash_request *req, const void *in)
{
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
const struct qce_sha_saved_state *import_state = in;
memset(rctx, 0, sizeof(*rctx));
@@ -183,7 +183,7 @@ static int qce_ahash_import(struct ahash_request *req, const void *in)
static int qce_ahash_update(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
struct qce_device *qce = tmpl->qce;
struct scatterlist *sg_last, *sg;
@@ -275,7 +275,7 @@ static int qce_ahash_update(struct ahash_request *req)
static int qce_ahash_final(struct ahash_request *req)
{
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
struct qce_device *qce = tmpl->qce;
@@ -302,7 +302,7 @@ static int qce_ahash_final(struct ahash_request *req)
static int qce_ahash_digest(struct ahash_request *req)
{
- struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
+ struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
struct qce_device *qce = tmpl->qce;
int ret;
@@ -395,7 +395,7 @@ static int qce_ahash_cra_init(struct crypto_tfm *tfm)
struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
struct qce_sha_ctx *ctx = crypto_tfm_ctx(tfm);
- crypto_ahash_set_reqsize(ahash, sizeof(struct qce_sha_reqctx));
+ crypto_ahash_set_reqsize_dma(ahash, sizeof(struct qce_sha_reqctx));
memset(ctx, 0, sizeof(*ctx));
return 0;
}
^ permalink raw reply related [flat|nested] 31+ messages in thread
* Re: [PATCH 0/10] crypto: Driver conversions for DMA alignment
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (9 preceding siblings ...)
2022-12-02 9:21 ` [PATCH 10/10] crypto: qce " Herbert Xu
@ 2022-12-02 9:49 ` liulongfang
2022-12-02 9:53 ` Herbert Xu
2022-12-04 9:32 ` Christoph Hellwig
11 siblings, 1 reply; 31+ messages in thread
From: liulongfang @ 2022-12-02 9:49 UTC (permalink / raw)
To: Herbert Xu, Catalin Marinas
Cc: Ard Biesheuvel, Will Deacon, Marc Zyngier, Arnd Bergmann,
Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM, LinuxKernelMailingList,
linux-kernel, David S. Miller, Linux Crypto Mailing List
On 2022/12/2 17:19, Herbert Xu wrote:
> These are the rest of the driver conversions in order for arm64
> to safely lower the kmalloc alignment below that required for DMA.
>
> My criteria for inclusion are:
>
> 1) the driver can be built on arm64.
> 2) the driver may perform DMA on the context structure.
>
> I have worked through all the drivers in crypto but if you think
> I've missed something please let me know.
>
Hi, Herbert:
There are also skcipher_request_ctx and aead_request_ctx in the hisilicon/sec2 driver
that need to be updated.
Thanks,
Longfang.
> Thanks,
>
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 0/10] crypto: Driver conversions for DMA alignment
2022-12-02 9:49 ` [PATCH 0/10] crypto: Driver conversions for DMA alignment liulongfang
@ 2022-12-02 9:53 ` Herbert Xu
0 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-02 9:53 UTC (permalink / raw)
To: liulongfang
Cc: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM, LinuxKernelMailingList,
linux-kernel, David S. Miller, Linux Crypto Mailing List
On Fri, Dec 02, 2022 at 05:49:21PM +0800, liulongfang wrote:
.
> Hi, Herbert:
> There are also skcipher_request_ctx and aead_request_ctx in the hisilicon/sec2 driver
> that need to be updated.
Please show me where they do DMA directly on the context structure.
I wasn't able to find such a place.
Thanks,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 0/10] crypto: Driver conversions for DMA alignment
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
` (10 preceding siblings ...)
2022-12-02 9:49 ` [PATCH 0/10] crypto: Driver conversions for DMA alignment liulongfang
@ 2022-12-04 9:32 ` Christoph Hellwig
2022-12-06 4:13 ` Herbert Xu
11 siblings, 1 reply; 31+ messages in thread
From: Christoph Hellwig @ 2022-12-04 9:32 UTC (permalink / raw)
To: Herbert Xu
Cc: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM, LinuxKernelMailingList,
linux-kernel, David S. Miller, Linux Crypto Mailing List
On Fri, Dec 02, 2022 at 05:19:43PM +0800, Herbert Xu wrote:
> These are the rest of the driver conversions in order for arm64
> to safely lower the kmalloc alignment below that required for DMA.
Btw, drivers/crypto/ has a lot of weird and most likely uses of
GFP_DMA. Can someone look into those while we're doing DMA audits
of the cryto drivers?
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 0/10] crypto: Driver conversions for DMA alignment
2022-12-04 9:32 ` Christoph Hellwig
@ 2022-12-06 4:13 ` Herbert Xu
2022-12-06 6:27 ` Christoph Hellwig
0 siblings, 1 reply; 31+ messages in thread
From: Herbert Xu @ 2022-12-06 4:13 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM, LinuxKernelMailingList,
linux-kernel, David S. Miller, Linux Crypto Mailing List
On Sun, Dec 04, 2022 at 01:32:08AM -0800, Christoph Hellwig wrote:
> On Fri, Dec 02, 2022 at 05:19:43PM +0800, Herbert Xu wrote:
> > These are the rest of the driver conversions in order for arm64
> > to safely lower the kmalloc alignment below that required for DMA.
>
> Btw, drivers/crypto/ has a lot of weird and most likely uses of
> GFP_DMA. Can someone look into those while we're doing DMA audits
> of the cryto drivers?
Yes they're clearly bogus. Basically they are saying they want
memory that is aligned sufficiently for DMA. So if Catalin's
patch-set will break this assumption, then all the GFP_DMA allocations
in drivers/crypto will need to be enlarged to take this into
account.
Cheers,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 0/10] crypto: Driver conversions for DMA alignment
2022-12-06 4:13 ` Herbert Xu
@ 2022-12-06 6:27 ` Christoph Hellwig
2022-12-06 8:34 ` Herbert Xu
0 siblings, 1 reply; 31+ messages in thread
From: Christoph Hellwig @ 2022-12-06 6:27 UTC (permalink / raw)
To: Herbert Xu
Cc: Christoph Hellwig, Catalin Marinas, Ard Biesheuvel, Will Deacon,
Marc Zyngier, Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton,
Linus Torvalds, Linux Memory Management List, Linux ARM,
LinuxKernelMailingList, linux-kernel, David S. Miller,
Linux Crypto Mailing List
On Tue, Dec 06, 2022 at 12:13:52PM +0800, Herbert Xu wrote:
> Yes they're clearly bogus. Basically they are saying they want
> memory that is aligned sufficiently for DMA. So if Catalin's
> patch-set will break this assumption, then all the GFP_DMA allocations
> in drivers/crypto will need to be enlarged to take this into
> account.
But GFP_DMA never did do anything at all about alignment. It picks
allocations from ZONE_DMA (which on x86 is the first 16MB only).
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 0/10] crypto: Driver conversions for DMA alignment
2022-12-06 6:27 ` Christoph Hellwig
@ 2022-12-06 8:34 ` Herbert Xu
2022-12-29 8:58 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Herbert Xu
0 siblings, 1 reply; 31+ messages in thread
From: Herbert Xu @ 2022-12-06 8:34 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM, LinuxKernelMailingList,
linux-kernel, David S. Miller, Linux Crypto Mailing List
On Mon, Dec 05, 2022 at 10:27:58PM -0800, Christoph Hellwig wrote:
.
> But GFP_DMA never did do anything at all about alignment. It picks
> allocations from ZONE_DMA (which on x86 is the first 16MB only).
Right. I'm not arguing that they are correct or anything. I'm
just saying that they are currently working on arm64 because of
the large minimum kmalloc alignment, and they will all be broken
afterwards.
Cheers,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding
2022-12-06 8:34 ` Herbert Xu
@ 2022-12-29 8:58 ` Herbert Xu
2022-12-30 5:21 ` [PATCH] crypto: caam " Herbert Xu
` (2 more replies)
0 siblings, 3 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-29 8:58 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM, LinuxKernelMailingList,
linux-kernel, David S. Miller, Linux Crypto Mailing List,
Corentin Labbe
GFP_DMA does not guarantee that the returned memory is aligned
for DMA. In fact for sun8i-ss it is superfluous and can be removed.
However, kmalloc may start returning DMA-unaligned memory in future
so fix this by adding the alignment by hand.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c
index 902f6be057ec..83c6dfad77e1 100644
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c
+++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c
@@ -452,7 +452,7 @@ int sun8i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
}
kfree_sensitive(op->key);
op->keylen = keylen;
- op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
+ op->key = kmemdup(key, keylen, GFP_KERNEL);
if (!op->key)
return -ENOMEM;
@@ -475,7 +475,7 @@ int sun8i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
kfree_sensitive(op->key);
op->keylen = keylen;
- op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
+ op->key = kmemdup(key, keylen, GFP_KERNEL);
if (!op->key)
return -ENOMEM;
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
index ac2329e2b0e5..c9dc06f97857 100644
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
+++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
@@ -16,6 +16,7 @@
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irq.h>
+#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_device.h>
@@ -527,7 +528,7 @@ static int allocate_flows(struct sun8i_ss_dev *ss)
init_completion(&ss->flows[i].complete);
ss->flows[i].biv = devm_kmalloc(ss->dev, AES_BLOCK_SIZE,
- GFP_KERNEL | GFP_DMA);
+ GFP_KERNEL);
if (!ss->flows[i].biv) {
err = -ENOMEM;
goto error_engine;
@@ -535,7 +536,7 @@ static int allocate_flows(struct sun8i_ss_dev *ss)
for (j = 0; j < MAX_SG; j++) {
ss->flows[i].iv[j] = devm_kmalloc(ss->dev, AES_BLOCK_SIZE,
- GFP_KERNEL | GFP_DMA);
+ GFP_KERNEL);
if (!ss->flows[i].iv[j]) {
err = -ENOMEM;
goto error_engine;
@@ -544,13 +545,15 @@ static int allocate_flows(struct sun8i_ss_dev *ss)
/* the padding could be up to two block. */
ss->flows[i].pad = devm_kmalloc(ss->dev, MAX_PAD_SIZE,
- GFP_KERNEL | GFP_DMA);
+ GFP_KERNEL);
if (!ss->flows[i].pad) {
err = -ENOMEM;
goto error_engine;
}
- ss->flows[i].result = devm_kmalloc(ss->dev, SHA256_DIGEST_SIZE,
- GFP_KERNEL | GFP_DMA);
+ ss->flows[i].result =
+ devm_kmalloc(ss->dev, max(SHA256_DIGEST_SIZE,
+ dma_get_cache_alignment()),
+ GFP_KERNEL);
if (!ss->flows[i].result) {
err = -ENOMEM;
goto error_engine;
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
index 36a82b22953c..577bf636f7fb 100644
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
+++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
@@ -79,10 +79,10 @@ int sun8i_ss_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
memcpy(tfmctx->key, key, keylen);
}
- tfmctx->ipad = kzalloc(bs, GFP_KERNEL | GFP_DMA);
+ tfmctx->ipad = kzalloc(bs, GFP_KERNEL);
if (!tfmctx->ipad)
return -ENOMEM;
- tfmctx->opad = kzalloc(bs, GFP_KERNEL | GFP_DMA);
+ tfmctx->opad = kzalloc(bs, GFP_KERNEL);
if (!tfmctx->opad) {
ret = -ENOMEM;
goto err_opad;
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c
index dd677e9ed06f..70c7b5d571b8 100644
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c
+++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c
@@ -11,6 +11,8 @@
*/
#include "sun8i-ss.h"
#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
#include <linux/pm_runtime.h>
#include <crypto/internal/rng.h>
@@ -25,7 +27,7 @@ int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed,
ctx->seed = NULL;
}
if (!ctx->seed)
- ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA);
+ ctx->seed = kmalloc(slen, GFP_KERNEL);
if (!ctx->seed)
return -ENOMEM;
@@ -58,6 +60,7 @@ int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
struct rng_alg *alg = crypto_rng_alg(tfm);
struct sun8i_ss_alg_template *algt;
+ unsigned int todo_with_padding;
struct sun8i_ss_dev *ss;
dma_addr_t dma_iv, dma_dst;
unsigned int todo;
@@ -81,7 +84,11 @@ int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
todo = dlen + PRNG_SEED_SIZE + PRNG_DATA_SIZE;
todo -= todo % PRNG_DATA_SIZE;
- d = kzalloc(todo, GFP_KERNEL | GFP_DMA);
+ todo_with_padding = ALIGN(todo, dma_get_cache_alignment());
+ if (todo_with_padding < todo || todo < dlen)
+ return -EOVERFLOW;
+
+ d = kzalloc(todo_with_padding, GFP_KERNEL);
if (!d)
return -ENOMEM;
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH] crypto: caam - Remove GFP_DMA and add DMA alignment padding
2022-12-29 8:58 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Herbert Xu
@ 2022-12-30 5:21 ` Herbert Xu
2022-12-30 7:31 ` [PATCH] crypto: talitos " Herbert Xu
` (2 more replies)
2023-01-03 6:51 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Corentin Labbe
2023-01-08 17:36 ` Christoph Hellwig
2 siblings, 3 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-30 5:21 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List, Horia Geantă,
Pankaj Gupta, Gaurav Jain
GFP_DMA does not guarantee that the returned memory is aligned
for DMA. It should be removed where it is superfluous.
However, kmalloc may start returning DMA-unaligned memory in future
so fix this by adding the alignment by hand.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
diff --git a/drivers/crypto/caam/blob_gen.c b/drivers/crypto/caam/blob_gen.c
index 1f65df489847..0c347ba6e692 100644
--- a/drivers/crypto/caam/blob_gen.c
+++ b/drivers/crypto/caam/blob_gen.c
@@ -83,7 +83,7 @@ int caam_process_blob(struct caam_blob_priv *priv,
output_len = info->input_len - CAAM_BLOB_OVERHEAD;
}
- desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL | GFP_DMA);
+ desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL);
if (!desc)
return -ENOMEM;
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index ecc15bc521db..4a9b998a8d26 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -59,6 +59,8 @@
#include <crypto/engine.h>
#include <crypto/xts.h>
#include <asm/unaligned.h>
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
/*
* crypto alg
@@ -1379,8 +1381,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
/* allocate space for base edesc and hw desc commands, link tables */
- edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
- GFP_DMA | flags);
+ edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, flags);
if (!edesc) {
caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
@@ -1608,6 +1609,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
u8 *iv;
int ivsize = crypto_skcipher_ivsize(skcipher);
int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
+ unsigned int aligned_size;
src_nents = sg_nents_for_len(req->src, req->cryptlen);
if (unlikely(src_nents < 0)) {
@@ -1681,15 +1683,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
/*
* allocate space for base edesc and hw desc commands, link tables, IV
*/
- edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,
- GFP_DMA | flags);
- if (!edesc) {
+ aligned_size = ALIGN(ivsize, __alignof__(*edesc));
+ aligned_size += sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
+ aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());
+ iv = kzalloc(aligned_size, flags);
+ if (!iv) {
dev_err(jrdev, "could not allocate extended descriptor\n");
caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
return ERR_PTR(-ENOMEM);
}
+ edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
edesc->src_nents = src_nents;
edesc->dst_nents = dst_nents;
edesc->mapped_src_nents = mapped_src_nents;
@@ -1701,7 +1706,6 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
/* Make sure IV is located in a DMAable area */
if (ivsize) {
- iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index c37b67be0492..5e218bf20d5b 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -20,6 +20,8 @@
#include "caamalg_desc.h"
#include <crypto/xts.h>
#include <asm/unaligned.h>
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
/*
* crypto alg
@@ -959,7 +961,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
return (struct aead_edesc *)drv_ctx;
/* allocate space for base edesc and hw desc commands, link tables */
- edesc = qi_cache_alloc(GFP_DMA | flags);
+ edesc = qi_cache_alloc(flags);
if (unlikely(!edesc)) {
dev_err(qidev, "could not allocate extended descriptor\n");
return ERR_PTR(-ENOMEM);
@@ -1317,8 +1319,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
- if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
- ivsize > CAAM_QI_MEMCACHE_SIZE)) {
+ if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) +
+ offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes >
+ CAAM_QI_MEMCACHE_SIZE)) {
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -1327,17 +1330,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
}
/* allocate space for base edesc, link tables and IV */
- edesc = qi_cache_alloc(GFP_DMA | flags);
- if (unlikely(!edesc)) {
+ iv = qi_cache_alloc(flags);
+ if (unlikely(!iv)) {
dev_err(qidev, "could not allocate extended descriptor\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
+ edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
+
/* Make sure IV is located in a DMAable area */
sg_table = &edesc->sgt[0];
- iv = (u8 *)(sg_table + qm_sg_ents);
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
index 1b0dd742c53f..0ddef9a033a1 100644
--- a/drivers/crypto/caam/caamalg_qi2.c
+++ b/drivers/crypto/caam/caamalg_qi2.c
@@ -16,7 +16,9 @@
#include "caamalg_desc.h"
#include "caamhash_desc.h"
#include "dpseci-debugfs.h"
+#include <linux/dma-mapping.h>
#include <linux/fsl/mc.h>
+#include <linux/kernel.h>
#include <soc/fsl/dpaa2-io.h>
#include <soc/fsl/dpaa2-fd.h>
#include <crypto/xts.h>
@@ -370,7 +372,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
struct dpaa2_sg_entry *sg_table;
/* allocate space for base edesc, link tables and IV */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (unlikely(!edesc)) {
dev_err(dev, "could not allocate extended descriptor\n");
return ERR_PTR(-ENOMEM);
@@ -1189,7 +1191,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
}
/* allocate space for base edesc, link tables and IV */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (unlikely(!edesc)) {
dev_err(dev, "could not allocate extended descriptor\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -3220,14 +3222,14 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
int ret = -ENOMEM;
struct dpaa2_fl_entry *in_fle, *out_fle;
- req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
+ req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL);
if (!req_ctx)
return -ENOMEM;
in_fle = &req_ctx->fd_flt[1];
out_fle = &req_ctx->fd_flt[0];
- flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
+ flc = kzalloc(sizeof(*flc), GFP_KERNEL);
if (!flc)
goto err_flc;
@@ -3316,7 +3318,13 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
if (keylen > blocksize) {
- hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
+ unsigned int aligned_len =
+ ALIGN(keylen, dma_get_cache_alignment());
+
+ if (aligned_len < keylen)
+ return -EOVERFLOW;
+
+ hashed_key = kmemdup(key, aligned_len, GFP_KERNEL);
if (!hashed_key)
return -ENOMEM;
ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
@@ -3560,7 +3568,7 @@ static int ahash_update_ctx(struct ahash_request *req)
}
/* allocate space for base edesc and link tables */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (!edesc) {
dma_unmap_sg(ctx->dev, req->src, src_nents,
DMA_TO_DEVICE);
@@ -3654,7 +3662,7 @@ static int ahash_final_ctx(struct ahash_request *req)
int ret;
/* allocate space for base edesc and link tables */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (!edesc)
return -ENOMEM;
@@ -3743,7 +3751,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
}
/* allocate space for base edesc and link tables */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (!edesc) {
dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
return -ENOMEM;
@@ -3836,7 +3844,7 @@ static int ahash_digest(struct ahash_request *req)
}
/* allocate space for base edesc and link tables */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (!edesc) {
dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
return ret;
@@ -3913,7 +3921,7 @@ static int ahash_final_no_ctx(struct ahash_request *req)
int ret = -ENOMEM;
/* allocate space for base edesc and link tables */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (!edesc)
return ret;
@@ -4012,7 +4020,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
}
/* allocate space for base edesc and link tables */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (!edesc) {
dma_unmap_sg(ctx->dev, req->src, src_nents,
DMA_TO_DEVICE);
@@ -4125,7 +4133,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
}
/* allocate space for base edesc and link tables */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (!edesc) {
dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
return ret;
@@ -4230,7 +4238,7 @@ static int ahash_update_first(struct ahash_request *req)
}
/* allocate space for base edesc and link tables */
- edesc = qi_cache_zalloc(GFP_DMA | flags);
+ edesc = qi_cache_zalloc(flags);
if (!edesc) {
dma_unmap_sg(ctx->dev, req->src, src_nents,
DMA_TO_DEVICE);
@@ -4926,6 +4934,7 @@ static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
{
struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
struct device *dev = priv->dev;
+ unsigned int alignmask;
int err;
/*
@@ -4936,13 +4945,14 @@ static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
!(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
return 0;
- priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
- GFP_KERNEL | GFP_DMA);
+ alignmask = DPAA2_CSCN_ALIGN - 1;
+ alignmask |= dma_get_cache_alignment() - 1;
+ priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1),
+ GFP_KERNEL);
if (!priv->cscn_mem)
return -ENOMEM;
- priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
- priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
+ priv->cscn_dma = dma_map_single(dev, priv->cscn_mem,
DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
if (dma_mapping_error(dev, priv->cscn_dma)) {
dev_err(dev, "Error mapping CSCN memory area\n");
@@ -5174,7 +5184,7 @@ static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
priv->domain = iommu_get_domain_for_dev(dev);
qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
- 0, SLAB_CACHE_DMA, NULL);
+ 0, 0, NULL);
if (!qi_cache) {
dev_err(dev, "Can't allocate SEC cache\n");
return -ENOMEM;
@@ -5451,7 +5461,7 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
DPAA2_CSCN_SIZE,
DMA_FROM_DEVICE);
- if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
+ if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) {
dev_dbg_ratelimited(dev, "Dropping request\n");
return -EBUSY;
}
diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h
index d35253407ade..abb502bb675c 100644
--- a/drivers/crypto/caam/caamalg_qi2.h
+++ b/drivers/crypto/caam/caamalg_qi2.h
@@ -7,13 +7,14 @@
#ifndef _CAAMALG_QI2_H_
#define _CAAMALG_QI2_H_
+#include <crypto/internal/skcipher.h>
+#include <linux/compiler_attributes.h>
#include <soc/fsl/dpaa2-io.h>
#include <soc/fsl/dpaa2-fd.h>
#include <linux/threads.h>
#include <linux/netdevice.h>
#include "dpseci.h"
#include "desc_constr.h"
-#include <crypto/skcipher.h>
#define DPAA2_CAAM_STORE_SIZE 16
/* NAPI weight *must* be a multiple of the store size. */
@@ -36,8 +37,6 @@
* @tx_queue_attr: array of Tx queue attributes
* @cscn_mem: pointer to memory region containing the congestion SCN
* it's size is larger than to accommodate alignment
- * @cscn_mem_aligned: pointer to congestion SCN; it is computed as
- * PTR_ALIGN(cscn_mem, DPAA2_CSCN_ALIGN)
* @cscn_dma: dma address used by the QMAN to write CSCN messages
* @dev: device associated with the DPSECI object
* @mc_io: pointer to MC portal's I/O object
@@ -58,7 +57,6 @@ struct dpaa2_caam_priv {
/* congestion */
void *cscn_mem;
- void *cscn_mem_aligned;
dma_addr_t cscn_dma;
struct device *dev;
@@ -158,7 +156,7 @@ struct ahash_edesc {
struct caam_flc {
u32 flc[16];
u32 sh_desc[MAX_SDLEN];
-} ____cacheline_aligned;
+} __aligned(CRYPTO_DMA_ALIGN);
enum optype {
ENCRYPT = 0,
@@ -180,7 +178,7 @@ enum optype {
* @edesc: extended descriptor; points to one of {skcipher,aead}_edesc
*/
struct caam_request {
- struct dpaa2_fl_entry fd_flt[2];
+ struct dpaa2_fl_entry fd_flt[2] __aligned(CRYPTO_DMA_ALIGN);
dma_addr_t fd_flt_dma;
struct caam_flc *flc;
dma_addr_t flc_dma;
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 1050e965a438..1f357f48c473 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -66,6 +66,8 @@
#include "key_gen.h"
#include "caamhash_desc.h"
#include <crypto/engine.h>
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
#define CAAM_CRA_PRIORITY 3000
@@ -365,7 +367,7 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
dma_addr_t key_dma;
int ret;
- desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
+ desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL);
if (!desc) {
dev_err(jrdev, "unable to allocate key input memory\n");
return -ENOMEM;
@@ -432,7 +434,13 @@ static int ahash_setkey(struct crypto_ahash *ahash,
dev_dbg(jrdev, "keylen %d\n", keylen);
if (keylen > blocksize) {
- hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
+ unsigned int aligned_len =
+ ALIGN(keylen, dma_get_cache_alignment());
+
+ if (aligned_len < keylen)
+ return -EOVERFLOW;
+
+ hashed_key = kmemdup(key, keylen, GFP_KERNEL);
if (!hashed_key)
return -ENOMEM;
ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
@@ -702,7 +710,7 @@ static struct ahash_edesc *ahash_edesc_alloc(struct ahash_request *req,
struct ahash_edesc *edesc;
unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
- edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
+ edesc = kzalloc(sizeof(*edesc) + sg_size, flags);
if (!edesc) {
dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
return NULL;
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index aef031946f33..e40614fef39d 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -16,6 +16,8 @@
#include "desc_constr.h"
#include "sg_sw_sec4.h"
#include "caampkc.h"
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + SIZEOF_RSA_PUB_PDB)
#define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \
@@ -310,8 +312,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
/* allocate space for base edesc, hw desc commands and link tables */
- edesc = kzalloc(sizeof(*edesc) + desclen + sec4_sg_bytes,
- GFP_DMA | flags);
+ edesc = kzalloc(sizeof(*edesc) + desclen + sec4_sg_bytes, flags);
if (!edesc)
goto dst_fail;
@@ -898,7 +899,7 @@ static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen)
if (!nbytes)
return NULL;
- dst = kzalloc(dstlen, GFP_DMA | GFP_KERNEL);
+ dst = kzalloc(dstlen, GFP_KERNEL);
if (!dst)
return NULL;
@@ -910,7 +911,7 @@ static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen)
/**
* caam_read_raw_data - Read a raw byte stream as a positive integer.
* The function skips buffer's leading zeros, copies the remained data
- * to a buffer allocated in the GFP_DMA | GFP_KERNEL zone and returns
+ * to a buffer allocated in the GFP_KERNEL zone and returns
* the address of the new buffer.
*
* @buf : The data to read
@@ -923,7 +924,7 @@ static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes)
if (!*nbytes)
return NULL;
- return kmemdup(buf, *nbytes, GFP_DMA | GFP_KERNEL);
+ return kmemdup(buf, *nbytes, GFP_KERNEL);
}
static int caam_rsa_check_key_length(unsigned int len)
@@ -949,13 +950,13 @@ static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
return ret;
/* Copy key in DMA zone */
- rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_DMA | GFP_KERNEL);
+ rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_KERNEL);
if (!rsa_key->e)
goto err;
/*
* Skip leading zeros and copy the positive integer to a buffer
- * allocated in the GFP_DMA | GFP_KERNEL zone. The decryption descriptor
+ * allocated in the GFP_KERNEL zone. The decryption descriptor
* expects a positive integer for the RSA modulus and uses its length as
* decryption output length.
*/
@@ -983,6 +984,7 @@ static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx,
struct caam_rsa_key *rsa_key = &ctx->key;
size_t p_sz = raw_key->p_sz;
size_t q_sz = raw_key->q_sz;
+ unsigned aligned_size;
rsa_key->p = caam_read_raw_data(raw_key->p, &p_sz);
if (!rsa_key->p)
@@ -994,11 +996,13 @@ static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx,
goto free_p;
rsa_key->q_sz = q_sz;
- rsa_key->tmp1 = kzalloc(raw_key->p_sz, GFP_DMA | GFP_KERNEL);
+ aligned_size = ALIGN(raw_key->p_sz, dma_get_cache_alignment());
+ rsa_key->tmp1 = kzalloc(aligned_size, GFP_KERNEL);
if (!rsa_key->tmp1)
goto free_q;
- rsa_key->tmp2 = kzalloc(raw_key->q_sz, GFP_DMA | GFP_KERNEL);
+ aligned_size = ALIGN(raw_key->q_sz, dma_get_cache_alignment());
+ rsa_key->tmp2 = kzalloc(aligned_size, GFP_KERNEL);
if (!rsa_key->tmp2)
goto free_tmp1;
@@ -1051,17 +1055,17 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
return ret;
/* Copy key in DMA zone */
- rsa_key->d = kmemdup(raw_key.d, raw_key.d_sz, GFP_DMA | GFP_KERNEL);
+ rsa_key->d = kmemdup(raw_key.d, raw_key.d_sz, GFP_KERNEL);
if (!rsa_key->d)
goto err;
- rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_DMA | GFP_KERNEL);
+ rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_KERNEL);
if (!rsa_key->e)
goto err;
/*
* Skip leading zeros and copy the positive integer to a buffer
- * allocated in the GFP_DMA | GFP_KERNEL zone. The decryption descriptor
+ * allocated in the GFP_KERNEL zone. The decryption descriptor
* expects a positive integer for the RSA modulus and uses its length as
* decryption output length.
*/
@@ -1185,8 +1189,7 @@ int caam_pkc_init(struct device *ctrldev)
return 0;
/* allocate zero buffer, used for padding input */
- zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_DMA |
- GFP_KERNEL);
+ zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_KERNEL);
if (!zero_buffer)
return -ENOMEM;
diff --git a/drivers/crypto/caam/caamprng.c b/drivers/crypto/caam/caamprng.c
index 4839e66300a2..6e4c1191cb28 100644
--- a/drivers/crypto/caam/caamprng.c
+++ b/drivers/crypto/caam/caamprng.c
@@ -8,6 +8,8 @@
#include <linux/completion.h>
#include <crypto/internal/rng.h>
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
#include "compat.h"
#include "regs.h"
#include "intern.h"
@@ -75,6 +77,7 @@ static int caam_prng_generate(struct crypto_rng *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int dlen)
{
+ unsigned int aligned_dlen = ALIGN(dlen, dma_get_cache_alignment());
struct caam_prng_ctx ctx;
struct device *jrdev;
dma_addr_t dst_dma;
@@ -82,7 +85,10 @@ static int caam_prng_generate(struct crypto_rng *tfm,
u8 *buf;
int ret;
- buf = kzalloc(dlen, GFP_KERNEL);
+ if (aligned_dlen < dlen)
+ return -EOVERFLOW;
+
+ buf = kzalloc(aligned_dlen, GFP_KERNEL);
if (!buf)
return -ENOMEM;
@@ -94,7 +100,7 @@ static int caam_prng_generate(struct crypto_rng *tfm,
return ret;
}
- desc = kzalloc(CAAM_PRNG_MAX_DESC_LEN, GFP_KERNEL | GFP_DMA);
+ desc = kzalloc(CAAM_PRNG_MAX_DESC_LEN, GFP_KERNEL);
if (!desc) {
ret = -ENOMEM;
goto out1;
@@ -156,7 +162,7 @@ static int caam_prng_seed(struct crypto_rng *tfm,
return ret;
}
- desc = kzalloc(CAAM_PRNG_MAX_DESC_LEN, GFP_KERNEL | GFP_DMA);
+ desc = kzalloc(CAAM_PRNG_MAX_DESC_LEN, GFP_KERNEL);
if (!desc) {
caam_jr_free(jrdev);
return -ENOMEM;
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index 1f0e82050976..1fd8ff965006 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -12,6 +12,8 @@
#include <linux/hw_random.h>
#include <linux/completion.h>
#include <linux/atomic.h>
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
#include <linux/kfifo.h>
#include "compat.h"
@@ -176,17 +178,18 @@ static int caam_init(struct hwrng *rng)
int err;
ctx->desc_sync = devm_kzalloc(ctx->ctrldev, CAAM_RNG_DESC_LEN,
- GFP_DMA | GFP_KERNEL);
+ GFP_KERNEL);
if (!ctx->desc_sync)
return -ENOMEM;
ctx->desc_async = devm_kzalloc(ctx->ctrldev, CAAM_RNG_DESC_LEN,
- GFP_DMA | GFP_KERNEL);
+ GFP_KERNEL);
if (!ctx->desc_async)
return -ENOMEM;
- if (kfifo_alloc(&ctx->fifo, CAAM_RNG_MAX_FIFO_STORE_SIZE,
- GFP_DMA | GFP_KERNEL))
+ if (kfifo_alloc(&ctx->fifo, ALIGN(CAAM_RNG_MAX_FIFO_STORE_SIZE,
+ dma_get_cache_alignment()),
+ GFP_KERNEL))
return -ENOMEM;
INIT_WORK(&ctx->worker, caam_rng_worker);
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 32253a064d0f..6278afb951c3 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -199,7 +199,7 @@ static int deinstantiate_rng(struct device *ctrldev, int state_handle_mask)
u32 *desc, status;
int sh_idx, ret = 0;
- desc = kmalloc(CAAM_CMD_SZ * 3, GFP_KERNEL | GFP_DMA);
+ desc = kmalloc(CAAM_CMD_SZ * 3, GFP_KERNEL);
if (!desc)
return -ENOMEM;
@@ -276,7 +276,7 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
int ret = 0, sh_idx;
ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
- desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL | GFP_DMA);
+ desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL);
if (!desc)
return -ENOMEM;
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index b0e8a4939b4f..88cc4fe2a585 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -64,7 +64,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
if (local_max > max_keylen)
return -EINVAL;
- desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
+ desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL);
if (!desc) {
dev_err(jrdev, "unable to allocate key input memory\n");
return ret;
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index c36f27376d7e..4c52c9365558 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -614,7 +614,7 @@ static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
struct qman_fq *fq;
int ret;
- fq = kzalloc(sizeof(*fq), GFP_KERNEL | GFP_DMA);
+ fq = kzalloc(sizeof(*fq), GFP_KERNEL);
if (!fq)
return -ENOMEM;
@@ -756,7 +756,7 @@ int caam_qi_init(struct platform_device *caam_pdev)
}
qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
- SLAB_CACHE_DMA, NULL);
+ 0, NULL);
if (!qi_cache) {
dev_err(qidev, "Can't allocate CAAM cache\n");
free_rsp_fqs();
diff --git a/drivers/crypto/caam/qi.h b/drivers/crypto/caam/qi.h
index 5894f16f8fe3..a96e3d213c06 100644
--- a/drivers/crypto/caam/qi.h
+++ b/drivers/crypto/caam/qi.h
@@ -9,6 +9,8 @@
#ifndef __QI_H__
#define __QI_H__
+#include <crypto/algapi.h>
+#include <linux/compiler_attributes.h>
#include <soc/fsl/qman.h>
#include "compat.h"
#include "desc.h"
@@ -58,8 +60,10 @@ enum optype {
* @qidev: device pointer for CAAM/QI backend
*/
struct caam_drv_ctx {
- u32 prehdr[2];
- u32 sh_desc[MAX_SDLEN];
+ struct {
+ u32 prehdr[2];
+ u32 sh_desc[MAX_SDLEN];
+ } __aligned(CRYPTO_DMA_ALIGN);
dma_addr_t context_a;
struct qman_fq *req_fq;
struct qman_fq *rsp_fq;
@@ -67,7 +71,7 @@ struct caam_drv_ctx {
int cpu;
enum optype op_type;
struct device *qidev;
-} ____cacheline_aligned;
+};
/**
* caam_drv_req - The request structure the driver application should fill while
@@ -88,7 +92,7 @@ struct caam_drv_req {
struct caam_drv_ctx *drv_ctx;
caam_qi_cbk cbk;
void *app_ctx;
-} ____cacheline_aligned;
+} __aligned(CRYPTO_DMA_ALIGN);
/**
* caam_drv_ctx_init - Initialise a CAAM/QI driver context
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH] crypto: talitos - Remove GFP_DMA and add DMA alignment padding
2022-12-30 5:21 ` [PATCH] crypto: caam " Herbert Xu
@ 2022-12-30 7:31 ` Herbert Xu
2023-01-08 17:37 ` Christoph Hellwig
2023-01-09 7:18 ` Christophe Leroy
2023-01-08 17:36 ` [PATCH] crypto: caam " Christoph Hellwig
[not found] ` <DU0PR04MB95635D72885111458C50DB6F8EA89@DU0PR04MB9563.eurprd04.prod.outlook.com>
2 siblings, 2 replies; 31+ messages in thread
From: Herbert Xu @ 2022-12-30 7:31 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List, Christophe Leroy
GFP_DMA does not guarantee that the returned memory is aligned
for DMA. It should be removed where it is superfluous.
However, kmalloc may start returning DMA-unaligned memory in future
so fix this by adding the alignment by hand.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index 71db6450b6aa..d62ec68e3183 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -1393,7 +1393,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
alloc_len += sizeof(struct talitos_desc);
alloc_len += ivsize;
- edesc = kmalloc(alloc_len, GFP_DMA | flags);
+ edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags);
if (!edesc)
return ERR_PTR(-ENOMEM);
if (ivsize) {
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply related [flat|nested] 31+ messages in thread
* Re: [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding
2022-12-29 8:58 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Herbert Xu
2022-12-30 5:21 ` [PATCH] crypto: caam " Herbert Xu
@ 2023-01-03 6:51 ` Corentin Labbe
2023-01-03 7:26 ` Herbert Xu
2023-01-08 17:36 ` Christoph Hellwig
2 siblings, 1 reply; 31+ messages in thread
From: Corentin Labbe @ 2023-01-03 6:51 UTC (permalink / raw)
To: Herbert Xu
Cc: Christoph Hellwig, Catalin Marinas, Ard Biesheuvel, Will Deacon,
Marc Zyngier, Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton,
Linus Torvalds, Linux Memory Management List, Linux ARM,
LinuxKernelMailingList, linux-kernel, David S. Miller,
Linux Crypto Mailing List
Le Thu, Dec 29, 2022 at 04:58:21PM +0800, Herbert Xu a écrit :
> GFP_DMA does not guarantee that the returned memory is aligned
> for DMA. In fact for sun8i-ss it is superfluous and can be removed.
>
> However, kmalloc may start returning DMA-unaligned memory in future
> so fix this by adding the alignment by hand.
>
> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Hello
Tested-by: Corentin Labbe <clabbe.montjoie@gmail.com>
Tested-on: sun8i-a83t-bananapi-m3
Acked-by: Corentin Labbe <clabbe.montjoie@gmail.com>
This means that lot of other crypto driver (sun8i-ce, amlogic, etc...) I maintain need the same fix, right ?
Regards
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding
2023-01-03 6:51 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Corentin Labbe
@ 2023-01-03 7:26 ` Herbert Xu
0 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2023-01-03 7:26 UTC (permalink / raw)
To: Corentin Labbe
Cc: Christoph Hellwig, Catalin Marinas, Ard Biesheuvel, Will Deacon,
Marc Zyngier, Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton,
Linus Torvalds, Linux Memory Management List, Linux ARM,
LinuxKernelMailingList, linux-kernel, David S. Miller,
Linux Crypto Mailing List
On Tue, Jan 03, 2023 at 07:51:24AM +0100, Corentin Labbe wrote:
.
> Tested-by: Corentin Labbe <clabbe.montjoie@gmail.com>
> Tested-on: sun8i-a83t-bananapi-m3
> Acked-by: Corentin Labbe <clabbe.montjoie@gmail.com>
Thanks,
> This means that lot of other crypto driver (sun8i-ce, amlogic, etc...) I maintain need the same fix, right ?
I checked the other drivers and none of them seem to be broken.
Only DMA from the device (DMA_FROM_DEVICE or DMA_BIDIRECTIONAL)
need to be aligned appropriately.
However, please do remove any unnecessary uses of GFP_DMA in those
drivers.
Cheers,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding
2022-12-29 8:58 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Herbert Xu
2022-12-30 5:21 ` [PATCH] crypto: caam " Herbert Xu
2023-01-03 6:51 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Corentin Labbe
@ 2023-01-08 17:36 ` Christoph Hellwig
2 siblings, 0 replies; 31+ messages in thread
From: Christoph Hellwig @ 2023-01-08 17:36 UTC (permalink / raw)
To: Herbert Xu
Cc: Christoph Hellwig, Catalin Marinas, Ard Biesheuvel, Will Deacon,
Marc Zyngier, Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton,
Linus Torvalds, Linux Memory Management List, Linux ARM,
LinuxKernelMailingList, linux-kernel, David S. Miller,
Linux Crypto Mailing List, Corentin Labbe
Looks good:
Reviewed-by: Christoph Hellwig <hch@lst.de>
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH] crypto: caam - Remove GFP_DMA and add DMA alignment padding
2022-12-30 5:21 ` [PATCH] crypto: caam " Herbert Xu
2022-12-30 7:31 ` [PATCH] crypto: talitos " Herbert Xu
@ 2023-01-08 17:36 ` Christoph Hellwig
[not found] ` <DU0PR04MB95635D72885111458C50DB6F8EA89@DU0PR04MB9563.eurprd04.prod.outlook.com>
2 siblings, 0 replies; 31+ messages in thread
From: Christoph Hellwig @ 2023-01-08 17:36 UTC (permalink / raw)
To: Herbert Xu
Cc: Christoph Hellwig, Catalin Marinas, Ard Biesheuvel, Will Deacon,
Marc Zyngier, Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton,
Linus Torvalds, Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List, Horia Geantă,
Pankaj Gupta, Gaurav Jain
Looks good:
Reviewed-by: Christoph Hellwig <hch@lst.de>
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH] crypto: talitos - Remove GFP_DMA and add DMA alignment padding
2022-12-30 7:31 ` [PATCH] crypto: talitos " Herbert Xu
@ 2023-01-08 17:37 ` Christoph Hellwig
2023-01-09 7:18 ` Christophe Leroy
1 sibling, 0 replies; 31+ messages in thread
From: Christoph Hellwig @ 2023-01-08 17:37 UTC (permalink / raw)
To: Herbert Xu
Cc: Christoph Hellwig, Catalin Marinas, Ard Biesheuvel, Will Deacon,
Marc Zyngier, Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton,
Linus Torvalds, Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List, Christophe Leroy
Looks good:
Reviewed-by: Christoph Hellwig <hch@lst.de>
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH] crypto: talitos - Remove GFP_DMA and add DMA alignment padding
2022-12-30 7:31 ` [PATCH] crypto: talitos " Herbert Xu
2023-01-08 17:37 ` Christoph Hellwig
@ 2023-01-09 7:18 ` Christophe Leroy
2023-01-09 7:37 ` Herbert Xu
1 sibling, 1 reply; 31+ messages in thread
From: Christophe Leroy @ 2023-01-09 7:18 UTC (permalink / raw)
To: Herbert Xu, Christoph Hellwig
Cc: Catalin Marinas, Ard Biesheuvel, Will Deacon, Marc Zyngier,
Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton, Linus Torvalds,
Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
Le 30/12/2022 à 08:31, Herbert Xu a écrit :
> GFP_DMA does not guarantee that the returned memory is aligned
> for DMA. It should be removed where it is superfluous.
Doesn't GFP_DMA guarantees that the provided memory is addressable for
DMA ? Or do we assume that all memory returned by kmalloc can be used
for DMA ?
>
> However, kmalloc may start returning DMA-unaligned memory in future
> so fix this by adding the alignment by hand.
kmalloc() already returns not DMA aligned memory, why does it becomes a
problem now ?
Ok, that may be suboptimal, but is that a problem at all ?
By the way, I'm not sure I understand the solution, what's the added
value of aligning allocation length to the cache alignment ?
>
> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
>
> diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
> index 71db6450b6aa..d62ec68e3183 100644
> --- a/drivers/crypto/talitos.c
> +++ b/drivers/crypto/talitos.c
> @@ -1393,7 +1393,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
> alloc_len += sizeof(struct talitos_desc);
> alloc_len += ivsize;
>
> - edesc = kmalloc(alloc_len, GFP_DMA | flags);
> + edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags);
> if (!edesc)
> return ERR_PTR(-ENOMEM);
> if (ivsize) {
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH] crypto: talitos - Remove GFP_DMA and add DMA alignment padding
2023-01-09 7:18 ` Christophe Leroy
@ 2023-01-09 7:37 ` Herbert Xu
0 siblings, 0 replies; 31+ messages in thread
From: Herbert Xu @ 2023-01-09 7:37 UTC (permalink / raw)
To: Christophe Leroy
Cc: Christoph Hellwig, Catalin Marinas, Ard Biesheuvel, Will Deacon,
Marc Zyngier, Arnd Bergmann, Greg Kroah-Hartman, Andrew Morton,
Linus Torvalds, Linux Memory Management List, Linux ARM,
Linux Kernel Mailing List, David S. Miller,
Linux Crypto Mailing List
On Mon, Jan 09, 2023 at 07:18:27AM +0000, Christophe Leroy wrote:
>
> Doesn't GFP_DMA guarantees that the provided memory is addressable for
> DMA ? Or do we assume that all memory returned by kmalloc can be used
> for DMA ?
No it does not. Please refer to the DMI API documentation.
> kmalloc() already returns not DMA aligned memory, why does it becomes a
> problem now ?
As it stands kmalloc on arm64 returns DMA-aligned memory. This
will soon no longer be the case.
Cheers,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH] crypto: caam - Fix edesc/iv ordering mixup
[not found] ` <DU0PR04MB95635D72885111458C50DB6F8EA89@DU0PR04MB9563.eurprd04.prod.outlook.com>
@ 2023-02-24 10:18 ` Herbert Xu
2023-02-27 5:20 ` Meenakshi Aggarwal
0 siblings, 1 reply; 31+ messages in thread
From: Herbert Xu @ 2023-02-24 10:18 UTC (permalink / raw)
To: Meenakshi Aggarwal
Cc: Linux Crypto Mailing List, Christoph Hellwig, Horia Geantă,
Pankaj Gupta, Gaurav Jain
Hi Meenakshi:
On Fri, Feb 24, 2023 at 06:23:15AM +0000, Meenakshi Aggarwal wrote:
>
> with this change, edesc is following IV but we need IV to follow edesc.
> Also, we are freeing edesc pointer in function, returning edesc pointer and calling function also free edesc pointer but you have allocated
> IV . So we are facing kernel crash.
>
> We need to fix this, please share why are you allocating IV in place of edesc ?
Sorry, my patch was completely broken. I was trying to place the
IV at the front in a vain effort to reduce the total allocation size.
Anyhow, please let me know if this patch fixes the problem, and I
will push it to Linus.
BTW, should we add you to the list of maintainers for caam? Perhaps
next time we can spot the problem earlier. Thanks!
---8<---
The attempt to add DMA alignment padding by moving IV to the front
of edesc was completely broken as it didn't change the places where
edesc was freed.
It's also wrong as the IV may still share a cache-line with the
edesc.
Fix this by restoring the original layout and simply reserving
enough memmory so that the IV is on a DMA cache-line by itself.
Reported-by: Meenakshi Aggarwal <meenakshi.aggarwal@nxp.com>
Fixes: 199354d7fb6e ("crypto: caam - Remove GFP_DMA and add DMA alignment padding")
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 4a9b998a8d26..c71955ac2252 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -60,7 +60,11 @@
#include <crypto/xts.h>
#include <asm/unaligned.h>
#include <linux/dma-mapping.h>
+#include <linux/device.h>
+#include <linux/err.h>
#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/string.h>
/*
* crypto alg
@@ -1683,18 +1687,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
/*
* allocate space for base edesc and hw desc commands, link tables, IV
*/
- aligned_size = ALIGN(ivsize, __alignof__(*edesc));
- aligned_size += sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
+ aligned_size = sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());
- iv = kzalloc(aligned_size, flags);
- if (!iv) {
+ aligned_size += ~(ARCH_KMALLOC_MINALIGN - 1) &
+ (dma_get_cache_alignment() - 1);
+ aligned_size += ALIGN(ivsize, dma_get_cache_alignment());
+ edesc = kzalloc(aligned_size, flags);
+ if (!edesc) {
dev_err(jrdev, "could not allocate extended descriptor\n");
caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
return ERR_PTR(-ENOMEM);
}
- edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
edesc->src_nents = src_nents;
edesc->dst_nents = dst_nents;
edesc->mapped_src_nents = mapped_src_nents;
@@ -1706,6 +1711,8 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
/* Make sure IV is located in a DMAable area */
if (ivsize) {
+ iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
+ iv = PTR_ALIGN(iv, dma_get_cache_alignment());
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index 5e218bf20d5b..5d17f5862b93 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -20,8 +20,11 @@
#include "caamalg_desc.h"
#include <crypto/xts.h>
#include <asm/unaligned.h>
+#include <linux/device.h>
+#include <linux/err.h>
#include <linux/dma-mapping.h>
#include <linux/kernel.h>
+#include <linux/string.h>
/*
* crypto alg
@@ -1259,6 +1262,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
struct qm_sg_entry *sg_table, *fd_sgt;
struct caam_drv_ctx *drv_ctx;
+ unsigned int len;
drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
if (IS_ERR(drv_ctx))
@@ -1319,9 +1323,12 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
- if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) +
- offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes >
- CAAM_QI_MEMCACHE_SIZE)) {
+
+ len = offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes;
+ len = ALIGN(len, dma_get_cache_alignment());
+ len += ivsize;
+
+ if (unlikely(len > CAAM_QI_MEMCACHE_SIZE)) {
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -1330,18 +1337,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
}
/* allocate space for base edesc, link tables and IV */
- iv = qi_cache_alloc(flags);
- if (unlikely(!iv)) {
+ edesc = qi_cache_alloc(flags);
+ if (unlikely(!edesc)) {
dev_err(qidev, "could not allocate extended descriptor\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
- edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
-
/* Make sure IV is located in a DMAable area */
sg_table = &edesc->sgt[0];
+ iv = (u8 *)(sg_table + qm_sg_ents);
+ iv = PTR_ALIGN(iv, dma_get_cache_alignment());
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index 4c52c9365558..2ad2c1035856 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -8,7 +8,13 @@
*/
#include <linux/cpumask.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
#include <linux/kthread.h>
+#include <linux/netdevice.h>
+#include <linux/slab.h>
+#include <linux/string.h>
#include <soc/fsl/qman.h>
#include "debugfs.h"
@@ -755,8 +761,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
napi_enable(irqtask);
}
- qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
- 0, NULL);
+ qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE,
+ dma_get_cache_alignment(), 0, NULL);
if (!qi_cache) {
dev_err(qidev, "Can't allocate CAAM cache\n");
free_rsp_fqs();
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply related [flat|nested] 31+ messages in thread
* RE: [PATCH] crypto: caam - Fix edesc/iv ordering mixup
2023-02-24 10:18 ` [PATCH] crypto: caam - Fix edesc/iv ordering mixup Herbert Xu
@ 2023-02-27 5:20 ` Meenakshi Aggarwal
2023-02-27 8:25 ` [v2 PATCH] " Herbert Xu
0 siblings, 1 reply; 31+ messages in thread
From: Meenakshi Aggarwal @ 2023-02-27 5:20 UTC (permalink / raw)
To: Herbert Xu
Cc: Linux Crypto Mailing List, Christoph Hellwig, Horia Geanta,
Pankaj Gupta, Gaurav Jain
Hi Herbert,
I have tested your changes, not facing a kernel crash now but still kernel warning messages are coming:
[ 6.359772] alg: skcipher: ctr-aes-caam encryption test failed (wrong output IV) on test vector 0, cfg="in-place (one sglist)"
[ 6.371179] 00000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
[ 6.377645] alg: self-tests for ctr(aes) using ctr-aes-caam failed (rc=-22)
[ 6.377649] ------------[ cut here ]------------
[ 6.389248] alg: self-tests for ctr(aes) using ctr-aes-caam failed (rc=-22)
[ 6.389269] WARNING: CPU: 0 PID: 246 at crypto/testmgr.c:6101 alg_test.part.0+0x3c8/0x3d0
[ 6.404400] Modules linked in:
[ 6.407446] CPU: 0 PID: 246 Comm: cryptomgr_test Not tainted 6.2.0-rc4-next-20230119-04770-gb8b0d08d8447 #3
[ 6.417181] Hardware name: LS1046A RDB Board (DT)
[ 6.421876] pstate: 60000005 (nZCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 6.428833] pc : alg_test.part.0+0x3c8/0x3d0
[ 6.433095] lr : alg_test.part.0+0x3c8/0x3d0
[ 6.437358] sp : ffff80000a51bd40
[ 6.440662] x29: ffff80000a51bd40 x28: 0000000000000000 x27: 0000000000000000
[ 6.447795] x26: 00000000ffffffff x25: 0000000000000400 x24: ffff125a1d7ad280
[ 6.454926] x23: ffff125a1d7ad200 x22: ffff125a1e24d100 x21: 0000000000011085
[ 6.462057] x20: 00000000ffffffea x19: ffffb57461018490 x18: 0000000000000001
[ 6.469188] x17: 74757074756f2067 x16: 6e6f727728206465 x15: ffff125a1e24d568
[ 6.476319] x14: 0000000000000000 x13: ffffb57462e8e300 x12: 00000000ffffefff
[ 6.483450] x11: 0000000000000003 x10: ffffb5746298ecd8 x9 : ffffb5745f902b34
[ 6.490581] x8 : 0000000000017fe8 x7 : c0000000ffffefff x6 : 000000000000bff4
[ 6.497712] x5 : 0000000000057fa8 x4 : 0000000000000000 x3 : ffff80000a51bb08
[ 6.504842] x2 : ffffb57462936a60 x1 : 78bd64a75e317900 x0 : 0000000000000000
[ 6.511973] Call trace:
[ 6.514409] alg_test.part.0+0x3c8/0x3d0
[ 6.518325] alg_test+0x24/0x68
[ 6.521458] cryptomgr_test+0x28/0x48
[ 6.525112] kthread+0x114/0x120
[ 6.528334] ret_from_fork+0x10/0x20
[ 6.531903] ---[ end trace 0000000000000000 ]---
[ 6.536608] alg: skcipher: cbc-des-caam encryption test failed (wrong output IV) on test vector 0, cfg="in-place (one sglist)"
[ 6.548010] 00000000: 00 00 00 00 00 00 00 00
[ 6.552370] alg: self-tests for cbc(des) using cbc-des-caam failed (rc=-22)
Thanks,
Meenakshi
> -----Original Message-----
> From: Herbert Xu <herbert@gondor.apana.org.au>
> Sent: Friday, February 24, 2023 3:48 PM
> To: Meenakshi Aggarwal <meenakshi.aggarwal@nxp.com>
> Cc: Linux Crypto Mailing List <linux-crypto@vger.kernel.org>; Christoph Hellwig
> <hch@infradead.org>; Horia Geanta <horia.geanta@nxp.com>; Pankaj Gupta
> <pankaj.gupta@nxp.com>; Gaurav Jain <gaurav.jain@nxp.com>
> Subject: [PATCH] crypto: caam - Fix edesc/iv ordering mixup
>
> Hi Meenakshi:
>
> On Fri, Feb 24, 2023 at 06:23:15AM +0000, Meenakshi Aggarwal wrote:
> >
> > with this change, edesc is following IV but we need IV to follow edesc.
> > Also, we are freeing edesc pointer in function, returning edesc
> > pointer and calling function also free edesc pointer but you have allocated IV .
> So we are facing kernel crash.
> >
> > We need to fix this, please share why are you allocating IV in place of edesc ?
>
> Sorry, my patch was completely broken. I was trying to place the IV at the front
> in a vain effort to reduce the total allocation size.
>
> Anyhow, please let me know if this patch fixes the problem, and I will push it to
> Linus.
>
> BTW, should we add you to the list of maintainers for caam? Perhaps next time
> we can spot the problem earlier. Thanks!
>
> ---8<---
> The attempt to add DMA alignment padding by moving IV to the front of edesc
> was completely broken as it didn't change the places where edesc was freed.
>
> It's also wrong as the IV may still share a cache-line with the edesc.
>
> Fix this by restoring the original layout and simply reserving enough memmory
> so that the IV is on a DMA cache-line by itself.
>
> Reported-by: Meenakshi Aggarwal <meenakshi.aggarwal@nxp.com>
> Fixes: 199354d7fb6e ("crypto: caam - Remove GFP_DMA and add DMA
> alignment padding")
> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
>
> diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
> index 4a9b998a8d26..c71955ac2252 100644
> --- a/drivers/crypto/caam/caamalg.c
> +++ b/drivers/crypto/caam/caamalg.c
> @@ -60,7 +60,11 @@
> #include <crypto/xts.h>
> #include <asm/unaligned.h>
> #include <linux/dma-mapping.h>
> +#include <linux/device.h>
> +#include <linux/err.h>
> #include <linux/kernel.h>
> +#include <linux/slab.h>
> +#include <linux/string.h>
>
> /*
> * crypto alg
> @@ -1683,18 +1687,19 @@ static struct skcipher_edesc
> *skcipher_edesc_alloc(struct skcipher_request *req,
> /*
> * allocate space for base edesc and hw desc commands, link tables, IV
> */
> - aligned_size = ALIGN(ivsize, __alignof__(*edesc));
> - aligned_size += sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
> + aligned_size = sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
> aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());
> - iv = kzalloc(aligned_size, flags);
> - if (!iv) {
> + aligned_size += ~(ARCH_KMALLOC_MINALIGN - 1) &
> + (dma_get_cache_alignment() - 1);
> + aligned_size += ALIGN(ivsize, dma_get_cache_alignment());
> + edesc = kzalloc(aligned_size, flags);
> + if (!edesc) {
> dev_err(jrdev, "could not allocate extended descriptor\n");
> caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
> 0, 0, 0);
> return ERR_PTR(-ENOMEM);
> }
>
> - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
> edesc->src_nents = src_nents;
> edesc->dst_nents = dst_nents;
> edesc->mapped_src_nents = mapped_src_nents; @@ -1706,6 +1711,8
> @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request
> *req,
>
> /* Make sure IV is located in a DMAable area */
> if (ivsize) {
> + iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
> + iv = PTR_ALIGN(iv, dma_get_cache_alignment());
> memcpy(iv, req->iv, ivsize);
>
> iv_dma = dma_map_single(jrdev, iv, ivsize,
> DMA_BIDIRECTIONAL); diff --git a/drivers/crypto/caam/caamalg_qi.c
> b/drivers/crypto/caam/caamalg_qi.c
> index 5e218bf20d5b..5d17f5862b93 100644
> --- a/drivers/crypto/caam/caamalg_qi.c
> +++ b/drivers/crypto/caam/caamalg_qi.c
> @@ -20,8 +20,11 @@
> #include "caamalg_desc.h"
> #include <crypto/xts.h>
> #include <asm/unaligned.h>
> +#include <linux/device.h>
> +#include <linux/err.h>
> #include <linux/dma-mapping.h>
> #include <linux/kernel.h>
> +#include <linux/string.h>
>
> /*
> * crypto alg
> @@ -1259,6 +1262,7 @@ static struct skcipher_edesc
> *skcipher_edesc_alloc(struct skcipher_request *req,
> int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
> struct qm_sg_entry *sg_table, *fd_sgt;
> struct caam_drv_ctx *drv_ctx;
> + unsigned int len;
>
> drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
> if (IS_ERR(drv_ctx))
> @@ -1319,9 +1323,12 @@ static struct skcipher_edesc
> *skcipher_edesc_alloc(struct skcipher_request *req,
> qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
>
> qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
> - if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) +
> - offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes >
> - CAAM_QI_MEMCACHE_SIZE)) {
> +
> + len = offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes;
> + len = ALIGN(len, dma_get_cache_alignment());
> + len += ivsize;
> +
> + if (unlikely(len > CAAM_QI_MEMCACHE_SIZE)) {
> dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
> qm_sg_ents, ivsize);
> caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
> @@ -1330,18 +1337,18 @@ static struct skcipher_edesc
> *skcipher_edesc_alloc(struct skcipher_request *req,
> }
>
> /* allocate space for base edesc, link tables and IV */
> - iv = qi_cache_alloc(flags);
> - if (unlikely(!iv)) {
> + edesc = qi_cache_alloc(flags);
> + if (unlikely(!edesc)) {
> dev_err(qidev, "could not allocate extended descriptor\n");
> caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
> 0, DMA_NONE, 0, 0);
> return ERR_PTR(-ENOMEM);
> }
>
> - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
> -
> /* Make sure IV is located in a DMAable area */
> sg_table = &edesc->sgt[0];
> + iv = (u8 *)(sg_table + qm_sg_ents);
> + iv = PTR_ALIGN(iv, dma_get_cache_alignment());
> memcpy(iv, req->iv, ivsize);
>
> iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); diff -
> -git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index
> 4c52c9365558..2ad2c1035856 100644
> --- a/drivers/crypto/caam/qi.c
> +++ b/drivers/crypto/caam/qi.c
> @@ -8,7 +8,13 @@
> */
>
> #include <linux/cpumask.h>
> +#include <linux/device.h>
> +#include <linux/dma-mapping.h>
> +#include <linux/kernel.h>
> #include <linux/kthread.h>
> +#include <linux/netdevice.h>
> +#include <linux/slab.h>
> +#include <linux/string.h>
> #include <soc/fsl/qman.h>
>
> #include "debugfs.h"
> @@ -755,8 +761,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
> napi_enable(irqtask);
> }
>
> - qi_cache = kmem_cache_create("caamqicache",
> CAAM_QI_MEMCACHE_SIZE, 0,
> - 0, NULL);
> + qi_cache = kmem_cache_create("caamqicache",
> CAAM_QI_MEMCACHE_SIZE,
> + dma_get_cache_alignment(), 0, NULL);
> if (!qi_cache) {
> dev_err(qidev, "Can't allocate CAAM cache\n");
> free_rsp_fqs();
> --
> Email: Herbert Xu <herbert@gondor.apana.org.au> Home Page:
> https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fgondor.ap
> ana.org.au%2F~herbert%2F&data=05%7C01%7Cmeenakshi.aggarwal%40nxp.co
> m%7C02a5c1fbb5f54a893c0a08db165073cb%7C686ea1d3bc2b4c6fa92cd99c5c
> 301635%7C0%7C0%7C638128307024161720%7CUnknown%7CTWFpbGZsb3d8e
> yJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7
> C3000%7C%7C%7C&sdata=51qldsOV%2FNUO5VEMSd4kKbQdqIOJdEWF99Us%2
> FR01sUA%3D&reserved=0
> PGP Key:
> https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fgondor.ap
> ana.org.au%2F~herbert%2Fpubkey.txt&data=05%7C01%7Cmeenakshi.aggarwal
> %40nxp.com%7C02a5c1fbb5f54a893c0a08db165073cb%7C686ea1d3bc2b4c6fa
> 92cd99c5c301635%7C0%7C0%7C638128307024161720%7CUnknown%7CTWFp
> bGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6
> Mn0%3D%7C3000%7C%7C%7C&sdata=ME%2BROVEsFMovZ1P4Y3KOONWHwA
> ezgFgxOOe0y5fToCA%3D&reserved=0
^ permalink raw reply [flat|nested] 31+ messages in thread
* [v2 PATCH] crypto: caam - Fix edesc/iv ordering mixup
2023-02-27 5:20 ` Meenakshi Aggarwal
@ 2023-02-27 8:25 ` Herbert Xu
2023-02-28 7:05 ` Meenakshi Aggarwal
0 siblings, 1 reply; 31+ messages in thread
From: Herbert Xu @ 2023-02-27 8:25 UTC (permalink / raw)
To: Meenakshi Aggarwal
Cc: Linux Crypto Mailing List, Christoph Hellwig, Horia Geanta,
Pankaj Gupta, Gaurav Jain
On Mon, Feb 27, 2023 at 05:20:32AM +0000, Meenakshi Aggarwal wrote:
> Hi Herbert,
>
> I have tested your changes, not facing a kernel crash now but still kernel warning messages are coming:
Thanks for testing! Indeed, I forgot to update the IV calculations
on the way out:
---8<---
The attempt to add DMA alignment padding by moving IV to the front
of edesc was completely broken as it didn't change the places where
edesc was freed.
It's also wrong as the IV may still share a cache-line with the
edesc.
Fix this by restoring the original layout and simply reserving
enough memmory so that the IV is on a DMA cache-line by itself.
Reported-by: Meenakshi Aggarwal <meenakshi.aggarwal@nxp.com>
Fixes: 199354d7fb6e ("crypto: caam - Remove GFP_DMA and add DMA alignment padding")
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 4a9b998a8d26..12b1c8346243 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -60,7 +60,11 @@
#include <crypto/xts.h>
#include <asm/unaligned.h>
#include <linux/dma-mapping.h>
+#include <linux/device.h>
+#include <linux/err.h>
#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/string.h>
/*
* crypto alg
@@ -1000,6 +1004,13 @@ static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,
crypto_finalize_aead_request(jrp->engine, req, ecode);
}
+static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc)
+{
+
+ return PTR_ALIGN((u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
+ dma_get_cache_alignment());
+}
+
static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
void *context)
{
@@ -1027,8 +1038,7 @@ static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
* This is used e.g. by the CTS mode.
*/
if (ivsize && !ecode) {
- memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
- ivsize);
+ memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize);
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
@@ -1683,18 +1693,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
/*
* allocate space for base edesc and hw desc commands, link tables, IV
*/
- aligned_size = ALIGN(ivsize, __alignof__(*edesc));
- aligned_size += sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
+ aligned_size = sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());
- iv = kzalloc(aligned_size, flags);
- if (!iv) {
+ aligned_size += ~(ARCH_KMALLOC_MINALIGN - 1) &
+ (dma_get_cache_alignment() - 1);
+ aligned_size += ALIGN(ivsize, dma_get_cache_alignment());
+ edesc = kzalloc(aligned_size, flags);
+ if (!edesc) {
dev_err(jrdev, "could not allocate extended descriptor\n");
caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
return ERR_PTR(-ENOMEM);
}
- edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
edesc->src_nents = src_nents;
edesc->dst_nents = dst_nents;
edesc->mapped_src_nents = mapped_src_nents;
@@ -1706,6 +1717,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
/* Make sure IV is located in a DMAable area */
if (ivsize) {
+ iv = skcipher_edesc_iv(edesc);
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index 5e218bf20d5b..743ce50c14f2 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -20,8 +20,11 @@
#include "caamalg_desc.h"
#include <crypto/xts.h>
#include <asm/unaligned.h>
+#include <linux/device.h>
+#include <linux/err.h>
#include <linux/dma-mapping.h>
#include <linux/kernel.h>
+#include <linux/string.h>
/*
* crypto alg
@@ -1204,6 +1207,12 @@ static int ipsec_gcm_decrypt(struct aead_request *req)
false);
}
+static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc)
+{
+ return PTR_ALIGN((u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
+ dma_get_cache_alignment());
+}
+
static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
{
struct skcipher_edesc *edesc;
@@ -1236,8 +1245,7 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
* This is used e.g. by the CTS mode.
*/
if (!ecode)
- memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
- ivsize);
+ memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize);
qi_cache_free(edesc);
skcipher_request_complete(req, ecode);
@@ -1259,6 +1267,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
struct qm_sg_entry *sg_table, *fd_sgt;
struct caam_drv_ctx *drv_ctx;
+ unsigned int len;
drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
if (IS_ERR(drv_ctx))
@@ -1319,9 +1328,12 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
- if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) +
- offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes >
- CAAM_QI_MEMCACHE_SIZE)) {
+
+ len = offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes;
+ len = ALIGN(len, dma_get_cache_alignment());
+ len += ivsize;
+
+ if (unlikely(len > CAAM_QI_MEMCACHE_SIZE)) {
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -1330,18 +1342,24 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
}
/* allocate space for base edesc, link tables and IV */
- iv = qi_cache_alloc(flags);
- if (unlikely(!iv)) {
+ edesc = qi_cache_alloc(flags);
+ if (unlikely(!edesc)) {
dev_err(qidev, "could not allocate extended descriptor\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
- edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
+ edesc->src_nents = src_nents;
+ edesc->dst_nents = dst_nents;
+ edesc->qm_sg_bytes = qm_sg_bytes;
+ edesc->drv_req.app_ctx = req;
+ edesc->drv_req.cbk = skcipher_done;
+ edesc->drv_req.drv_ctx = drv_ctx;
/* Make sure IV is located in a DMAable area */
sg_table = &edesc->sgt[0];
+ iv = skcipher_edesc_iv(edesc);
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
@@ -1353,13 +1371,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
return ERR_PTR(-ENOMEM);
}
- edesc->src_nents = src_nents;
- edesc->dst_nents = dst_nents;
edesc->iv_dma = iv_dma;
- edesc->qm_sg_bytes = qm_sg_bytes;
- edesc->drv_req.app_ctx = req;
- edesc->drv_req.cbk = skcipher_done;
- edesc->drv_req.drv_ctx = drv_ctx;
dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index 4c52c9365558..2ad2c1035856 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -8,7 +8,13 @@
*/
#include <linux/cpumask.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
#include <linux/kthread.h>
+#include <linux/netdevice.h>
+#include <linux/slab.h>
+#include <linux/string.h>
#include <soc/fsl/qman.h>
#include "debugfs.h"
@@ -755,8 +761,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
napi_enable(irqtask);
}
- qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
- 0, NULL);
+ qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE,
+ dma_get_cache_alignment(), 0, NULL);
if (!qi_cache) {
dev_err(qidev, "Can't allocate CAAM cache\n");
free_rsp_fqs();
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply related [flat|nested] 31+ messages in thread
* RE: [v2 PATCH] crypto: caam - Fix edesc/iv ordering mixup
2023-02-27 8:25 ` [v2 PATCH] " Herbert Xu
@ 2023-02-28 7:05 ` Meenakshi Aggarwal
0 siblings, 0 replies; 31+ messages in thread
From: Meenakshi Aggarwal @ 2023-02-28 7:05 UTC (permalink / raw)
To: Herbert Xu
Cc: Linux Crypto Mailing List, Christoph Hellwig, Horia Geanta,
Pankaj Gupta, Gaurav Jain
Hi Herbert,
Changes are working for me.
Thanks,
Meenakshi
> -----Original Message-----
> From: Herbert Xu <herbert@gondor.apana.org.au>
> Sent: Monday, February 27, 2023 1:56 PM
> To: Meenakshi Aggarwal <meenakshi.aggarwal@nxp.com>
> Cc: Linux Crypto Mailing List <linux-crypto@vger.kernel.org>; Christoph Hellwig
> <hch@infradead.org>; Horia Geanta <horia.geanta@nxp.com>; Pankaj Gupta
> <pankaj.gupta@nxp.com>; Gaurav Jain <gaurav.jain@nxp.com>
> Subject: [v2 PATCH] crypto: caam - Fix edesc/iv ordering mixup
>
> On Mon, Feb 27, 2023 at 05:20:32AM +0000, Meenakshi Aggarwal wrote:
> > Hi Herbert,
> >
> > I have tested your changes, not facing a kernel crash now but still kernel
> warning messages are coming:
>
> Thanks for testing! Indeed, I forgot to update the IV calculations on the way out:
>
> ---8<---
> The attempt to add DMA alignment padding by moving IV to the front of edesc
> was completely broken as it didn't change the places where edesc was freed.
>
> It's also wrong as the IV may still share a cache-line with the edesc.
>
> Fix this by restoring the original layout and simply reserving enough memmory
> so that the IV is on a DMA cache-line by itself.
>
> Reported-by: Meenakshi Aggarwal <meenakshi.aggarwal@nxp.com>
> Fixes: 199354d7fb6e ("crypto: caam - Remove GFP_DMA and add DMA
> alignment padding")
> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
>
> diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
> index 4a9b998a8d26..12b1c8346243 100644
> --- a/drivers/crypto/caam/caamalg.c
> +++ b/drivers/crypto/caam/caamalg.c
> @@ -60,7 +60,11 @@
> #include <crypto/xts.h>
> #include <asm/unaligned.h>
> #include <linux/dma-mapping.h>
> +#include <linux/device.h>
> +#include <linux/err.h>
> #include <linux/kernel.h>
> +#include <linux/slab.h>
> +#include <linux/string.h>
>
> /*
> * crypto alg
> @@ -1000,6 +1004,13 @@ static void aead_crypt_done(struct device *jrdev,
> u32 *desc, u32 err,
> crypto_finalize_aead_request(jrp->engine, req, ecode); }
>
> +static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc) {
> +
> + return PTR_ALIGN((u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
> + dma_get_cache_alignment());
> +}
> +
> static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
> void *context)
> {
> @@ -1027,8 +1038,7 @@ static void skcipher_crypt_done(struct device *jrdev,
> u32 *desc, u32 err,
> * This is used e.g. by the CTS mode.
> */
> if (ivsize && !ecode) {
> - memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
> - ivsize);
> + memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize);
>
> print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
> DUMP_PREFIX_ADDRESS, 16, 4, req->iv, @@
> -1683,18 +1693,19 @@ static struct skcipher_edesc
> *skcipher_edesc_alloc(struct skcipher_request *req,
> /*
> * allocate space for base edesc and hw desc commands, link tables, IV
> */
> - aligned_size = ALIGN(ivsize, __alignof__(*edesc));
> - aligned_size += sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
> + aligned_size = sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
> aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());
> - iv = kzalloc(aligned_size, flags);
> - if (!iv) {
> + aligned_size += ~(ARCH_KMALLOC_MINALIGN - 1) &
> + (dma_get_cache_alignment() - 1);
> + aligned_size += ALIGN(ivsize, dma_get_cache_alignment());
> + edesc = kzalloc(aligned_size, flags);
> + if (!edesc) {
> dev_err(jrdev, "could not allocate extended descriptor\n");
> caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
> 0, 0, 0);
> return ERR_PTR(-ENOMEM);
> }
>
> - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
> edesc->src_nents = src_nents;
> edesc->dst_nents = dst_nents;
> edesc->mapped_src_nents = mapped_src_nents; @@ -1706,6 +1717,7
> @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request
> *req,
>
> /* Make sure IV is located in a DMAable area */
> if (ivsize) {
> + iv = skcipher_edesc_iv(edesc);
> memcpy(iv, req->iv, ivsize);
>
> iv_dma = dma_map_single(jrdev, iv, ivsize,
> DMA_BIDIRECTIONAL); diff --git a/drivers/crypto/caam/caamalg_qi.c
> b/drivers/crypto/caam/caamalg_qi.c
> index 5e218bf20d5b..743ce50c14f2 100644
> --- a/drivers/crypto/caam/caamalg_qi.c
> +++ b/drivers/crypto/caam/caamalg_qi.c
> @@ -20,8 +20,11 @@
> #include "caamalg_desc.h"
> #include <crypto/xts.h>
> #include <asm/unaligned.h>
> +#include <linux/device.h>
> +#include <linux/err.h>
> #include <linux/dma-mapping.h>
> #include <linux/kernel.h>
> +#include <linux/string.h>
>
> /*
> * crypto alg
> @@ -1204,6 +1207,12 @@ static int ipsec_gcm_decrypt(struct aead_request
> *req)
> false);
> }
>
> +static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc) {
> + return PTR_ALIGN((u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
> + dma_get_cache_alignment());
> +}
> +
> static void skcipher_done(struct caam_drv_req *drv_req, u32 status) {
> struct skcipher_edesc *edesc;
> @@ -1236,8 +1245,7 @@ static void skcipher_done(struct caam_drv_req
> *drv_req, u32 status)
> * This is used e.g. by the CTS mode.
> */
> if (!ecode)
> - memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
> - ivsize);
> + memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize);
>
> qi_cache_free(edesc);
> skcipher_request_complete(req, ecode); @@ -1259,6 +1267,7 @@
> static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
> int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
> struct qm_sg_entry *sg_table, *fd_sgt;
> struct caam_drv_ctx *drv_ctx;
> + unsigned int len;
>
> drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
> if (IS_ERR(drv_ctx))
> @@ -1319,9 +1328,12 @@ static struct skcipher_edesc
> *skcipher_edesc_alloc(struct skcipher_request *req,
> qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
>
> qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
> - if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) +
> - offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes >
> - CAAM_QI_MEMCACHE_SIZE)) {
> +
> + len = offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes;
> + len = ALIGN(len, dma_get_cache_alignment());
> + len += ivsize;
> +
> + if (unlikely(len > CAAM_QI_MEMCACHE_SIZE)) {
> dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
> qm_sg_ents, ivsize);
> caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
> @@ -1330,18 +1342,24 @@ static struct skcipher_edesc
> *skcipher_edesc_alloc(struct skcipher_request *req,
> }
>
> /* allocate space for base edesc, link tables and IV */
> - iv = qi_cache_alloc(flags);
> - if (unlikely(!iv)) {
> + edesc = qi_cache_alloc(flags);
> + if (unlikely(!edesc)) {
> dev_err(qidev, "could not allocate extended descriptor\n");
> caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
> 0, DMA_NONE, 0, 0);
> return ERR_PTR(-ENOMEM);
> }
>
> - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc)));
> + edesc->src_nents = src_nents;
> + edesc->dst_nents = dst_nents;
> + edesc->qm_sg_bytes = qm_sg_bytes;
> + edesc->drv_req.app_ctx = req;
> + edesc->drv_req.cbk = skcipher_done;
> + edesc->drv_req.drv_ctx = drv_ctx;
>
> /* Make sure IV is located in a DMAable area */
> sg_table = &edesc->sgt[0];
> + iv = skcipher_edesc_iv(edesc);
> memcpy(iv, req->iv, ivsize);
>
> iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); @@
> -1353,13 +1371,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct
> skcipher_request *req,
> return ERR_PTR(-ENOMEM);
> }
>
> - edesc->src_nents = src_nents;
> - edesc->dst_nents = dst_nents;
> edesc->iv_dma = iv_dma;
> - edesc->qm_sg_bytes = qm_sg_bytes;
> - edesc->drv_req.app_ctx = req;
> - edesc->drv_req.cbk = skcipher_done;
> - edesc->drv_req.drv_ctx = drv_ctx;
>
> dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
> sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); diff --git
> a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index
> 4c52c9365558..2ad2c1035856 100644
> --- a/drivers/crypto/caam/qi.c
> +++ b/drivers/crypto/caam/qi.c
> @@ -8,7 +8,13 @@
> */
>
> #include <linux/cpumask.h>
> +#include <linux/device.h>
> +#include <linux/dma-mapping.h>
> +#include <linux/kernel.h>
> #include <linux/kthread.h>
> +#include <linux/netdevice.h>
> +#include <linux/slab.h>
> +#include <linux/string.h>
> #include <soc/fsl/qman.h>
>
> #include "debugfs.h"
> @@ -755,8 +761,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
> napi_enable(irqtask);
> }
>
> - qi_cache = kmem_cache_create("caamqicache",
> CAAM_QI_MEMCACHE_SIZE, 0,
> - 0, NULL);
> + qi_cache = kmem_cache_create("caamqicache",
> CAAM_QI_MEMCACHE_SIZE,
> + dma_get_cache_alignment(), 0, NULL);
> if (!qi_cache) {
> dev_err(qidev, "Can't allocate CAAM cache\n");
> free_rsp_fqs();
> --
> Email: Herbert Xu <herbert@gondor.apana.org.au> Home Page:
> https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fgondor.ap
> ana.org.au%2F~herbert%2F&data=05%7C01%7Cmeenakshi.aggarwal%40nxp.co
> m%7Cfa91dc99b1b147f0b13008db189c4c71%7C686ea1d3bc2b4c6fa92cd99c5c
> 301635%7C0%7C0%7C638130831812121500%7CUnknown%7CTWFpbGZsb3d8e
> yJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7
> C3000%7C%7C%7C&sdata=UScavImC4%2FcLjGiqTDANWb%2FG1cOg4J18Ijfoicl
> %2FRSg%3D&reserved=0
> PGP Key:
> https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fgondor.ap
> ana.org.au%2F~herbert%2Fpubkey.txt&data=05%7C01%7Cmeenakshi.aggarwal
> %40nxp.com%7Cfa91dc99b1b147f0b13008db189c4c71%7C686ea1d3bc2b4c6fa
> 92cd99c5c301635%7C0%7C0%7C638130831812121500%7CUnknown%7CTWFp
> bGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6
> Mn0%3D%7C3000%7C%7C%7C&sdata=ub4VeaCOKFt4iDVcxy3jnN0EXJBPJZEs%2
> B1CrnK5gimg%3D&reserved=0
^ permalink raw reply [flat|nested] 31+ messages in thread
end of thread, other threads:[~2023-02-28 7:11 UTC | newest]
Thread overview: 31+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-12-02 9:19 [PATCH 0/10] crypto: Driver conversions for DMA alignment Herbert Xu
2022-12-02 9:20 ` [PATCH 1/10] crypto: cavium - Set DMA alignment explicitly Herbert Xu
2022-12-02 9:20 ` [PATCH 2/10] crypto: ccp " Herbert Xu
2022-12-02 9:20 ` [PATCH 3/10] crypto: ccree " Herbert Xu
2022-12-02 9:20 ` [PATCH 4/10] crypto: chelsio " Herbert Xu
2022-12-02 9:20 ` [PATCH 5/10] crypto: hisilicon/hpre " Herbert Xu
2022-12-02 9:20 ` [PATCH 6/10] crypto: safexcel " Herbert Xu
2022-12-02 9:20 ` [PATCH 7/10] crypto: keembay " Herbert Xu
2022-12-02 9:21 ` [PATCH 8/10] crypto: octeontx " Herbert Xu
2022-12-02 9:21 ` [PATCH 9/10] crypto: octeontx2 " Herbert Xu
2022-12-02 9:21 ` [PATCH 10/10] crypto: qce " Herbert Xu
2022-12-02 9:49 ` [PATCH 0/10] crypto: Driver conversions for DMA alignment liulongfang
2022-12-02 9:53 ` Herbert Xu
2022-12-04 9:32 ` Christoph Hellwig
2022-12-06 4:13 ` Herbert Xu
2022-12-06 6:27 ` Christoph Hellwig
2022-12-06 8:34 ` Herbert Xu
2022-12-29 8:58 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Herbert Xu
2022-12-30 5:21 ` [PATCH] crypto: caam " Herbert Xu
2022-12-30 7:31 ` [PATCH] crypto: talitos " Herbert Xu
2023-01-08 17:37 ` Christoph Hellwig
2023-01-09 7:18 ` Christophe Leroy
2023-01-09 7:37 ` Herbert Xu
2023-01-08 17:36 ` [PATCH] crypto: caam " Christoph Hellwig
[not found] ` <DU0PR04MB95635D72885111458C50DB6F8EA89@DU0PR04MB9563.eurprd04.prod.outlook.com>
2023-02-24 10:18 ` [PATCH] crypto: caam - Fix edesc/iv ordering mixup Herbert Xu
2023-02-27 5:20 ` Meenakshi Aggarwal
2023-02-27 8:25 ` [v2 PATCH] " Herbert Xu
2023-02-28 7:05 ` Meenakshi Aggarwal
2023-01-03 6:51 ` [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding Corentin Labbe
2023-01-03 7:26 ` Herbert Xu
2023-01-08 17:36 ` Christoph Hellwig
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).