[PATCH 0/8] update mediatek crypto driver

[PATCH 0/8] update mediatek crypto driver

[Ryder Lee]

Hi,

This series of patches is a global rework of the mtk driver.
Fix bug - incomplete DMA data transfer when SG buffer dst.len != src.len

It also updates some part of the code to make them more generic. For
instance the crypto request queue management supports both async block
cipher and AEAD requests, which allows us to add support the the GCM mode.
GMAC mode is not supported yet.

Current implementation was validated using the tcrypt
module running modes:
- 10: ecb(aes), cbc(aes), ctr(aes), rfc3686(ctr(aes))
- 35: gcm(aes)
- 2,6,11,12: sha1, sha2 family

tcrypt speed test was run with modes:
- 211: rfc4106(gcm(aes)), gcm(aes)
- 500: ecb(aes), cbc(aes), ctr(aes), rfc3686(ctr(aes))
- 403 ~ 406: sha1, sha2 family

IxChariot multiple pairs throughput 24 hours test:
- IPSec VPN
- MACSec

Ryder Lee (8):
  crypto: mediatek - move HW control data to transformation context
  crypto: mediatek - fix incorrect data transfer result
  crypto: mediatek - make crypto request queue management more generic
  crypto: mediatek - rework crypto request completion
  crypto: mediatek - regroup functions by usage
  crypto: mediatek - fix typo and indentation
  crypto: mediatek - add support to CTR mode
  crypto: mediatek - add support to GCM mode

 drivers/crypto/Kconfig                 |    2 +
 drivers/crypto/mediatek/mtk-aes.c      | 1026 ++++++++++++++++++++++++--------
 drivers/crypto/mediatek/mtk-platform.h |   47 +-
 drivers/crypto/mediatek/mtk-sha.c      |  170 +++---
 4 files changed, 886 insertions(+), 359 deletions(-)

-- 
1.9.1

[PATCH 2/8] crypto: mediatek - fix incorrect data transfer result

[Ryder Lee]

This patch fixes mtk_aes_xmit() data transfer bug.

The original function uses the same loop and ring->pos
to handle both command and result descriptors. But this
produces incomplete results when src.sg_len != dst.sg_len.

To solve the problem, we splits the descriptors into different
loops and uses cmd_pos and res_pos to record them respectively.

Signed-off-by: Ryder Lee <ryder.lee@mediatek.com>
---
 drivers/crypto/mediatek/mtk-aes.c      | 44 ++++++++++++++++++++--------------
 drivers/crypto/mediatek/mtk-platform.h |  6 +++--
 drivers/crypto/mediatek/mtk-sha.c      | 29 ++++++++++++----------
 3 files changed, 47 insertions(+), 32 deletions(-)

diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index 126b93c..b658cb9 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -225,29 +225,25 @@ static int mtk_aes_info_map(struct mtk_cryp *cryp,
 	return 0;
 }
 
+/*
+ * Write descriptors for processing. This will configure the engine, load
+ * the transform information and then start the packet processing.
+ */
 static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 {
 	struct mtk_ring *ring = cryp->ring[aes->id];
 	struct mtk_desc *cmd = NULL, *res = NULL;
-	struct scatterlist *ssg, *dsg;
-	u32 len = aes->src.sg_len;
+	struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg;
+	u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len;
 	int nents;
 
-	/* Fill in the command/result descriptors */
-	for (nents = 0; nents < len; ++nents) {
-		ssg = &aes->src.sg[nents];
-		dsg = &aes->dst.sg[nents];
-
-		cmd = ring->cmd_base + ring->pos;
+	/* Write command descriptors */
+	for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) {
+		cmd = ring->cmd_base + ring->cmd_pos;
 		cmd->hdr = MTK_DESC_BUF_LEN(ssg->length);
 		cmd->buf = cpu_to_le32(sg_dma_address(ssg));
 
-		res = ring->res_base + ring->pos;
-		res->hdr = MTK_DESC_BUF_LEN(dsg->length);
-		res->buf = cpu_to_le32(sg_dma_address(dsg));
-
 		if (nents == 0) {
-			res->hdr |= MTK_DESC_FIRST;
 			cmd->hdr |= MTK_DESC_FIRST |
 				    MTK_DESC_CT_LEN(aes->ctx->ct_size);
 			cmd->ct = cpu_to_le32(aes->ctx->ct_dma);
@@ -255,11 +251,23 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 			cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma);
 		}
 
-		if (++ring->pos == MTK_DESC_NUM)
-			ring->pos = 0;
+		if (++ring->cmd_pos == MTK_DESC_NUM)
+			ring->cmd_pos = 0;
 	}
-
 	cmd->hdr |= MTK_DESC_LAST;
+
+	/* Prepare result descriptors */
+	for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) {
+		res = ring->res_base + ring->res_pos;
+		res->hdr = MTK_DESC_BUF_LEN(dsg->length);
+		res->buf = cpu_to_le32(sg_dma_address(dsg));
+
+		if (nents == 0)
+			res->hdr |= MTK_DESC_FIRST;
+
+		if (++ring->res_pos == MTK_DESC_NUM)
+			ring->res_pos = 0;
+	}
 	res->hdr |= MTK_DESC_LAST;
 
 	/*
@@ -268,8 +276,8 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 	 */
 	wmb();
 	/* Start DMA transfer */
-	mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(len));
-	mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(len));
+	mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen));
+	mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen));
 
 	return -EINPROGRESS;
 }
diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h
index 1516786..8c50b74 100644
--- a/drivers/crypto/mediatek/mtk-platform.h
+++ b/drivers/crypto/mediatek/mtk-platform.h
@@ -83,9 +83,10 @@ struct mtk_desc {
  * struct mtk_ring - Descriptor ring
  * @cmd_base:	pointer to command descriptor ring base
  * @cmd_dma:	DMA address of command descriptor ring
+ * @cmd_pos:	current position in the command descriptor ring
  * @res_base:	pointer to result descriptor ring base
  * @res_dma:	DMA address of result descriptor ring
- * @pos:	current position in the ring
+ * @res_pos:	current position in the result descriptor ring
  *
  * A descriptor ring is a circular buffer that is used to manage
  * one or more descriptors. There are two type of descriptor rings;
@@ -94,9 +95,10 @@ struct mtk_desc {
 struct mtk_ring {
 	struct mtk_desc *cmd_base;
 	dma_addr_t cmd_dma;
+	u32 cmd_pos;
 	struct mtk_desc *res_base;
 	dma_addr_t res_dma;
-	u32 pos;
+	u32 res_pos;
 };
 
 /**
diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c
index 8cbff21..2536ebc 100644
--- a/drivers/crypto/mediatek/mtk-sha.c
+++ b/drivers/crypto/mediatek/mtk-sha.c
@@ -426,8 +426,8 @@ static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha,
 {
 	struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
 	struct mtk_ring *ring = cryp->ring[sha->id];
-	struct mtk_desc *cmd = ring->cmd_base + ring->pos;
-	struct mtk_desc *res = ring->res_base + ring->pos;
+	struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos;
+	struct mtk_desc *res = ring->res_base + ring->res_pos;
 	int err;
 
 	err = mtk_sha_info_map(cryp, sha, len);
@@ -451,9 +451,10 @@ static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha,
 	cmd->ct_hdr = ctx->ct_hdr;
 	cmd->tfm = cpu_to_le32(ctx->tfm_dma);
 
-	if (++ring->pos == MTK_DESC_NUM)
-		ring->pos = 0;
+	if (++ring->cmd_pos == MTK_DESC_NUM)
+		ring->cmd_pos = 0;
 
+	ring->res_pos = ring->cmd_pos;
 	/*
 	 * Make sure that all changes to the DMA ring are done before we
 	 * start engine.
@@ -472,8 +473,8 @@ static int mtk_sha_xmit2(struct mtk_cryp *cryp,
 			 size_t len1, size_t len2)
 {
 	struct mtk_ring *ring = cryp->ring[sha->id];
-	struct mtk_desc *cmd = ring->cmd_base + ring->pos;
-	struct mtk_desc *res = ring->res_base + ring->pos;
+	struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos;
+	struct mtk_desc *res = ring->res_base + ring->res_pos;
 	int err;
 
 	err = mtk_sha_info_map(cryp, sha, len1 + len2);
@@ -492,11 +493,13 @@ static int mtk_sha_xmit2(struct mtk_cryp *cryp,
 	cmd->ct_hdr = ctx->ct_hdr;
 	cmd->tfm = cpu_to_le32(ctx->tfm_dma);
 
-	if (++ring->pos == MTK_DESC_NUM)
-		ring->pos = 0;
+	if (++ring->cmd_pos == MTK_DESC_NUM)
+		ring->cmd_pos = 0;
 
-	cmd = ring->cmd_base + ring->pos;
-	res = ring->res_base + ring->pos;
+	ring->res_pos = ring->cmd_pos;
+
+	cmd = ring->cmd_base + ring->cmd_pos;
+	res = ring->res_base + ring->res_pos;
 
 	res->hdr = MTK_DESC_BUF_LEN(len2) | MTK_DESC_LAST;
 	res->buf = cpu_to_le32(cryp->tmp_dma);
@@ -504,8 +507,10 @@ static int mtk_sha_xmit2(struct mtk_cryp *cryp,
 	cmd->hdr = MTK_DESC_BUF_LEN(len2) | MTK_DESC_LAST;
 	cmd->buf = cpu_to_le32(ctx->dma_addr);
 
-	if (++ring->pos == MTK_DESC_NUM)
-		ring->pos = 0;
+	if (++ring->cmd_pos == MTK_DESC_NUM)
+		ring->cmd_pos = 0;
+
+	ring->res_pos = ring->cmd_pos;
 
 	/*
 	 * Make sure that all changes to the DMA ring are done before we
-- 
1.9.1

[PATCH 3/8] crypto: mediatek - make crypto request queue management more generic

[Ryder Lee]

This patch changes mtk_aes_handle_queue() to make it more generic.
The function argument is now a pointer to struct crypto_async_request,
which is the common base of struct ablkcipher_request and
struct aead_request.

Also this patch introduces struct mtk_aes_base_ctx which will be the
common base of all the transformation contexts.

Hence the very same queue will be used to manage both block cipher and
AEAD requests (such as gcm and authenc implemented in further patches).

Signed-off-by: Ryder Lee <ryder.lee@mediatek.com>
---
 drivers/crypto/mediatek/mtk-aes.c      | 75 ++++++++++++++++++++--------------
 drivers/crypto/mediatek/mtk-platform.h | 14 ++++---
 2 files changed, 53 insertions(+), 36 deletions(-)

diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index b658cb9..7e5a8e0 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -73,9 +73,10 @@ struct mtk_aes_reqctx {
 	u64 mode;
 };
 
-struct mtk_aes_ctx {
+struct mtk_aes_base_ctx {
 	struct mtk_cryp *cryp;
 	u32 keylen;
+	mtk_aes_fn start;
 
 	struct mtk_aes_ct ct;
 	dma_addr_t ct_dma;
@@ -86,6 +87,10 @@ struct mtk_aes_ctx {
 	u32 ct_size;
 };
 
+struct mtk_aes_ctx {
+	struct mtk_aes_base_ctx	base;
+};
+
 struct mtk_aes_drv {
 	struct list_head dev_list;
 	/* Device list lock */
@@ -108,7 +113,7 @@ static inline void mtk_aes_write(struct mtk_cryp *cryp,
 	writel_relaxed(value, cryp->base + offset);
 }
 
-static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_ctx *ctx)
+static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_base_ctx *ctx)
 {
 	struct mtk_cryp *cryp = NULL;
 	struct mtk_cryp *tmp;
@@ -170,7 +175,8 @@ static int mtk_aes_info_map(struct mtk_cryp *cryp,
 			    struct mtk_aes_rec *aes,
 			    size_t len)
 {
-	struct mtk_aes_ctx *ctx = aes->ctx;
+	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
+	struct mtk_aes_base_ctx *ctx = aes->ctx;
 
 	ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
 	ctx->ct.cmd[0] = AES_CMD0 | cpu_to_le32(len);
@@ -189,7 +195,7 @@ static int mtk_aes_info_map(struct mtk_cryp *cryp,
 		ctx->tfm.ctrl[0] |= AES_TFM_192BITS;
 
 	if (aes->flags & AES_FLAGS_CBC) {
-		const u32 *iv = (const u32 *)aes->req->info;
+		const u32 *iv = (const u32 *)req->info;
 		u32 *iv_state = ctx->tfm.state + ctx->keylen;
 		int i;
 
@@ -299,11 +305,10 @@ static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma)
 	sg->length += dma->remainder;
 }
 
-static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
+static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
+		       struct scatterlist *src, struct scatterlist *dst,
+		       size_t len)
 {
-	struct scatterlist *src = aes->req->src;
-	struct scatterlist *dst = aes->req->dst;
-	size_t len = aes->req->nbytes;
 	size_t padlen = 0;
 	bool src_aligned, dst_aligned;
 
@@ -366,18 +371,17 @@ static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 }
 
 static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
-				struct ablkcipher_request *req)
+				struct crypto_async_request *new_areq)
 {
 	struct mtk_aes_rec *aes = cryp->aes[id];
 	struct crypto_async_request *areq, *backlog;
-	struct mtk_aes_reqctx *rctx;
-	struct mtk_aes_ctx *ctx;
+	struct mtk_aes_base_ctx *ctx;
 	unsigned long flags;
-	int err, ret = 0;
+	int ret = 0;
 
 	spin_lock_irqsave(&aes->lock, flags);
-	if (req)
-		ret = ablkcipher_enqueue_request(&aes->queue, req);
+	if (new_areq)
+		ret = crypto_enqueue_request(&aes->queue, new_areq);
 	if (aes->flags & AES_FLAGS_BUSY) {
 		spin_unlock_irqrestore(&aes->lock, flags);
 		return ret;
@@ -394,16 +398,25 @@ static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
 	if (backlog)
 		backlog->complete(backlog, -EINPROGRESS);
 
-	req = ablkcipher_request_cast(areq);
-	ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
+	ctx = crypto_tfm_ctx(areq->tfm);
+
+	aes->areq = areq;
+	aes->ctx = ctx;
+
+	return ctx->start(cryp, aes);
+}
+
+static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
+{
+	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
+	struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
+	int err;
+
 	rctx = ablkcipher_request_ctx(req);
 	rctx->mode &= AES_FLAGS_MODE_MSK;
-	/* Assign new request to device */
-	aes->req = req;
-	aes->ctx = ctx;
 	aes->flags = (aes->flags & ~AES_FLAGS_MODE_MSK) | rctx->mode;
 
-	err = mtk_aes_map(cryp, aes);
+	err = mtk_aes_map(cryp, aes, req->src, req->dst, req->nbytes);
 	if (err)
 		return err;
 
@@ -412,7 +425,7 @@ static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
 
 static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 {
-	struct mtk_aes_ctx *ctx = aes->ctx;
+	struct mtk_aes_base_ctx *ctx = aes->ctx;
 
 	dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct),
 			 DMA_TO_DEVICE);
@@ -449,8 +462,7 @@ static inline void mtk_aes_complete(struct mtk_cryp *cryp,
 				    struct mtk_aes_rec *aes)
 {
 	aes->flags &= ~AES_FLAGS_BUSY;
-
-	aes->req->base.complete(&aes->req->base, 0);
+	aes->areq->complete(aes->areq, 0);
 
 	/* Handle new request */
 	mtk_aes_handle_queue(cryp, aes->id, NULL);
@@ -460,7 +472,7 @@ static inline void mtk_aes_complete(struct mtk_cryp *cryp,
 static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
 			  const u8 *key, u32 keylen)
 {
-	struct mtk_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+	struct mtk_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	const u32 *key_tmp = (const u32 *)key;
 	u32 *key_state = ctx->tfm.state;
 	int i;
@@ -482,14 +494,15 @@ static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
 
 static int mtk_aes_crypt(struct ablkcipher_request *req, u64 mode)
 {
-	struct mtk_aes_ctx *ctx = crypto_ablkcipher_ctx(
-			crypto_ablkcipher_reqtfm(req));
-	struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
+	struct mtk_aes_base_ctx *ctx;
+	struct mtk_aes_reqctx *rctx;
 
+	ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
+	rctx = ablkcipher_request_ctx(req);
 	rctx->mode = mode;
 
 	return mtk_aes_handle_queue(ctx->cryp,
-			!(mode & AES_FLAGS_ENCRYPT), req);
+			!(mode & AES_FLAGS_ENCRYPT), &req->base);
 }
 
 static int mtk_ecb_encrypt(struct ablkcipher_request *req)
@@ -517,14 +530,14 @@ static int mtk_aes_cra_init(struct crypto_tfm *tfm)
 	struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct mtk_cryp *cryp = NULL;
 
-	tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
-
-	cryp = mtk_aes_find_dev(ctx);
+	cryp = mtk_aes_find_dev(&ctx->base);
 	if (!cryp) {
 		pr_err("can't find crypto device\n");
 		return -ENODEV;
 	}
 
+	tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
+	ctx->base.start = mtk_aes_start;
 	return 0;
 }
 
diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h
index 8c50b74..9f5210c 100644
--- a/drivers/crypto/mediatek/mtk-platform.h
+++ b/drivers/crypto/mediatek/mtk-platform.h
@@ -115,12 +115,16 @@ struct mtk_aes_dma {
 	u32 sg_len;
 };
 
-struct mtk_aes_ctx;
+struct mtk_aes_base_ctx;
+struct mtk_aes_rec;
+struct mtk_cryp;
+
+typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes);
 
 /**
  * struct mtk_aes_rec - AES operation record
  * @queue:	crypto request queue
- * @req:	pointer to ablkcipher request
+ * @req:	pointer to async request
  * @task:	the tasklet is use in AES interrupt
  * @ctx:	pointer to current context
  * @src:	the structure that holds source sg list info
@@ -131,15 +135,15 @@ struct mtk_aes_dma {
  * @buf:	pointer to page buffer
  * @id:		record identification
  * @flags:	it's describing AES operation state
- * @lock:	the ablkcipher queue lock
+ * @lock:	the async queue lock
  *
  * Structure used to record AES execution state.
  */
 struct mtk_aes_rec {
 	struct crypto_queue queue;
-	struct ablkcipher_request *req;
+	struct crypto_async_request *areq;
 	struct tasklet_struct task;
-	struct mtk_aes_ctx *ctx;
+	struct mtk_aes_base_ctx *ctx;
 	struct mtk_aes_dma src;
 	struct mtk_aes_dma dst;
 
-- 
1.9.1

[PATCH 1/8] crypto: mediatek - move HW control data to transformation context

[Ryder Lee]

This patch moves hardware control block members from
mtk_*_rec to transformation context and refines related
definition. This makes operational context to manage its
own control information easily for each DMA transfer.

Signed-off-by: Ryder Lee <ryder.lee-NuS5LvNUpcJWk0Htik3J/w@public.gmane.org>
---
 drivers/crypto/mediatek/mtk-aes.c      | 144 ++++++++++++++++-----------------
 drivers/crypto/mediatek/mtk-platform.h |  26 +-----
 drivers/crypto/mediatek/mtk-sha.c      | 101 ++++++++++++-----------
 3 files changed, 126 insertions(+), 145 deletions(-)

diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index 1370cab..126b93c 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -20,23 +20,25 @@
 #define AES_BUF_SIZE		((PAGE_SIZE << AES_BUF_ORDER) \
 				& ~(AES_BLOCK_SIZE - 1))
 
-/* AES command token */
+/* AES command token size */
 #define AES_CT_SIZE_ECB		2
 #define AES_CT_SIZE_CBC		3
 #define AES_CT_CTRL_HDR		cpu_to_le32(0x00220000)
-#define AES_COMMAND0		cpu_to_le32(0x05000000)
-#define AES_COMMAND1		cpu_to_le32(0x2d060000)
-#define AES_COMMAND2		cpu_to_le32(0xe4a63806)
-
-/* AES transform information */
-#define AES_TFM_ECB		cpu_to_le32(0x0 << 0)
-#define AES_TFM_CBC		cpu_to_le32(0x1 << 0)
-#define AES_TFM_DECRYPT		cpu_to_le32(0x5 << 0)
-#define AES_TFM_ENCRYPT		cpu_to_le32(0x4 << 0)
+/* AES-CBC/ECB command token */
+#define AES_CMD0		cpu_to_le32(0x05000000)
+#define AES_CMD1		cpu_to_le32(0x2d060000)
+#define AES_CMD2		cpu_to_le32(0xe4a63806)
+
+/* AES transform information word 0 fields */
+#define AES_TFM_BASIC_OUT	cpu_to_le32(0x4 << 0)
+#define AES_TFM_BASIC_IN	cpu_to_le32(0x5 << 0)
 #define AES_TFM_SIZE(x)		cpu_to_le32((x) << 8)
 #define AES_TFM_128BITS		cpu_to_le32(0xb << 16)
 #define AES_TFM_192BITS		cpu_to_le32(0xd << 16)
 #define AES_TFM_256BITS		cpu_to_le32(0xf << 16)
+/* AES transform information word 1 fields */
+#define AES_TFM_ECB		cpu_to_le32(0x0 << 0)
+#define AES_TFM_CBC		cpu_to_le32(0x1 << 0)
 #define AES_TFM_FULL_IV		cpu_to_le32(0xf << 5)
 
 /* AES flags */
@@ -47,47 +49,41 @@
 #define AES_FLAGS_BUSY		BIT(3)
 
 /**
- * mtk_aes_ct is a set of hardware instructions(command token)
- * that are used to control engine's processing flow of AES.
+ * Command token(CT) is a set of hardware instructions that
+ * are used to control engine's processing flow of AES.
+ *
+ * Transform information(TFM) is used to define AES state and
+ * contains all keys and initial vectors.
+ *
+ * The engine requires CT and TFM to do:
+ * - Commands decoding and control of the engine's data path.
+ * - Coordinating hardware data fetch and store operations.
+ * - Result token construction and output.
  */
 struct mtk_aes_ct {
-	__le32 ct_ctrl0;
-	__le32 ct_ctrl1;
-	__le32 ct_ctrl2;
+	__le32 cmd[AES_CT_SIZE_CBC];
 };
 
-/**
- * mtk_aes_tfm is used to define AES transform state
- * and contains all keys and initial vectors.
- */
 struct mtk_aes_tfm {
-	__le32 tfm_ctrl0;
-	__le32 tfm_ctrl1;
+	__le32 ctrl[2];
 	__le32 state[SIZE_IN_WORDS(AES_KEYSIZE_256 + AES_BLOCK_SIZE)];
 };
 
-/**
- * mtk_aes_info consists of command token and transform state of AES,
- * which should be encapsulated in command and result descriptors.
- *
- * The engine requires this information to do:
- * - Commands decoding and control of the engine's data path.
- * - Coordinating hardware data fetch and store operations.
- * - Result token construction and output.
- */
-struct mtk_aes_info {
-	struct mtk_aes_ct ct;
-	struct mtk_aes_tfm tfm;
-};
-
 struct mtk_aes_reqctx {
 	u64 mode;
 };
 
 struct mtk_aes_ctx {
 	struct mtk_cryp *cryp;
-	struct mtk_aes_info info;
 	u32 keylen;
+
+	struct mtk_aes_ct ct;
+	dma_addr_t ct_dma;
+	struct mtk_aes_tfm tfm;
+	dma_addr_t tfm_dma;
+
+	__le32 ct_hdr;
+	u32 ct_size;
 };
 
 struct mtk_aes_drv {
@@ -174,57 +170,57 @@ static int mtk_aes_info_map(struct mtk_cryp *cryp,
 			    struct mtk_aes_rec *aes,
 			    size_t len)
 {
-	struct mtk_aes_ctx *ctx = crypto_ablkcipher_ctx(
-			crypto_ablkcipher_reqtfm(aes->req));
-	struct mtk_aes_info *info = aes->info;
-	struct mtk_aes_ct *ct = &info->ct;
-	struct mtk_aes_tfm *tfm = &info->tfm;
+	struct mtk_aes_ctx *ctx = aes->ctx;
 
-	aes->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
+	ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
+	ctx->ct.cmd[0] = AES_CMD0 | cpu_to_le32(len);
+	ctx->ct.cmd[1] = AES_CMD1;
 
 	if (aes->flags & AES_FLAGS_ENCRYPT)
-		tfm->tfm_ctrl0 = AES_TFM_ENCRYPT;
+		ctx->tfm.ctrl[0] = AES_TFM_BASIC_OUT;
 	else
-		tfm->tfm_ctrl0 = AES_TFM_DECRYPT;
+		ctx->tfm.ctrl[0] = AES_TFM_BASIC_IN;
 
 	if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128))
-		tfm->tfm_ctrl0 |= AES_TFM_128BITS;
+		ctx->tfm.ctrl[0] |= AES_TFM_128BITS;
 	else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256))
-		tfm->tfm_ctrl0 |= AES_TFM_256BITS;
+		ctx->tfm.ctrl[0] |= AES_TFM_256BITS;
 	else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_192))
-		tfm->tfm_ctrl0 |= AES_TFM_192BITS;
-
-	ct->ct_ctrl0 = AES_COMMAND0 | cpu_to_le32(len);
-	ct->ct_ctrl1 = AES_COMMAND1;
+		ctx->tfm.ctrl[0] |= AES_TFM_192BITS;
 
 	if (aes->flags & AES_FLAGS_CBC) {
 		const u32 *iv = (const u32 *)aes->req->info;
-		u32 *iv_state = tfm->state + ctx->keylen;
+		u32 *iv_state = ctx->tfm.state + ctx->keylen;
 		int i;
 
-		aes->ct_size = AES_CT_SIZE_CBC;
-		ct->ct_ctrl2 = AES_COMMAND2;
-
-		tfm->tfm_ctrl0 |= AES_TFM_SIZE(ctx->keylen +
+		ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen +
 				  SIZE_IN_WORDS(AES_BLOCK_SIZE));
-		tfm->tfm_ctrl1 = AES_TFM_CBC | AES_TFM_FULL_IV;
+		ctx->tfm.ctrl[1] = AES_TFM_CBC | AES_TFM_FULL_IV;
 
 		for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++)
 			iv_state[i] = cpu_to_le32(iv[i]);
 
+		ctx->ct.cmd[2] = AES_CMD2;
+		ctx->ct_size  = AES_CT_SIZE_CBC;
 	} else if (aes->flags & AES_FLAGS_ECB) {
-		aes->ct_size = AES_CT_SIZE_ECB;
-		tfm->tfm_ctrl0 |= AES_TFM_SIZE(ctx->keylen);
-		tfm->tfm_ctrl1 = AES_TFM_ECB;
+		ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen);
+		ctx->tfm.ctrl[1] = AES_TFM_ECB;
+
+		ctx->ct_size = AES_CT_SIZE_ECB;
 	}
 
-	aes->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
-					DMA_TO_DEVICE);
-	if (unlikely(dma_mapping_error(cryp->dev, aes->ct_dma))) {
-		dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info));
+	ctx->ct_dma = dma_map_single(cryp->dev, &ctx->ct, sizeof(ctx->ct),
+				     DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma)))
+		return -EINVAL;
+
+	ctx->tfm_dma = dma_map_single(cryp->dev, &ctx->tfm, sizeof(ctx->tfm),
+				      DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(cryp->dev, ctx->tfm_dma))) {
+		dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm),
+				 DMA_TO_DEVICE);
 		return -EINVAL;
 	}
-	aes->tfm_dma = aes->ct_dma + sizeof(*ct);
 
 	return 0;
 }
@@ -253,10 +249,10 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 		if (nents == 0) {
 			res->hdr |= MTK_DESC_FIRST;
 			cmd->hdr |= MTK_DESC_FIRST |
-				    MTK_DESC_CT_LEN(aes->ct_size);
-			cmd->ct = cpu_to_le32(aes->ct_dma);
-			cmd->ct_hdr = aes->ct_hdr;
-			cmd->tfm = cpu_to_le32(aes->tfm_dma);
+				    MTK_DESC_CT_LEN(aes->ctx->ct_size);
+			cmd->ct = cpu_to_le32(aes->ctx->ct_dma);
+			cmd->ct_hdr = aes->ctx->ct_hdr;
+			cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma);
 		}
 
 		if (++ring->pos == MTK_DESC_NUM)
@@ -396,7 +392,7 @@ static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
 	rctx->mode &= AES_FLAGS_MODE_MSK;
 	/* Assign new request to device */
 	aes->req = req;
-	aes->info = &ctx->info;
+	aes->ctx = ctx;
 	aes->flags = (aes->flags & ~AES_FLAGS_MODE_MSK) | rctx->mode;
 
 	err = mtk_aes_map(cryp, aes);
@@ -408,8 +404,12 @@ static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
 
 static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 {
-	dma_unmap_single(cryp->dev, aes->ct_dma,
-			 sizeof(struct mtk_aes_info), DMA_TO_DEVICE);
+	struct mtk_aes_ctx *ctx = aes->ctx;
+
+	dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct),
+			 DMA_TO_DEVICE);
+	dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm),
+			 DMA_TO_DEVICE);
 
 	if (aes->src.sg == aes->dst.sg) {
 		dma_unmap_sg(cryp->dev, aes->src.sg,
@@ -454,7 +454,7 @@ static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
 {
 	struct mtk_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	const u32 *key_tmp = (const u32 *)key;
-	u32 *key_state = ctx->info.tfm.state;
+	u32 *key_state = ctx->tfm.state;
 	int i;
 
 	if (keylen != AES_KEYSIZE_128 &&
diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h
index 4d4309a..1516786 100644
--- a/drivers/crypto/mediatek/mtk-platform.h
+++ b/drivers/crypto/mediatek/mtk-platform.h
@@ -113,22 +113,20 @@ struct mtk_aes_dma {
 	u32 sg_len;
 };
 
+struct mtk_aes_ctx;
+
 /**
  * struct mtk_aes_rec - AES operation record
  * @queue:	crypto request queue
  * @req:	pointer to ablkcipher request
  * @task:	the tasklet is use in AES interrupt
+ * @ctx:	pointer to current context
  * @src:	the structure that holds source sg list info
  * @dst:	the structure that holds destination sg list info
  * @aligned_sg:	the scatter list is use to alignment
  * @real_dst:	pointer to the destination sg list
  * @total:	request buffer length
  * @buf:	pointer to page buffer
- * @info:	pointer to AES transform state and command token
- * @ct_hdr:	AES command token control field
- * @ct_size:	size of AES command token
- * @ct_dma:	DMA address of AES command token
- * @tfm_dma:	DMA address of AES transform state
  * @id:		record identification
  * @flags:	it's describing AES operation state
  * @lock:	the ablkcipher queue lock
@@ -139,6 +137,7 @@ struct mtk_aes_rec {
 	struct crypto_queue queue;
 	struct ablkcipher_request *req;
 	struct tasklet_struct task;
+	struct mtk_aes_ctx *ctx;
 	struct mtk_aes_dma src;
 	struct mtk_aes_dma dst;
 
@@ -148,12 +147,6 @@ struct mtk_aes_rec {
 	size_t total;
 	void *buf;
 
-	void *info;
-	__le32 ct_hdr;
-	u32 ct_size;
-	dma_addr_t ct_dma;
-	dma_addr_t tfm_dma;
-
 	u8 id;
 	unsigned long flags;
 	/* queue lock */
@@ -165,11 +158,6 @@ struct mtk_aes_rec {
  * @queue:	crypto request queue
  * @req:	pointer to ahash request
  * @task:	the tasklet is use in SHA interrupt
- * @info:	pointer to SHA transform state and command token
- * @ct_hdr:	SHA command token control field
- * @ct_size:	size of SHA command token
- * @ct_dma:	DMA address of SHA command token
- * @tfm_dma:	DMA address of SHA transform state
  * @id:		record identification
  * @flags:	it's describing SHA operation state
  * @lock:	the ablkcipher queue lock
@@ -181,12 +169,6 @@ struct mtk_sha_rec {
 	struct ahash_request *req;
 	struct tasklet_struct task;
 
-	void *info;
-	__le32 ct_hdr;
-	u32 ct_size;
-	dma_addr_t ct_dma;
-	dma_addr_t tfm_dma;
-
 	u8 id;
 	unsigned long flags;
 	/* queue lock */
diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c
index f1e188b..8cbff21 100644
--- a/drivers/crypto/mediatek/mtk-sha.c
+++ b/drivers/crypto/mediatek/mtk-sha.c
@@ -28,9 +28,9 @@
 /* SHA command token */
 #define SHA_CT_SIZE		5
 #define SHA_CT_CTRL_HDR		cpu_to_le32(0x02220000)
-#define SHA_COMMAND0		cpu_to_le32(0x03020000)
-#define SHA_COMMAND1		cpu_to_le32(0x21060000)
-#define SHA_COMMAND2		cpu_to_le32(0xe0e63802)
+#define SHA_CMD0		cpu_to_le32(0x03020000)
+#define SHA_CMD1		cpu_to_le32(0x21060000)
+#define SHA_CMD2		cpu_to_le32(0xe0e63802)
 
 /* SHA transform information */
 #define SHA_TFM_HASH		cpu_to_le32(0x2 << 0)
@@ -66,11 +66,8 @@
  * and it contains the first two words of transform state.
  */
 struct mtk_sha_ct {
-	__le32 tfm_ctrl0;
-	__le32 tfm_ctrl1;
-	__le32 ct_ctrl0;
-	__le32 ct_ctrl1;
-	__le32 ct_ctrl2;
+	__le32 ctrl[2];
+	__le32 cmd[3];
 };
 
 /**
@@ -78,8 +75,7 @@ struct mtk_sha_ct {
  * and store result digest that produced by engine.
  */
 struct mtk_sha_tfm {
-	__le32 tfm_ctrl0;
-	__le32 tfm_ctrl1;
+	__le32 ctrl[2];
 	__le32 digest[SIZE_IN_WORDS(SHA512_DIGEST_SIZE)];
 };
 
@@ -102,6 +98,11 @@ struct mtk_sha_reqctx {
 	size_t bufcnt;
 	dma_addr_t dma_addr;
 
+	__le32 ct_hdr;
+	u32 ct_size;
+	dma_addr_t ct_dma;
+	dma_addr_t tfm_dma;
+
 	/* Walk state */
 	struct scatterlist *sg;
 	u32 offset;	/* Offset in current sg */
@@ -270,34 +271,32 @@ static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len)
 }
 
 /* Initialize basic transform information of SHA */
-static void mtk_sha_info_init(struct mtk_sha_rec *sha,
-			      struct mtk_sha_reqctx *ctx)
+static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx)
 {
-	struct mtk_sha_info *info = sha->info;
-	struct mtk_sha_ct *ct = &info->ct;
-	struct mtk_sha_tfm *tfm = &info->tfm;
+	struct mtk_sha_ct *ct = &ctx->info.ct;
+	struct mtk_sha_tfm *tfm = &ctx->info.tfm;
 
-	sha->ct_hdr = SHA_CT_CTRL_HDR;
-	sha->ct_size = SHA_CT_SIZE;
+	ctx->ct_hdr = SHA_CT_CTRL_HDR;
+	ctx->ct_size = SHA_CT_SIZE;
 
-	tfm->tfm_ctrl0 = SHA_TFM_HASH | SHA_TFM_INNER_DIG |
-			 SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds));
+	tfm->ctrl[0] = SHA_TFM_HASH | SHA_TFM_INNER_DIG |
+		       SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds));
 
 	switch (ctx->flags & SHA_FLAGS_ALGO_MSK) {
 	case SHA_FLAGS_SHA1:
-		tfm->tfm_ctrl0 |= SHA_TFM_SHA1;
+		tfm->ctrl[0] |= SHA_TFM_SHA1;
 		break;
 	case SHA_FLAGS_SHA224:
-		tfm->tfm_ctrl0 |= SHA_TFM_SHA224;
+		tfm->ctrl[0] |= SHA_TFM_SHA224;
 		break;
 	case SHA_FLAGS_SHA256:
-		tfm->tfm_ctrl0 |= SHA_TFM_SHA256;
+		tfm->ctrl[0] |= SHA_TFM_SHA256;
 		break;
 	case SHA_FLAGS_SHA384:
-		tfm->tfm_ctrl0 |= SHA_TFM_SHA384;
+		tfm->ctrl[0] |= SHA_TFM_SHA384;
 		break;
 	case SHA_FLAGS_SHA512:
-		tfm->tfm_ctrl0 |= SHA_TFM_SHA512;
+		tfm->ctrl[0] |= SHA_TFM_SHA512;
 		break;
 
 	default:
@@ -305,13 +304,13 @@ static void mtk_sha_info_init(struct mtk_sha_rec *sha,
 		return;
 	}
 
-	tfm->tfm_ctrl1 = SHA_TFM_HASH_STORE;
-	ct->tfm_ctrl0 = tfm->tfm_ctrl0 | SHA_TFM_CONTINUE | SHA_TFM_START;
-	ct->tfm_ctrl1 = tfm->tfm_ctrl1;
+	tfm->ctrl[1] = SHA_TFM_HASH_STORE;
+	ct->ctrl[0] = tfm->ctrl[0] | SHA_TFM_CONTINUE | SHA_TFM_START;
+	ct->ctrl[1] = tfm->ctrl[1];
 
-	ct->ct_ctrl0 = SHA_COMMAND0;
-	ct->ct_ctrl1 = SHA_COMMAND1;
-	ct->ct_ctrl2 = SHA_COMMAND2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds));
+	ct->cmd[0] = SHA_CMD0;
+	ct->cmd[1] = SHA_CMD1;
+	ct->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds));
 }
 
 /*
@@ -323,28 +322,28 @@ static int mtk_sha_info_map(struct mtk_cryp *cryp,
 			    size_t len)
 {
 	struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
-	struct mtk_sha_info *info = sha->info;
+	struct mtk_sha_info *info = &ctx->info;
 	struct mtk_sha_ct *ct = &info->ct;
 
 	if (ctx->start)
 		ctx->start = false;
 	else
-		ct->tfm_ctrl0 &= ~SHA_TFM_START;
+		ct->ctrl[0] &= ~SHA_TFM_START;
 
-	sha->ct_hdr &= ~SHA_DATA_LEN_MSK;
-	sha->ct_hdr |= cpu_to_le32(len);
-	ct->ct_ctrl0 &= ~SHA_DATA_LEN_MSK;
-	ct->ct_ctrl0 |= cpu_to_le32(len);
+	ctx->ct_hdr &= ~SHA_DATA_LEN_MSK;
+	ctx->ct_hdr |= cpu_to_le32(len);
+	ct->cmd[0] &= ~SHA_DATA_LEN_MSK;
+	ct->cmd[0] |= cpu_to_le32(len);
 
 	ctx->digcnt += len;
 
-	sha->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
+	ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
 				      DMA_BIDIRECTIONAL);
-	if (unlikely(dma_mapping_error(cryp->dev, sha->ct_dma))) {
+	if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) {
 		dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info));
 		return -EINVAL;
 	}
-	sha->tfm_dma = sha->ct_dma + sizeof(*ct);
+	ctx->tfm_dma = ctx->ct_dma + sizeof(*ct);
 
 	return 0;
 }
@@ -425,6 +424,7 @@ static int mtk_sha_init(struct ahash_request *req)
 static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha,
 			dma_addr_t addr, size_t len)
 {
+	struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
 	struct mtk_ring *ring = cryp->ring[sha->id];
 	struct mtk_desc *cmd = ring->cmd_base + ring->pos;
 	struct mtk_desc *res = ring->res_base + ring->pos;
@@ -444,12 +444,12 @@ static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha,
 	cmd->hdr = MTK_DESC_FIRST |
 		   MTK_DESC_LAST |
 		   MTK_DESC_BUF_LEN(len) |
-		   MTK_DESC_CT_LEN(sha->ct_size);
+		   MTK_DESC_CT_LEN(ctx->ct_size);
 
 	cmd->buf = cpu_to_le32(addr);
-	cmd->ct = cpu_to_le32(sha->ct_dma);
-	cmd->ct_hdr = sha->ct_hdr;
-	cmd->tfm = cpu_to_le32(sha->tfm_dma);
+	cmd->ct = cpu_to_le32(ctx->ct_dma);
+	cmd->ct_hdr = ctx->ct_hdr;
+	cmd->tfm = cpu_to_le32(ctx->tfm_dma);
 
 	if (++ring->pos == MTK_DESC_NUM)
 		ring->pos = 0;
@@ -486,11 +486,11 @@ static int mtk_sha_xmit2(struct mtk_cryp *cryp,
 
 	cmd->hdr = MTK_DESC_BUF_LEN(len1) |
 		   MTK_DESC_FIRST |
-		   MTK_DESC_CT_LEN(sha->ct_size);
+		   MTK_DESC_CT_LEN(ctx->ct_size);
 	cmd->buf = cpu_to_le32(sg_dma_address(ctx->sg));
-	cmd->ct = cpu_to_le32(sha->ct_dma);
-	cmd->ct_hdr = sha->ct_hdr;
-	cmd->tfm = cpu_to_le32(sha->tfm_dma);
+	cmd->ct = cpu_to_le32(ctx->ct_dma);
+	cmd->ct_hdr = ctx->ct_hdr;
+	cmd->tfm = cpu_to_le32(ctx->tfm_dma);
 
 	if (++ring->pos == MTK_DESC_NUM)
 		ring->pos = 0;
@@ -732,9 +732,8 @@ static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id,
 	ctx = ahash_request_ctx(req);
 
 	sha->req = req;
-	sha->info = &ctx->info;
 
-	mtk_sha_info_init(sha, ctx);
+	mtk_sha_info_init(ctx);
 
 	if (ctx->op == SHA_OP_UPDATE) {
 		err = mtk_sha_update_start(cryp, sha);
@@ -766,8 +765,8 @@ static void mtk_sha_unmap(struct mtk_cryp *cryp, struct mtk_sha_rec *sha)
 {
 	struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
 
-	dma_unmap_single(cryp->dev, sha->ct_dma,
-			 sizeof(struct mtk_sha_info), DMA_BIDIRECTIONAL);
+	dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info),
+			 DMA_BIDIRECTIONAL);
 
 	if (ctx->flags & SHA_FLAGS_SG) {
 		dma_unmap_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE);
-- 
1.9.1

[PATCH 4/8] crypto: mediatek - rework crypto request completion

[Ryder Lee]

This patch introduces a new callback 'resume' in the struct mtk_aes_rec.
This callback is run to resume/complete the processing of the crypto
request when woken up by AES interrupts when DMA completion.

This callback will help implementing the GCM mode support in further
patches.

Signed-off-by: Ryder Lee <ryder.lee-NuS5LvNUpcJWk0Htik3J/w@public.gmane.org>
---
 drivers/crypto/mediatek/mtk-aes.c      | 25 +++++++++++++------------
 drivers/crypto/mediatek/mtk-platform.h |  3 +++
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index 7e5a8e0..9c4e468 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -406,6 +406,15 @@ static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
 	return ctx->start(cryp, aes);
 }
 
+static int mtk_aes_complete(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
+{
+	aes->flags &= ~AES_FLAGS_BUSY;
+	aes->areq->complete(aes->areq, 0);
+
+	/* Handle new request */
+	return mtk_aes_handle_queue(cryp, aes->id, NULL);
+}
+
 static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 {
 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
@@ -416,6 +425,8 @@ static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 	rctx->mode &= AES_FLAGS_MODE_MSK;
 	aes->flags = (aes->flags & ~AES_FLAGS_MODE_MSK) | rctx->mode;
 
+	aes->resume = mtk_aes_complete;
+
 	err = mtk_aes_map(cryp, aes, req->src, req->dst, req->nbytes);
 	if (err)
 		return err;
@@ -458,16 +469,6 @@ static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 				    aes->buf, aes->total);
 }
 
-static inline void mtk_aes_complete(struct mtk_cryp *cryp,
-				    struct mtk_aes_rec *aes)
-{
-	aes->flags &= ~AES_FLAGS_BUSY;
-	aes->areq->complete(aes->areq, 0);
-
-	/* Handle new request */
-	mtk_aes_handle_queue(cryp, aes->id, NULL);
-}
-
 /* Check and set the AES key to transform state buffer */
 static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
 			  const u8 *key, u32 keylen)
@@ -591,7 +592,7 @@ static void mtk_aes_enc_task(unsigned long data)
 	struct mtk_aes_rec *aes = cryp->aes[0];
 
 	mtk_aes_unmap(cryp, aes);
-	mtk_aes_complete(cryp, aes);
+	aes->resume(cryp, aes);
 }
 
 static void mtk_aes_dec_task(unsigned long data)
@@ -600,7 +601,7 @@ static void mtk_aes_dec_task(unsigned long data)
 	struct mtk_aes_rec *aes = cryp->aes[1];
 
 	mtk_aes_unmap(cryp, aes);
-	mtk_aes_complete(cryp, aes);
+	aes->resume(cryp, aes);
 }
 
 static irqreturn_t mtk_aes_enc_irq(int irq, void *dev_id)
diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h
index 9f5210c..36d166b 100644
--- a/drivers/crypto/mediatek/mtk-platform.h
+++ b/drivers/crypto/mediatek/mtk-platform.h
@@ -131,6 +131,7 @@ struct mtk_aes_dma {
  * @dst:	the structure that holds destination sg list info
  * @aligned_sg:	the scatter list is use to alignment
  * @real_dst:	pointer to the destination sg list
+ * @resume:	pointer to resume function
  * @total:	request buffer length
  * @buf:	pointer to page buffer
  * @id:		record identification
@@ -150,6 +151,8 @@ struct mtk_aes_rec {
 	struct scatterlist aligned_sg;
 	struct scatterlist *real_dst;
 
+	mtk_aes_fn resume;
+
 	size_t total;
 	void *buf;
 
-- 
1.9.1

[PATCH 5/8] crypto: mediatek - regroup functions by usage

[Ryder Lee]

This patch only regroup functions by usage.
This will help to integrate the GCM support patch later by
adjusting some shared code section, such as common code which
will be reused by GCM, AES mode setting, and DMA transfer.

Signed-off-by: Ryder Lee <ryder.lee@mediatek.com>
---
 drivers/crypto/mediatek/mtk-aes.c | 272 ++++++++++++++++++++------------------
 1 file changed, 141 insertions(+), 131 deletions(-)

diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index 9c4e468..b5946e9 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -42,7 +42,6 @@
 #define AES_TFM_FULL_IV		cpu_to_le32(0xf << 5)
 
 /* AES flags */
-#define AES_FLAGS_MODE_MSK	0x7
 #define AES_FLAGS_ECB		BIT(0)
 #define AES_FLAGS_CBC		BIT(1)
 #define AES_FLAGS_ENCRYPT	BIT(2)
@@ -170,65 +169,28 @@ static bool mtk_aes_check_aligned(struct scatterlist *sg, size_t len,
 	return false;
 }
 
-/* Initialize and map transform information of AES */
-static int mtk_aes_info_map(struct mtk_cryp *cryp,
-			    struct mtk_aes_rec *aes,
-			    size_t len)
+static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes,
+				    const struct mtk_aes_reqctx *rctx)
 {
-	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
-	struct mtk_aes_base_ctx *ctx = aes->ctx;
-
-	ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
-	ctx->ct.cmd[0] = AES_CMD0 | cpu_to_le32(len);
-	ctx->ct.cmd[1] = AES_CMD1;
-
-	if (aes->flags & AES_FLAGS_ENCRYPT)
-		ctx->tfm.ctrl[0] = AES_TFM_BASIC_OUT;
-	else
-		ctx->tfm.ctrl[0] = AES_TFM_BASIC_IN;
-
-	if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128))
-		ctx->tfm.ctrl[0] |= AES_TFM_128BITS;
-	else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256))
-		ctx->tfm.ctrl[0] |= AES_TFM_256BITS;
-	else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_192))
-		ctx->tfm.ctrl[0] |= AES_TFM_192BITS;
-
-	if (aes->flags & AES_FLAGS_CBC) {
-		const u32 *iv = (const u32 *)req->info;
-		u32 *iv_state = ctx->tfm.state + ctx->keylen;
-		int i;
-
-		ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen +
-				  SIZE_IN_WORDS(AES_BLOCK_SIZE));
-		ctx->tfm.ctrl[1] = AES_TFM_CBC | AES_TFM_FULL_IV;
-
-		for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++)
-			iv_state[i] = cpu_to_le32(iv[i]);
+	/* Clear all but persistent flags and set request flags. */
+	aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode;
+}
 
-		ctx->ct.cmd[2] = AES_CMD2;
-		ctx->ct_size  = AES_CT_SIZE_CBC;
-	} else if (aes->flags & AES_FLAGS_ECB) {
-		ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen);
-		ctx->tfm.ctrl[1] = AES_TFM_ECB;
+static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma)
+{
+	struct scatterlist *sg = dma->sg;
+	int nents = dma->nents;
 
-		ctx->ct_size = AES_CT_SIZE_ECB;
-	}
+	if (!dma->remainder)
+		return;
 
-	ctx->ct_dma = dma_map_single(cryp->dev, &ctx->ct, sizeof(ctx->ct),
-				     DMA_TO_DEVICE);
-	if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma)))
-		return -EINVAL;
+	while (--nents > 0 && sg)
+		sg = sg_next(sg);
 
-	ctx->tfm_dma = dma_map_single(cryp->dev, &ctx->tfm, sizeof(ctx->tfm),
-				      DMA_TO_DEVICE);
-	if (unlikely(dma_mapping_error(cryp->dev, ctx->tfm_dma))) {
-		dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm),
-				 DMA_TO_DEVICE);
-		return -EINVAL;
-	}
+	if (!sg)
+		return;
 
-	return 0;
+	sg->length += dma->remainder;
 }
 
 /*
@@ -288,24 +250,134 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 	return -EINPROGRESS;
 }
 
-static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma)
+static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 {
-	struct scatterlist *sg = dma->sg;
-	int nents = dma->nents;
+	struct mtk_aes_base_ctx *ctx = aes->ctx;
 
-	if (!dma->remainder)
-		return;
+	dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct),
+			 DMA_TO_DEVICE);
+	dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm),
+			 DMA_TO_DEVICE);
 
-	while (--nents > 0 && sg)
-		sg = sg_next(sg);
+	if (aes->src.sg == aes->dst.sg) {
+		dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
+			     DMA_BIDIRECTIONAL);
 
-	if (!sg)
-		return;
+		if (aes->src.sg != &aes->aligned_sg)
+			mtk_aes_restore_sg(&aes->src);
+	} else {
+		dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents,
+			     DMA_FROM_DEVICE);
 
-	sg->length += dma->remainder;
+		if (aes->dst.sg != &aes->aligned_sg)
+			mtk_aes_restore_sg(&aes->dst);
+
+		dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
+			     DMA_TO_DEVICE);
+
+		if (aes->src.sg != &aes->aligned_sg)
+			mtk_aes_restore_sg(&aes->src);
+	}
+
+	if (aes->dst.sg == &aes->aligned_sg)
+		sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst),
+				    aes->buf, aes->total);
 }
 
-static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
+static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
+{
+	struct mtk_aes_base_ctx *ctx = aes->ctx;
+
+	ctx->ct_dma = dma_map_single(cryp->dev, &ctx->ct, sizeof(ctx->ct),
+				     DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma)))
+		return -EINVAL;
+
+	ctx->tfm_dma = dma_map_single(cryp->dev, &ctx->tfm, sizeof(ctx->tfm),
+				      DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(cryp->dev, ctx->tfm_dma)))
+		goto tfm_map_err;
+
+	if (aes->src.sg == aes->dst.sg) {
+		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
+					     aes->src.nents,
+					     DMA_BIDIRECTIONAL);
+		aes->dst.sg_len = aes->src.sg_len;
+		if (unlikely(!aes->src.sg_len))
+			goto sg_map_err;
+	} else {
+		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
+					     aes->src.nents, DMA_TO_DEVICE);
+		if (unlikely(!aes->src.sg_len))
+			goto sg_map_err;
+
+		aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,
+					     aes->dst.nents, DMA_FROM_DEVICE);
+		if (unlikely(!aes->dst.sg_len)) {
+			dma_unmap_sg(cryp->dev, aes->src.sg,
+				     aes->src.nents, DMA_TO_DEVICE);
+			goto sg_map_err;
+		}
+	}
+
+	return mtk_aes_xmit(cryp, aes);
+
+sg_map_err:
+	dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm),
+			 DMA_TO_DEVICE);
+tfm_map_err:
+	dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct),
+			 DMA_TO_DEVICE);
+
+	return -EINVAL;
+}
+
+/* Initialize transform information of CBC/ECB mode */
+static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
+			      size_t len)
+{
+	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
+	struct mtk_aes_base_ctx *ctx = aes->ctx;
+
+	ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
+	ctx->ct.cmd[0] = AES_CMD0 | cpu_to_le32(len);
+	ctx->ct.cmd[1] = AES_CMD1;
+
+	if (aes->flags & AES_FLAGS_ENCRYPT)
+		ctx->tfm.ctrl[0] = AES_TFM_BASIC_OUT;
+	else
+		ctx->tfm.ctrl[0] = AES_TFM_BASIC_IN;
+
+	if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128))
+		ctx->tfm.ctrl[0] |= AES_TFM_128BITS;
+	else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256))
+		ctx->tfm.ctrl[0] |= AES_TFM_256BITS;
+	else
+		ctx->tfm.ctrl[0] |= AES_TFM_192BITS;
+
+	if (aes->flags & AES_FLAGS_CBC) {
+		const u32 *iv = (const u32 *)req->info;
+		u32 *iv_state = ctx->tfm.state + ctx->keylen;
+		int i;
+
+		ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen +
+				    SIZE_IN_WORDS(AES_BLOCK_SIZE));
+		ctx->tfm.ctrl[1] = AES_TFM_CBC | AES_TFM_FULL_IV;
+
+		for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++)
+			iv_state[i] = cpu_to_le32(iv[i]);
+
+		ctx->ct.cmd[2] = AES_CMD2;
+		ctx->ct_size = AES_CT_SIZE_CBC;
+	} else if (aes->flags & AES_FLAGS_ECB) {
+		ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen);
+		ctx->tfm.ctrl[1] = AES_TFM_ECB;
+
+		ctx->ct_size = AES_CT_SIZE_ECB;
+	}
+}
+
+static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
 		       struct scatterlist *src, struct scatterlist *dst,
 		       size_t len)
 {
@@ -346,28 +418,9 @@ static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
 		sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen);
 	}
 
-	if (aes->src.sg == aes->dst.sg) {
-		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
-				aes->src.nents, DMA_BIDIRECTIONAL);
-		aes->dst.sg_len = aes->src.sg_len;
-		if (unlikely(!aes->src.sg_len))
-			return -EFAULT;
-	} else {
-		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
-				aes->src.nents, DMA_TO_DEVICE);
-		if (unlikely(!aes->src.sg_len))
-			return -EFAULT;
-
-		aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,
-				aes->dst.nents, DMA_FROM_DEVICE);
-		if (unlikely(!aes->dst.sg_len)) {
-			dma_unmap_sg(cryp->dev, aes->src.sg,
-				     aes->src.nents, DMA_TO_DEVICE);
-			return -EFAULT;
-		}
-	}
+	mtk_aes_info_init(cryp, aes, len + padlen);
 
-	return mtk_aes_info_map(cryp, aes, len + padlen);
+	return mtk_aes_map(cryp, aes);
 }
 
 static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
@@ -419,54 +472,11 @@ static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 {
 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
 	struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
-	int err;
-
-	rctx = ablkcipher_request_ctx(req);
-	rctx->mode &= AES_FLAGS_MODE_MSK;
-	aes->flags = (aes->flags & ~AES_FLAGS_MODE_MSK) | rctx->mode;
 
+	mtk_aes_set_mode(aes, rctx);
 	aes->resume = mtk_aes_complete;
 
-	err = mtk_aes_map(cryp, aes, req->src, req->dst, req->nbytes);
-	if (err)
-		return err;
-
-	return mtk_aes_xmit(cryp, aes);
-}
-
-static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
-{
-	struct mtk_aes_base_ctx *ctx = aes->ctx;
-
-	dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct),
-			 DMA_TO_DEVICE);
-	dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm),
-			 DMA_TO_DEVICE);
-
-	if (aes->src.sg == aes->dst.sg) {
-		dma_unmap_sg(cryp->dev, aes->src.sg,
-			     aes->src.nents, DMA_BIDIRECTIONAL);
-
-		if (aes->src.sg != &aes->aligned_sg)
-			mtk_aes_restore_sg(&aes->src);
-	} else {
-		dma_unmap_sg(cryp->dev, aes->dst.sg,
-			     aes->dst.nents, DMA_FROM_DEVICE);
-
-		if (aes->dst.sg != &aes->aligned_sg)
-			mtk_aes_restore_sg(&aes->dst);
-
-		dma_unmap_sg(cryp->dev, aes->src.sg,
-			     aes->src.nents, DMA_TO_DEVICE);
-
-		if (aes->src.sg != &aes->aligned_sg)
-			mtk_aes_restore_sg(&aes->src);
-	}
-
-	if (aes->dst.sg == &aes->aligned_sg)
-		sg_copy_from_buffer(aes->real_dst,
-				    sg_nents(aes->real_dst),
-				    aes->buf, aes->total);
+	return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes);
 }
 
 /* Check and set the AES key to transform state buffer */
-- 
1.9.1

[PATCH 6/8] crypto: mediatek - fix typo and indentation

[Ryder Lee]

Dummy patch to fix typo and indentation.

Signed-off-by: Ryder Lee <ryder.lee@mediatek.com>
---
 drivers/crypto/mediatek/mtk-aes.c      | 90 +++++++++++++++++-----------------
 drivers/crypto/mediatek/mtk-platform.h |  2 +-
 drivers/crypto/mediatek/mtk-sha.c      | 40 +++++++--------
 3 files changed, 63 insertions(+), 69 deletions(-)

diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index b5946e9..5e7c3ce 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -314,8 +314,8 @@ static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 		aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,
 					     aes->dst.nents, DMA_FROM_DEVICE);
 		if (unlikely(!aes->dst.sg_len)) {
-			dma_unmap_sg(cryp->dev, aes->src.sg,
-				     aes->src.nents, DMA_TO_DEVICE);
+			dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
+				     DMA_TO_DEVICE);
 			goto sg_map_err;
 		}
 	}
@@ -484,7 +484,7 @@ static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
 			  const u8 *key, u32 keylen)
 {
 	struct mtk_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-	const u32 *key_tmp = (const u32 *)key;
+	const u32 *aes_key = (const u32 *)key;
 	u32 *key_state = ctx->tfm.state;
 	int i;
 
@@ -498,7 +498,7 @@ static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
 	ctx->keylen = SIZE_IN_WORDS(keylen);
 
 	for (i = 0; i < ctx->keylen; i++)
-		key_state[i] = cpu_to_le32(key_tmp[i]);
+		key_state[i] = cpu_to_le32(aes_key[i]);
 
 	return 0;
 }
@@ -512,26 +512,26 @@ static int mtk_aes_crypt(struct ablkcipher_request *req, u64 mode)
 	rctx = ablkcipher_request_ctx(req);
 	rctx->mode = mode;
 
-	return mtk_aes_handle_queue(ctx->cryp,
-			!(mode & AES_FLAGS_ENCRYPT), &req->base);
+	return mtk_aes_handle_queue(ctx->cryp, !(mode & AES_FLAGS_ENCRYPT),
+				    &req->base);
 }
 
-static int mtk_ecb_encrypt(struct ablkcipher_request *req)
+static int mtk_aes_ecb_encrypt(struct ablkcipher_request *req)
 {
 	return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_ECB);
 }
 
-static int mtk_ecb_decrypt(struct ablkcipher_request *req)
+static int mtk_aes_ecb_decrypt(struct ablkcipher_request *req)
 {
 	return mtk_aes_crypt(req, AES_FLAGS_ECB);
 }
 
-static int mtk_cbc_encrypt(struct ablkcipher_request *req)
+static int mtk_aes_cbc_encrypt(struct ablkcipher_request *req)
 {
 	return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CBC);
 }
 
-static int mtk_cbc_decrypt(struct ablkcipher_request *req)
+static int mtk_aes_cbc_decrypt(struct ablkcipher_request *req)
 {
 	return mtk_aes_crypt(req, AES_FLAGS_CBC);
 }
@@ -554,44 +554,44 @@ static int mtk_aes_cra_init(struct crypto_tfm *tfm)
 
 static struct crypto_alg aes_algs[] = {
 {
-	.cra_name		=	"cbc(aes)",
-	.cra_driver_name	=	"cbc-aes-mtk",
-	.cra_priority		=	400,
-	.cra_flags		=	CRYPTO_ALG_TYPE_ABLKCIPHER |
-						CRYPTO_ALG_ASYNC,
-	.cra_init		=	mtk_aes_cra_init,
-	.cra_blocksize		=	AES_BLOCK_SIZE,
-	.cra_ctxsize		=	sizeof(struct mtk_aes_ctx),
-	.cra_alignmask		=	15,
-	.cra_type		=	&crypto_ablkcipher_type,
-	.cra_module		=	THIS_MODULE,
-	.cra_u.ablkcipher	=	{
-		.min_keysize	=	AES_MIN_KEY_SIZE,
-		.max_keysize	=	AES_MAX_KEY_SIZE,
-		.setkey		=	mtk_aes_setkey,
-		.encrypt	=	mtk_cbc_encrypt,
-		.decrypt	=	mtk_cbc_decrypt,
-		.ivsize		=	AES_BLOCK_SIZE,
+	.cra_name		= "cbc(aes)",
+	.cra_driver_name	= "cbc-aes-mtk",
+	.cra_priority		= 400,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_init		= mtk_aes_cra_init,
+	.cra_blocksize		= AES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct mtk_aes_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u.ablkcipher = {
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.setkey		= mtk_aes_setkey,
+		.encrypt	= mtk_aes_cbc_encrypt,
+		.decrypt	= mtk_aes_cbc_decrypt,
+		.ivsize		= AES_BLOCK_SIZE,
 	}
 },
 {
-	.cra_name		=	"ecb(aes)",
-	.cra_driver_name	=	"ecb-aes-mtk",
-	.cra_priority		=	400,
-	.cra_flags		=	CRYPTO_ALG_TYPE_ABLKCIPHER |
-						CRYPTO_ALG_ASYNC,
-	.cra_init		=	mtk_aes_cra_init,
-	.cra_blocksize		=	AES_BLOCK_SIZE,
-	.cra_ctxsize		=	sizeof(struct mtk_aes_ctx),
-	.cra_alignmask		=	15,
-	.cra_type		=	&crypto_ablkcipher_type,
-	.cra_module		=	THIS_MODULE,
-	.cra_u.ablkcipher	=	{
-		.min_keysize	=	AES_MIN_KEY_SIZE,
-		.max_keysize	=	AES_MAX_KEY_SIZE,
-		.setkey		=	mtk_aes_setkey,
-		.encrypt	=	mtk_ecb_encrypt,
-		.decrypt	=	mtk_ecb_decrypt,
+	.cra_name		= "ecb(aes)",
+	.cra_driver_name	= "ecb-aes-mtk",
+	.cra_priority		= 400,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_init		= mtk_aes_cra_init,
+	.cra_blocksize		= AES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct mtk_aes_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u.ablkcipher = {
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.setkey		= mtk_aes_setkey,
+		.encrypt	= mtk_aes_ecb_encrypt,
+		.decrypt	= mtk_aes_ecb_decrypt,
 	}
 },
 };
diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h
index 36d166b..7cd5f98 100644
--- a/drivers/crypto/mediatek/mtk-platform.h
+++ b/drivers/crypto/mediatek/mtk-platform.h
@@ -124,7 +124,7 @@ struct mtk_aes_dma {
 /**
  * struct mtk_aes_rec - AES operation record
  * @queue:	crypto request queue
- * @req:	pointer to async request
+ * @areq:	pointer to async request
  * @task:	the tasklet is use in AES interrupt
  * @ctx:	pointer to current context
  * @src:	the structure that holds source sg list info
diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c
index 2536ebc..55e3805 100644
--- a/drivers/crypto/mediatek/mtk-sha.c
+++ b/drivers/crypto/mediatek/mtk-sha.c
@@ -317,9 +317,9 @@ static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx)
  * Update input data length field of transform information and
  * map it to DMA region.
  */
-static int mtk_sha_info_map(struct mtk_cryp *cryp,
-			    struct mtk_sha_rec *sha,
-			    size_t len)
+static int mtk_sha_info_update(struct mtk_cryp *cryp,
+			       struct mtk_sha_rec *sha,
+			       size_t len)
 {
 	struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
 	struct mtk_sha_info *info = &ctx->info;
@@ -338,7 +338,7 @@ static int mtk_sha_info_map(struct mtk_cryp *cryp,
 	ctx->digcnt += len;
 
 	ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
-				      DMA_BIDIRECTIONAL);
+				     DMA_BIDIRECTIONAL);
 	if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) {
 		dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info));
 		return -EINVAL;
@@ -430,20 +430,15 @@ static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha,
 	struct mtk_desc *res = ring->res_base + ring->res_pos;
 	int err;
 
-	err = mtk_sha_info_map(cryp, sha, len);
+	err = mtk_sha_info_update(cryp, sha, len);
 	if (err)
 		return err;
 
 	/* Fill in the command/result descriptors */
-	res->hdr = MTK_DESC_FIRST |
-		   MTK_DESC_LAST |
-		   MTK_DESC_BUF_LEN(len);
-
+	res->hdr = MTK_DESC_FIRST | MTK_DESC_LAST | MTK_DESC_BUF_LEN(len);
 	res->buf = cpu_to_le32(cryp->tmp_dma);
 
-	cmd->hdr = MTK_DESC_FIRST |
-		   MTK_DESC_LAST |
-		   MTK_DESC_BUF_LEN(len) |
+	cmd->hdr = MTK_DESC_FIRST | MTK_DESC_LAST | MTK_DESC_BUF_LEN(len) |
 		   MTK_DESC_CT_LEN(ctx->ct_size);
 
 	cmd->buf = cpu_to_le32(addr);
@@ -477,7 +472,7 @@ static int mtk_sha_xmit2(struct mtk_cryp *cryp,
 	struct mtk_desc *res = ring->res_base + ring->res_pos;
 	int err;
 
-	err = mtk_sha_info_map(cryp, sha, len1 + len2);
+	err = mtk_sha_info_update(cryp, sha, len1 + len2);
 	if (err)
 		return err;
 
@@ -485,8 +480,7 @@ static int mtk_sha_xmit2(struct mtk_cryp *cryp,
 	res->hdr = MTK_DESC_BUF_LEN(len1) | MTK_DESC_FIRST;
 	res->buf = cpu_to_le32(cryp->tmp_dma);
 
-	cmd->hdr = MTK_DESC_BUF_LEN(len1) |
-		   MTK_DESC_FIRST |
+	cmd->hdr = MTK_DESC_BUF_LEN(len1) | MTK_DESC_FIRST |
 		   MTK_DESC_CT_LEN(ctx->ct_size);
 	cmd->buf = cpu_to_le32(sg_dma_address(ctx->sg));
 	cmd->ct = cpu_to_le32(ctx->ct_dma);
@@ -530,7 +524,7 @@ static int mtk_sha_dma_map(struct mtk_cryp *cryp,
 			   size_t count)
 {
 	ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer,
-				SHA_BUF_SIZE, DMA_TO_DEVICE);
+				       SHA_BUF_SIZE, DMA_TO_DEVICE);
 	if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) {
 		dev_err(cryp->dev, "dma map error\n");
 		return -EINVAL;
@@ -619,7 +613,7 @@ static int mtk_sha_update_start(struct mtk_cryp *cryp,
 		mtk_sha_fill_padding(ctx, len);
 
 		ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer,
-			SHA_BUF_SIZE, DMA_TO_DEVICE);
+					       SHA_BUF_SIZE, DMA_TO_DEVICE);
 		if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) {
 			dev_err(cryp->dev, "dma map bytes error\n");
 			return -EINVAL;
@@ -658,8 +652,7 @@ static int mtk_sha_update_start(struct mtk_cryp *cryp,
 static int mtk_sha_final_req(struct mtk_cryp *cryp,
 			     struct mtk_sha_rec *sha)
 {
-	struct ahash_request *req = sha->req;
-	struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
+	struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
 	size_t count;
 
 	mtk_sha_fill_padding(ctx, 0);
@@ -690,7 +683,8 @@ static int mtk_sha_finish(struct ahash_request *req)
 }
 
 static void mtk_sha_finish_req(struct mtk_cryp *cryp,
-			       struct mtk_sha_rec *sha, int err)
+			       struct mtk_sha_rec *sha,
+			       int err)
 {
 	if (likely(!err && (SHA_FLAGS_FINAL & sha->flags)))
 		err = mtk_sha_finish(sha->req);
@@ -850,8 +844,8 @@ static int mtk_sha_digest(struct ahash_request *req)
 	return mtk_sha_init(req) ?: mtk_sha_finup(req);
 }
 
-static int mtk_sha_setkey(struct crypto_ahash *tfm,
-			  const unsigned char *key, u32 keylen)
+static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
+			  u32 keylen)
 {
 	struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm);
 	struct mtk_sha_hmac_ctx *bctx = tctx->base;
@@ -863,7 +857,7 @@ static int mtk_sha_setkey(struct crypto_ahash *tfm,
 
 	shash->tfm = bctx->shash;
 	shash->flags = crypto_shash_get_flags(bctx->shash) &
-			CRYPTO_TFM_REQ_MAY_SLEEP;
+		       CRYPTO_TFM_REQ_MAY_SLEEP;
 
 	if (keylen > bs) {
 		err = crypto_shash_digest(shash, key, keylen, bctx->ipad);
-- 
1.9.1

[PATCH 7/8] crypto: mediatek - add support to CTR mode

[Ryder Lee]

This patch adds support to the CTR mode.

Signed-off-by: Ryder Lee <ryder.lee@mediatek.com>
---
 drivers/crypto/mediatek/mtk-aes.c | 151 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 146 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index 5e7c3ce..bb5b4ff 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -23,8 +23,10 @@
 /* AES command token size */
 #define AES_CT_SIZE_ECB		2
 #define AES_CT_SIZE_CBC		3
+#define AES_CT_SIZE_CTR		3
 #define AES_CT_CTRL_HDR		cpu_to_le32(0x00220000)
-/* AES-CBC/ECB command token */
+
+/* AES-CBC/ECB/CTR command token */
 #define AES_CMD0		cpu_to_le32(0x05000000)
 #define AES_CMD1		cpu_to_le32(0x2d060000)
 #define AES_CMD2		cpu_to_le32(0xe4a63806)
@@ -39,13 +41,15 @@
 /* AES transform information word 1 fields */
 #define AES_TFM_ECB		cpu_to_le32(0x0 << 0)
 #define AES_TFM_CBC		cpu_to_le32(0x1 << 0)
-#define AES_TFM_FULL_IV		cpu_to_le32(0xf << 5)
+#define AES_TFM_CTR_LOAD	cpu_to_le32(0x6 << 0)	/* load/reuse counter */
+#define AES_TFM_FULL_IV		cpu_to_le32(0xf << 5)	/* using IV 0-3 */
 
 /* AES flags */
 #define AES_FLAGS_ECB		BIT(0)
 #define AES_FLAGS_CBC		BIT(1)
-#define AES_FLAGS_ENCRYPT	BIT(2)
-#define AES_FLAGS_BUSY		BIT(3)
+#define AES_FLAGS_CTR		BIT(2)
+#define AES_FLAGS_ENCRYPT	BIT(3)
+#define AES_FLAGS_BUSY		BIT(4)
 
 /**
  * Command token(CT) is a set of hardware instructions that
@@ -90,6 +94,15 @@ struct mtk_aes_ctx {
 	struct mtk_aes_base_ctx	base;
 };
 
+struct mtk_aes_ctr_ctx {
+	struct mtk_aes_base_ctx base;
+
+	u32	iv[AES_BLOCK_SIZE / sizeof(u32)];
+	size_t offset;
+	struct scatterlist src[2];
+	struct scatterlist dst[2];
+};
+
 struct mtk_aes_drv {
 	struct list_head dev_list;
 	/* Device list lock */
@@ -332,7 +345,7 @@ static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 	return -EINVAL;
 }
 
-/* Initialize transform information of CBC/ECB mode */
+/* Initialize transform information of CBC/ECB/CTR mode */
 static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
 			      size_t len)
 {
@@ -374,6 +387,13 @@ static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
 		ctx->tfm.ctrl[1] = AES_TFM_ECB;
 
 		ctx->ct_size = AES_CT_SIZE_ECB;
+	} else if (aes->flags & AES_FLAGS_CTR) {
+		ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen +
+				    SIZE_IN_WORDS(AES_BLOCK_SIZE));
+		ctx->tfm.ctrl[1] = AES_TFM_CTR_LOAD | AES_TFM_FULL_IV;
+
+		ctx->ct.cmd[2] = AES_CMD2;
+		ctx->ct_size = AES_CT_SIZE_CTR;
 	}
 }
 
@@ -479,6 +499,80 @@ static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 	return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes);
 }
 
+static inline struct mtk_aes_ctr_ctx *
+mtk_aes_ctr_ctx_cast(struct mtk_aes_base_ctx *ctx)
+{
+	return container_of(ctx, struct mtk_aes_ctr_ctx, base);
+}
+
+static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
+{
+	struct mtk_aes_base_ctx *ctx = aes->ctx;
+	struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx);
+	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
+	struct scatterlist *src, *dst;
+	int i;
+	u32 start, end, ctr, blocks, *iv_state;
+	size_t datalen;
+	bool fragmented = false;
+
+	/* Check for transfer completion. */
+	cctx->offset += aes->total;
+	if (cctx->offset >= req->nbytes)
+		return mtk_aes_complete(cryp, aes);
+
+	/* Compute data length. */
+	datalen = req->nbytes - cctx->offset;
+	blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE);
+	ctr = be32_to_cpu(cctx->iv[3]);
+
+	/* Check 32bit counter overflow. */
+	start = ctr;
+	end = start + blocks - 1;
+	if (end < start) {
+		ctr |= 0xffffffff;
+		datalen = AES_BLOCK_SIZE * -start;
+		fragmented = true;
+	}
+
+	/* Jump to offset. */
+	src = scatterwalk_ffwd(cctx->src, req->src, cctx->offset);
+	dst = ((req->src == req->dst) ? src :
+	       scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset));
+
+	/* Write IVs into transform state buffer. */
+	iv_state = ctx->tfm.state + ctx->keylen;
+	for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++)
+		iv_state[i] = cpu_to_le32(cctx->iv[i]);
+
+	if (unlikely(fragmented)) {
+	/*
+	 * Increment the counter manually to cope with the hardware
+	 * counter overflow.
+	 */
+		cctx->iv[3] = cpu_to_be32(ctr);
+		crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE);
+	}
+	aes->resume = mtk_aes_ctr_transfer;
+
+	return mtk_aes_dma(cryp, aes, src, dst, datalen);
+}
+
+static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
+{
+	struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx);
+	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
+	struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
+
+	mtk_aes_set_mode(aes, rctx);
+
+	memcpy(cctx->iv, req->info, AES_BLOCK_SIZE);
+	cctx->offset = 0;
+	aes->total = 0;
+
+	return mtk_aes_ctr_transfer(cryp, aes);
+}
+
 /* Check and set the AES key to transform state buffer */
 static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
 			  const u8 *key, u32 keylen)
@@ -536,6 +630,16 @@ static int mtk_aes_cbc_decrypt(struct ablkcipher_request *req)
 	return mtk_aes_crypt(req, AES_FLAGS_CBC);
 }
 
+static int mtk_aes_ctr_encrypt(struct ablkcipher_request *req)
+{
+	return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CTR);
+}
+
+static int mtk_aes_ctr_decrypt(struct ablkcipher_request *req)
+{
+	return mtk_aes_crypt(req, AES_FLAGS_CTR);
+}
+
 static int mtk_aes_cra_init(struct crypto_tfm *tfm)
 {
 	struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -552,6 +656,22 @@ static int mtk_aes_cra_init(struct crypto_tfm *tfm)
 	return 0;
 }
 
+static int mtk_aes_ctr_cra_init(struct crypto_tfm *tfm)
+{
+	struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct mtk_cryp *cryp = NULL;
+
+	cryp = mtk_aes_find_dev(&ctx->base);
+	if (!cryp) {
+		pr_err("can't find crypto device\n");
+		return -ENODEV;
+	}
+
+	tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
+	ctx->base.start = mtk_aes_ctr_start;
+	return 0;
+}
+
 static struct crypto_alg aes_algs[] = {
 {
 	.cra_name		= "cbc(aes)",
@@ -594,6 +714,27 @@ static int mtk_aes_cra_init(struct crypto_tfm *tfm)
 		.decrypt	= mtk_aes_ecb_decrypt,
 	}
 },
+{
+	.cra_name		= "ctr(aes)",
+	.cra_driver_name	= "ctr-aes-mtk",
+	.cra_priority		= 400,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_init		= mtk_aes_ctr_cra_init,
+	.cra_blocksize		= 1,
+	.cra_ctxsize		= sizeof(struct mtk_aes_ctr_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u.ablkcipher = {
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.ivsize		= AES_BLOCK_SIZE,
+		.setkey		= mtk_aes_setkey,
+		.encrypt	= mtk_aes_ctr_encrypt,
+		.decrypt	= mtk_aes_ctr_decrypt,
+	}
+},
 };
 
 static void mtk_aes_enc_task(unsigned long data)
-- 
1.9.1

[PATCH 8/8] crypto: mediatek - add support to GCM mode

[Ryder Lee]

This patch adds support to the GCM mode.

Signed-off-by: Ryder Lee <ryder.lee@mediatek.com>
---
 drivers/crypto/Kconfig                 |   2 +
 drivers/crypto/mediatek/mtk-aes.c      | 369 ++++++++++++++++++++++++++++++++-
 drivers/crypto/mediatek/mtk-platform.h |   2 +
 3 files changed, 369 insertions(+), 4 deletions(-)

diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index ee5057a..bf7da55 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -557,7 +557,9 @@ config CRYPTO_DEV_MEDIATEK
 	tristate "MediaTek's EIP97 Cryptographic Engine driver"
 	depends on (ARM && ARCH_MEDIATEK) || COMPILE_TEST
 	select CRYPTO_AES
+	select CRYPTO_AEAD
 	select CRYPTO_BLKCIPHER
+	select CRYPTO_CTR
 	select CRYPTO_SHA1
 	select CRYPTO_SHA256
 	select CRYPTO_SHA512
diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index bb5b4ff..3a47cdb 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -24,16 +24,28 @@
 #define AES_CT_SIZE_ECB		2
 #define AES_CT_SIZE_CBC		3
 #define AES_CT_SIZE_CTR		3
+#define AES_CT_SIZE_GCM_OUT	5
+#define AES_CT_SIZE_GCM_IN	6
 #define AES_CT_CTRL_HDR		cpu_to_le32(0x00220000)
 
 /* AES-CBC/ECB/CTR command token */
 #define AES_CMD0		cpu_to_le32(0x05000000)
 #define AES_CMD1		cpu_to_le32(0x2d060000)
 #define AES_CMD2		cpu_to_le32(0xe4a63806)
+/* AES-GCM command token */
+#define AES_GCM_CMD0		cpu_to_le32(0x0b000000)
+#define AES_GCM_CMD1		cpu_to_le32(0xa0800000)
+#define AES_GCM_CMD2		cpu_to_le32(0x25000010)
+#define AES_GCM_CMD3		cpu_to_le32(0x0f020000)
+#define AES_GCM_CMD4		cpu_to_le32(0x21e60000)
+#define AES_GCM_CMD5		cpu_to_le32(0x40e60000)
+#define AES_GCM_CMD6		cpu_to_le32(0xd0070000)
 
 /* AES transform information word 0 fields */
 #define AES_TFM_BASIC_OUT	cpu_to_le32(0x4 << 0)
 #define AES_TFM_BASIC_IN	cpu_to_le32(0x5 << 0)
+#define AES_TFM_GCM_OUT		cpu_to_le32(0x6 << 0)
+#define AES_TFM_GCM_IN		cpu_to_le32(0xf << 0)
 #define AES_TFM_SIZE(x)		cpu_to_le32((x) << 8)
 #define AES_TFM_128BITS		cpu_to_le32(0xb << 16)
 #define AES_TFM_192BITS		cpu_to_le32(0xd << 16)
@@ -41,15 +53,22 @@
 /* AES transform information word 1 fields */
 #define AES_TFM_ECB		cpu_to_le32(0x0 << 0)
 #define AES_TFM_CBC		cpu_to_le32(0x1 << 0)
+#define AES_TFM_CTR_INIT	cpu_to_le32(0x2 << 0)	/* init counter to 1 */
 #define AES_TFM_CTR_LOAD	cpu_to_le32(0x6 << 0)	/* load/reuse counter */
+#define AES_TFM_3IV		cpu_to_le32(0x7 << 5)	/* using IV 0-2 */
 #define AES_TFM_FULL_IV		cpu_to_le32(0xf << 5)	/* using IV 0-3 */
+#define AES_TFM_IV_CTR_MODE	cpu_to_le32(0x1 << 10)
+#define AES_TFM_ENC_HASH	cpu_to_le32(0x1 << 17)
+#define AES_TFM_GHASH_DIG	cpu_to_le32(0x2 << 21)
+#define AES_TFM_GHASH		cpu_to_le32(0x4 << 23)
 
 /* AES flags */
 #define AES_FLAGS_ECB		BIT(0)
 #define AES_FLAGS_CBC		BIT(1)
 #define AES_FLAGS_CTR		BIT(2)
-#define AES_FLAGS_ENCRYPT	BIT(3)
-#define AES_FLAGS_BUSY		BIT(4)
+#define AES_FLAGS_GCM		BIT(3)
+#define AES_FLAGS_ENCRYPT	BIT(4)
+#define AES_FLAGS_BUSY		BIT(5)
 
 /**
  * Command token(CT) is a set of hardware instructions that
@@ -62,14 +81,23 @@
  * - Commands decoding and control of the engine's data path.
  * - Coordinating hardware data fetch and store operations.
  * - Result token construction and output.
+ *
+ * Memory map of GCM's TFM:
+ * /-----------\
+ * |  AES KEY  | 128/196/256 bits
+ * |-----------|
+ * |  HASH KEY | a string 128 zero bits encrypted using the block cipher
+ * |-----------|
+ * |    IVs    | 4 * 4 bytes
+ * \-----------/
  */
 struct mtk_aes_ct {
-	__le32 cmd[AES_CT_SIZE_CBC];
+	__le32 cmd[AES_CT_SIZE_GCM_IN];
 };
 
 struct mtk_aes_tfm {
 	__le32 ctrl[2];
-	__le32 state[SIZE_IN_WORDS(AES_KEYSIZE_256 + AES_BLOCK_SIZE)];
+	__le32 state[SIZE_IN_WORDS(AES_KEYSIZE_256 + AES_BLOCK_SIZE * 2)];
 };
 
 struct mtk_aes_reqctx {
@@ -103,6 +131,20 @@ struct mtk_aes_ctr_ctx {
 	struct scatterlist dst[2];
 };
 
+struct mtk_aes_gcm_ctx {
+	struct mtk_aes_base_ctx base;
+
+	u32 authsize;
+	size_t textlen;
+
+	struct crypto_skcipher *ctr;
+};
+
+struct mtk_aes_gcm_setkey_result {
+	int err;
+	struct completion completion;
+};
+
 struct mtk_aes_drv {
 	struct list_head dev_list;
 	/* Device list lock */
@@ -251,6 +293,10 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
 	}
 	res->hdr |= MTK_DESC_LAST;
 
+	/* Prepare enough space for authenticated tag */
+	if (aes->flags & AES_FLAGS_GCM)
+		res->hdr += AES_BLOCK_SIZE;
+
 	/*
 	 * Make sure that all changes to the DMA ring are done before we
 	 * start engine.
@@ -737,6 +783,315 @@ static int mtk_aes_ctr_cra_init(struct crypto_tfm *tfm)
 },
 };
 
+static inline struct mtk_aes_gcm_ctx *
+mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx)
+{
+	return container_of(ctx, struct mtk_aes_gcm_ctx, base);
+}
+
+/* Initialize transform information of GCM mode */
+static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp,
+				  struct mtk_aes_rec *aes,
+				  size_t len)
+{
+	struct aead_request *req = aead_request_cast(aes->areq);
+	struct mtk_aes_base_ctx *ctx = aes->ctx;
+	struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
+	const u32 *iv = (const u32 *)req->iv;
+	u32 *iv_state = ctx->tfm.state + ctx->keylen +
+			SIZE_IN_WORDS(AES_BLOCK_SIZE);
+	u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
+	int i;
+
+	ctx->ct_hdr = AES_CT_CTRL_HDR | len;
+
+	ctx->ct.cmd[0] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen);
+	ctx->ct.cmd[1] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen);
+	ctx->ct.cmd[2] = AES_GCM_CMD2;
+	ctx->ct.cmd[3] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen);
+
+	if (aes->flags & AES_FLAGS_ENCRYPT) {
+		ctx->ct.cmd[4] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize);
+		ctx->ct_size = AES_CT_SIZE_GCM_OUT;
+		ctx->tfm.ctrl[0] = AES_TFM_GCM_OUT;
+	} else {
+		ctx->ct.cmd[4] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize);
+		ctx->ct.cmd[5] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize);
+		ctx->ct_size = AES_CT_SIZE_GCM_IN;
+		ctx->tfm.ctrl[0] = AES_TFM_GCM_IN;
+	}
+
+	if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128))
+		ctx->tfm.ctrl[0] |= AES_TFM_128BITS;
+	else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256))
+		ctx->tfm.ctrl[0] |= AES_TFM_256BITS;
+	else
+		ctx->tfm.ctrl[0] |= AES_TFM_192BITS;
+
+	ctx->tfm.ctrl[0] |= AES_TFM_GHASH_DIG | AES_TFM_GHASH |
+			    AES_TFM_SIZE(ctx->keylen + SIZE_IN_WORDS(
+			    AES_BLOCK_SIZE + ivsize));
+	ctx->tfm.ctrl[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE |
+			   AES_TFM_3IV | AES_TFM_ENC_HASH;
+
+	for (i = 0; i < SIZE_IN_WORDS(ivsize); i++)
+		iv_state[i] = cpu_to_le32(iv[i]);
+}
+
+static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
+			   struct scatterlist *src, struct scatterlist *dst,
+			   size_t len)
+{
+	bool src_aligned, dst_aligned;
+
+	aes->src.sg = src;
+	aes->dst.sg = dst;
+	aes->real_dst = dst;
+
+	src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
+	if (src == dst)
+		dst_aligned = src_aligned;
+	else
+		dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
+
+	if (!src_aligned || !dst_aligned) {
+		if (aes->total > AES_BUF_SIZE)
+			return -ENOMEM;
+
+		if (!src_aligned) {
+			sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
+			aes->src.sg = &aes->aligned_sg;
+			aes->src.nents = 1;
+			aes->src.remainder = 0;
+		}
+
+		if (!dst_aligned) {
+			aes->dst.sg = &aes->aligned_sg;
+			aes->dst.nents = 1;
+			aes->dst.remainder = 0;
+		}
+
+		sg_init_table(&aes->aligned_sg, 1);
+		sg_set_buf(&aes->aligned_sg, aes->buf, aes->total);
+	}
+
+	mtk_aes_gcm_info_init(cryp, aes, len);
+
+	return mtk_aes_map(cryp, aes);
+}
+
+/* Todo: GMAC */
+static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
+{
+	struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx);
+	struct aead_request *req = aead_request_cast(aes->areq);
+	struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
+	u32 len = req->assoclen + req->cryptlen;
+
+	mtk_aes_set_mode(aes, rctx);
+
+	if (aes->flags & AES_FLAGS_ENCRYPT) {
+		u32 tag[4];
+		/* Compute total process length. */
+		aes->total = len + gctx->authsize;
+		/* Compute text length. */
+		gctx->textlen = req->cryptlen;
+		/* Hardware will append authenticated tag to output buffer */
+		scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1);
+	} else {
+		aes->total = len;
+		gctx->textlen = req->cryptlen - gctx->authsize;
+	}
+	aes->resume = mtk_aes_complete;
+
+	return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);
+}
+
+static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode)
+{
+	struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+	struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
+
+	rctx->mode = AES_FLAGS_GCM | mode;
+
+	return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT),
+								&req->base);
+}
+
+static void mtk_gcm_setkey_done(struct crypto_async_request *req, int err)
+{
+	struct mtk_aes_gcm_setkey_result *result = req->data;
+
+	if (err == -EINPROGRESS)
+		return;
+
+	result->err = err;
+	complete(&result->completion);
+}
+
+/*
+ * Because of the hardware limitation, we need to pre-calculate key(H)
+ * for the GHASH operation. The result of the encryption operation
+ * need to be stored in the transform state buffer.
+ */
+static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
+			      u32 keylen)
+{
+	struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
+	struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
+	struct crypto_skcipher *ctr = gctx->ctr;
+	struct {
+		u32 hash[4];
+		u8 iv[8];
+
+		struct mtk_aes_gcm_setkey_result result;
+
+		struct scatterlist sg[1];
+		struct skcipher_request req;
+	} *data;
+	const u32 *aes_key;
+	u32 *key_state, *hash_state;
+	int err, i;
+
+	if (keylen != AES_KEYSIZE_256 &&
+	    keylen != AES_KEYSIZE_192 &&
+	    keylen != AES_KEYSIZE_128) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	key_state = ctx->tfm.state;
+	aes_key = (u32 *)key;
+	ctx->keylen = SIZE_IN_WORDS(keylen);
+
+	for (i = 0; i < ctx->keylen; i++)
+		ctx->tfm.state[i] = cpu_to_le32(aes_key[i]);
+
+	/* Same as crypto_gcm_setkey() from crypto/gcm.c */
+	crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
+				  CRYPTO_TFM_REQ_MASK);
+	err = crypto_skcipher_setkey(ctr, key, keylen);
+	crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
+			      CRYPTO_TFM_RES_MASK);
+	if (err)
+		return err;
+
+	data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
+		       GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	init_completion(&data->result.completion);
+	sg_init_one(data->sg, &data->hash, AES_BLOCK_SIZE);
+	skcipher_request_set_tfm(&data->req, ctr);
+	skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
+				      CRYPTO_TFM_REQ_MAY_BACKLOG,
+				      mtk_gcm_setkey_done, &data->result);
+	skcipher_request_set_crypt(&data->req, data->sg, data->sg,
+				   AES_BLOCK_SIZE, data->iv);
+
+	err = crypto_skcipher_encrypt(&data->req);
+	if (err == -EINPROGRESS || err == -EBUSY) {
+		err = wait_for_completion_interruptible(
+			&data->result.completion);
+		if (!err)
+			err = data->result.err;
+	}
+	if (err)
+		goto out;
+
+	hash_state = key_state + ctx->keylen;
+
+	for (i = 0; i < 4; i++)
+		hash_state[i] = cpu_to_be32(data->hash[i]);
+out:
+	kzfree(data);
+	return err;
+}
+
+static int mtk_aes_gcm_setauthsize(struct crypto_aead *aead,
+				   u32 authsize)
+{
+	struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
+	struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
+
+	/* Same as crypto_gcm_authsize() from crypto/gcm.c */
+	switch (authsize) {
+	case 8:
+	case 12:
+	case 16:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	gctx->authsize = authsize;
+	return 0;
+}
+
+static int mtk_aes_gcm_encrypt(struct aead_request *req)
+{
+	return mtk_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT);
+}
+
+static int mtk_aes_gcm_decrypt(struct aead_request *req)
+{
+	return mtk_aes_gcm_crypt(req, 0);
+}
+
+static int mtk_aes_gcm_init(struct crypto_aead *aead)
+{
+	struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
+	struct mtk_cryp *cryp = NULL;
+
+	cryp = mtk_aes_find_dev(&ctx->base);
+	if (!cryp) {
+		pr_err("can't find crypto device\n");
+		return -ENODEV;
+	}
+
+	ctx->ctr = crypto_alloc_skcipher("ctr(aes)", 0,
+					 CRYPTO_ALG_ASYNC);
+	if (IS_ERR(ctx->ctr)) {
+		pr_err("Error allocating ctr(aes)\n");
+		return PTR_ERR(ctx->ctr);
+	}
+
+	crypto_aead_set_reqsize(aead, sizeof(struct mtk_aes_reqctx));
+	ctx->base.start = mtk_aes_gcm_start;
+	return 0;
+}
+
+static void mtk_aes_gcm_exit(struct crypto_aead *aead)
+{
+	struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
+
+	crypto_free_skcipher(ctx->ctr);
+}
+
+static struct aead_alg aes_gcm_alg = {
+	.setkey		= mtk_aes_gcm_setkey,
+	.setauthsize	= mtk_aes_gcm_setauthsize,
+	.encrypt	= mtk_aes_gcm_encrypt,
+	.decrypt	= mtk_aes_gcm_decrypt,
+	.init		= mtk_aes_gcm_init,
+	.exit		= mtk_aes_gcm_exit,
+	.ivsize		= 12,
+	.maxauthsize	= AES_BLOCK_SIZE,
+
+	.base = {
+		.cra_name		= "gcm(aes)",
+		.cra_driver_name	= "gcm-aes-mtk",
+		.cra_priority		= 400,
+		.cra_flags		= CRYPTO_ALG_ASYNC,
+		.cra_blocksize		= 1,
+		.cra_ctxsize		= sizeof(struct mtk_aes_gcm_ctx),
+		.cra_alignmask		= 0xf,
+		.cra_module		= THIS_MODULE,
+	},
+};
+
 static void mtk_aes_enc_task(unsigned long data)
 {
 	struct mtk_cryp *cryp = (struct mtk_cryp *)data;
@@ -851,6 +1206,8 @@ static void mtk_aes_unregister_algs(void)
 {
 	int i;
 
+	crypto_unregister_aead(&aes_gcm_alg);
+
 	for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
 		crypto_unregister_alg(&aes_algs[i]);
 }
@@ -865,6 +1222,10 @@ static int mtk_aes_register_algs(void)
 			goto err_aes_algs;
 	}
 
+	err = crypto_register_aead(&aes_gcm_alg);
+	if (err)
+		goto err_aes_algs;
+
 	return 0;
 
 err_aes_algs:
diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h
index 7cd5f98..ed6d871 100644
--- a/drivers/crypto/mediatek/mtk-platform.h
+++ b/drivers/crypto/mediatek/mtk-platform.h
@@ -13,8 +13,10 @@
 #define __MTK_PLATFORM_H_
 
 #include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
 #include <crypto/internal/hash.h>
 #include <crypto/scatterwalk.h>
+#include <crypto/skcipher.h>
 #include <linux/crypto.h>
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
-- 
1.9.1

Re: [PATCH 0/8] update mediatek crypto driver

[Herbert Xu]

On Fri, Jan 20, 2017 at 01:41:07PM +0800, Ryder Lee wrote:
> Hi,
> 
> This series of patches is a global rework of the mtk driver.
> Fix bug - incomplete DMA data transfer when SG buffer dst.len != src.len
> 
> It also updates some part of the code to make them more generic. For
> instance the crypto request queue management supports both async block
> cipher and AEAD requests, which allows us to add support the the GCM mode.
> GMAC mode is not supported yet.
> 
> Current implementation was validated using the tcrypt
> module running modes:
> - 10: ecb(aes), cbc(aes), ctr(aes), rfc3686(ctr(aes))
> - 35: gcm(aes)
> - 2,6,11,12: sha1, sha2 family
> 
> tcrypt speed test was run with modes:
> - 211: rfc4106(gcm(aes)), gcm(aes)
> - 500: ecb(aes), cbc(aes), ctr(aes), rfc3686(ctr(aes))
> - 403 ~ 406: sha1, sha2 family
> 
> IxChariot multiple pairs throughput 24 hours test:
> - IPSec VPN
> - MACSec
> 
> Ryder Lee (8):
>   crypto: mediatek - move HW control data to transformation context
>   crypto: mediatek - fix incorrect data transfer result
>   crypto: mediatek - make crypto request queue management more generic
>   crypto: mediatek - rework crypto request completion
>   crypto: mediatek - regroup functions by usage
>   crypto: mediatek - fix typo and indentation
>   crypto: mediatek - add support to CTR mode
>   crypto: mediatek - add support to GCM mode

All applied.  Thanks.
-- 
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt