[RFC] crypto: Add support for the Geode AES engine

[RFC] crypto: Add support for the Geode AES engine

[Jordan Crouse]

[-- Attachment #1: Type: text/plain, Size: 780 bytes --]

Hey all -

I've had a poor excuse for a crypto driver for the Geode LX AES hardware
sitting in my Geode GIT tree (and -mm) for some time now.  I sent out the 
original version some time ago (June) and I got some very smart 
recommendations for changes, but other events transpired, and I didn't do 
a very good job of following up.

So now, in the interest of getting this out of everybody's hair and on
the path to mainstreaming, I'm sending out a newly updated version of the 
driver for comments.  This should be fully updated for all the new API
changes in the crypto GIT tree.  Once this is acceptable, I would like to get
this pulled into the crypto tree.

Thanks,
Jordan
-- 
Jordan Crouse
Senior Linux Engineer
Advanced Micro Devices, Inc.
<www.amd.com/embeddedprocessors>

[-- Attachment #2: geode-lx-crypto.patch --]
[-- Type: text/plain, Size: 13871 bytes --]

[PATCH] crypto:  Add support for the Geode LX hardware AES engine

This adds support for the AES engine built into the Geode LX processor.

Signed-off-by: Jordan Crouse <jordan.crouse@amd.com>

diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index adb5541..e816535 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -51,4 +51,17 @@ config CRYPTO_DEV_PADLOCK_SHA
 	  If unsure say M. The compiled module will be
 	  called padlock-sha.ko
 
+config CRYPTO_DEV_GEODE
+	tristate "Support for the Geode LX AES engine"
+	depends on CRYPTO && X86_32
+	select CRYPTO_ALGAPI
+	select CRYPTO_BLKCIPHER
+	default m
+	help
+	  Say 'Y' here to use the AMD Geode LX processor on-board AES
+	  engine for the CryptoAPI AES alogrithm.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called geode-aes.
+
 endmenu
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 4c3d0ec..6059cf8 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK) += padlock.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
+obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c
new file mode 100644
index 0000000..95860e5
--- /dev/null
+++ b/drivers/crypto/geode-aes.c
@@ -0,0 +1,477 @@
+/* CryptoAPI interface for the Geode LX hardware AES encryption module
+ * Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/crypto.h>
+#include <crypto/algapi.h>
+
+#include <asm/io.h>
+#include <asm/delay.h>
+
+#include "geode-aes.h"
+
+/* Register definitions */
+
+#define AES_CTRLA_REG  0x0000
+
+#define AES_CTRL_START     0x01
+#define AES_CTRL_DECRYPT   0x00
+#define AES_CTRL_ENCRYPT   0x02
+#define AES_CTRL_WRKEY     0x04
+#define AES_CTRL_DCA       0x08
+#define AES_CTRL_SCA       0x10
+#define AES_CTRL_CBC       0x20
+
+#define AES_INTR_REG  0x0008
+
+#define AES_INTRA_PENDING (1 << 16)
+#define AES_INTRB_PENDING (1 << 17)
+
+#define AES_INTR_PENDING  (AES_INTRA_PENDING | AES_INTRB_PENDING)
+#define AES_INTR_MASK     0x07
+
+#define AES_SOURCEA_REG   0x0010
+#define AES_DSTA_REG      0x0014
+#define AES_LENA_REG      0x0018
+#define AES_WRITEKEY0_REG 0x0030
+#define AES_WRITEIV0_REG  0x0040
+
+/*  A very large counter that is used to gracefully bail out of an
+ *  operation in case of trouble
+ */
+
+#define AES_OP_TIMEOUT    0x50000
+
+/* Useful macros */
+
+#define SET_KEY(key) _writefield(AES_WRITEKEY0_REG, key)
+#define SET_IV(iv)   _writefield(AES_WRITEIV0_REG, iv)
+#define GET_IV(iv)   _readfield(AES_WRITEIV0_REG, iv)
+#define AWRITE(val, reg) (iowrite32(val, _iobase + reg))
+#define AREAD(reg)  (ioread32(_iobase + reg))
+
+/* Static structures */
+
+static void __iomem * _iobase;
+
+/* Write a 128 bit field (either a writable key or IV) */
+static inline void
+_writefield(u32 offset, void *value)
+{
+	int i;
+	for(i = 0; i < 4; i++)
+		AWRITE(((u32 *) value)[i], offset + (i * 4));
+}
+
+/* Read a 128 bit field (either a writable key or IV) */
+static inline void
+_readfield(u32 offset, void *value)
+{
+	int i;
+	for(i = 0; i < 4; i++)
+		((u32 *) value)[i] = AREAD(offset + (i * 4));
+}
+
+static void
+_crypt(void *src, void *dst, int len, u32 flags)
+{
+	u32 status;
+	u32 counter = AES_OP_TIMEOUT;
+
+	AWRITE(virt_to_phys(src), AES_SOURCEA_REG);
+	AWRITE(virt_to_phys(dst), AES_DSTA_REG);
+	AWRITE(len,  AES_LENA_REG);
+
+	/* Start the operation */
+	AWRITE(AES_CTRL_START | flags, AES_CTRLA_REG);
+
+	do
+		status = AREAD(AES_INTR_REG);
+	while(!(status & AES_INTRA_PENDING) && --counter);
+
+	/* Clear the event */
+	AWRITE((status & 0xFF) | AES_INTRA_PENDING, AES_INTR_REG);
+}
+
+unsigned int
+geode_aes_crypt(struct geode_aes_op *op)
+{
+	u32 flags = 0;
+
+	if (op->len == 0 || op->src == op->dst)
+		return 0;
+
+	/* Make sure nobody bothers us */
+	preempt_disable();
+
+	if (op->mode == AES_MODE_CBC) {
+		flags |= AES_CTRL_CBC;
+		SET_IV(op->iv);
+	}
+
+	if (op->flags & AES_FLAGS_USRKEY) {
+		flags |= AES_CTRL_WRKEY;
+		SET_KEY(op->key);
+	}
+
+	if (op->flags & AES_FLAGS_COHERENT)
+		flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
+
+	if (op->dir == AES_DIR_ENCRYPT)
+		flags |= AES_CTRL_ENCRYPT;
+
+	_crypt(op->src, op->dst, op->len, flags);
+
+	if (op->mode == AES_MODE_CBC)
+		GET_IV(op->iv);
+
+	preempt_enable();
+	return op->len;
+}
+
+/* CRYPTO-API Functions */
+
+#define blk_ctx(tfm) ((struct geode_aes_op *) crypto_blkcipher_ctx(tfm))
+#define ctx(tfm) ((struct geode_aes_op *) crypto_tfm_ctx(tfm))
+
+static int
+geode_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int len)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if (len != AES_KEY_LENGTH) {
+		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+
+	memcpy(op->key, key, len);
+	return 0;
+}
+
+static void
+geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if ((out == NULL) || (in == NULL))
+		return;
+
+	op->src = (void *) in;
+	op->dst = (void *) out;
+	op->mode = AES_MODE_ECB;
+	op->flags = 0;
+	op->len = AES_MIN_BLOCK_SIZE;
+	op->dir = AES_DIR_ENCRYPT;
+
+	geode_aes_crypt(op);
+}
+
+
+static void
+geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if ((out == NULL) || (in == NULL))
+		return;
+
+	op->src = (void *) in;
+	op->dst = (void *) out;
+	op->mode = AES_MODE_ECB;
+	op->flags = 0;
+	op->len = AES_MIN_BLOCK_SIZE;
+	op->dir = AES_DIR_DECRYPT;
+
+	geode_aes_crypt(op);
+}
+
+
+static struct crypto_alg geode_alg = {
+	.cra_name               =       "aes",
+	.cra_driver_name	=       "geode-aes-128",
+	.cra_priority           =       300,
+	.cra_alignmask          =       15,
+	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_alg.cra_list),
+	.cra_u			=	{
+		.cipher = {
+			.cia_min_keysize	=  AES_KEY_LENGTH,
+			.cia_max_keysize	=  AES_KEY_LENGTH,
+			.cia_setkey		=  geode_setkey,
+			.cia_encrypt		=  geode_encrypt,
+			.cia_decrypt		=  geode_decrypt
+		}
+	}
+};
+
+static int
+geode_cbc_decrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_CBC;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_DECRYPT;
+
+		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
+
+		ret = geode_aes_crypt(op);
+
+		memcpy(walk.iv, op->iv, AES_IV_LENGTH);
+		nbytes -= ret;
+
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static int
+geode_cbc_encrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_CBC;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_ENCRYPT;
+
+		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static struct crypto_alg geode_cbc_alg = {
+	.cra_name		=	"cbc(aes)",
+	.cra_driver_name	=	"cbc-aes-geode-128",
+	.cra_priority		=	400,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_alignmask		=	15,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_cbc_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_KEY_LENGTH,
+			.max_keysize		=	AES_KEY_LENGTH,
+			.setkey			=	geode_setkey,
+			.encrypt		=	geode_cbc_encrypt,
+			.decrypt		=	geode_cbc_decrypt,
+		}
+	}
+};
+
+static int
+geode_ecb_decrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_ECB;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_DECRYPT;
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static int
+geode_ecb_encrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_ECB;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_ENCRYPT;
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		ret =  blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static struct crypto_alg geode_ecb_alg = {
+	.cra_name		=	"ecb(aes)",
+	.cra_driver_name	=	"ecb-aes-geode-128",
+	.cra_priority		=	400,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_alignmask		=	15,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_ecb_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_KEY_LENGTH,
+			.max_keysize		=	AES_KEY_LENGTH,
+			.setkey			=	geode_setkey,
+			.encrypt		=	geode_ecb_encrypt,
+			.decrypt		=	geode_ecb_decrypt,
+		}
+	}
+};
+
+static void
+geode_aes_remove(struct pci_dev *dev)
+{
+	crypto_unregister_alg(&geode_alg);
+	crypto_unregister_alg(&geode_ecb_alg);
+	crypto_unregister_alg(&geode_cbc_alg);
+
+	pci_iounmap(dev, _iobase);
+	_iobase = NULL;
+
+	pci_release_regions(dev);
+	pci_disable_device(dev);
+}
+
+
+static int
+geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	int ret;
+
+	if ((ret = pci_enable_device(dev)))
+		return ret;
+
+	if ((ret = pci_request_regions(dev, "geode-aes-128")))
+		goto eenable;
+
+	_iobase = pci_iomap(dev, 0, 0);
+
+	if (_iobase == NULL) {
+		ret = -ENOMEM;
+		goto erequest;
+	}
+
+	/* Clear any pending activity */
+	AWRITE(AES_INTR_PENDING | AES_INTR_MASK, AES_INTR_REG);
+
+	if ((ret = crypto_register_alg(&geode_alg)))
+		goto eiomap;
+
+	if ((ret = crypto_register_alg(&geode_ecb_alg)))
+		goto ealg;
+
+	if ((ret = crypto_register_alg(&geode_cbc_alg)))
+		goto eecb;
+
+	printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n");
+	return 0;
+
+ eecb:
+	crypto_unregister_alg(&geode_ecb_alg);
+
+ ealg:
+	crypto_unregister_alg(&geode_alg);
+
+ eiomap:
+	pci_iounmap(dev, _iobase);
+
+ erequest:
+	pci_release_regions(dev);
+
+ eenable:
+	pci_disable_device(dev);
+
+	printk(KERN_ERR "geode-aes:  GEODE AES initialization failed.\n");
+	return ret;
+}
+
+struct pci_device_id geode_aes_tbl[] = {
+	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, PCI_ANY_ID, PCI_ANY_ID} ,
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
+
+static struct pci_driver geode_aes_driver = {
+	name:      "Geode LX AES",
+	id_table:  geode_aes_tbl,
+	probe:     geode_aes_probe,
+	remove:    __devexit_p(geode_aes_remove)
+};
+
+static int __devinit
+geode_aes_init(void)
+{
+	return pci_module_init(&geode_aes_driver);
+}
+
+static void __devexit
+geode_aes_exit(void)
+{
+	pci_unregister_driver(&geode_aes_driver);
+}
+
+MODULE_AUTHOR("Advanced Micro Devices, Inc.");
+MODULE_DESCRIPTION("Geode LX Hardware AES driver");
+MODULE_LICENSE("GPL");
+
+module_init(geode_aes_init);
+module_exit(geode_aes_exit);
diff --git a/drivers/crypto/geode-aes.h b/drivers/crypto/geode-aes.h
new file mode 100644
index 0000000..97f9eee
--- /dev/null
+++ b/drivers/crypto/geode-aes.h
@@ -0,0 +1,42 @@
+/* Copyright (C) 2003-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef _GEODE_AES_H_
+#define _GEODE_AES_H_
+
+#define AES_KEY_LENGTH 16
+#define AES_IV_LENGTH  16
+
+#define AES_MIN_BLOCK_SIZE 16
+
+#define AES_MODE_ECB 0
+#define AES_MODE_CBC 1
+
+#define AES_DIR_DECRYPT 0
+#define AES_DIR_ENCRYPT 1
+
+#define AES_FLAGS_USRKEY   (1 << 0)
+#define AES_FLAGS_COHERENT (1 << 1)
+
+struct geode_aes_op {
+
+  void *src;
+  void *dst;
+
+  u32 mode;
+  u32 dir;
+  u32 flags;
+  int len;
+
+  u8 key[AES_KEY_LENGTH];
+  u8 iv[AES_IV_LENGTH];
+};
+
+unsigned int geode_aes_crypt(struct geode_aes_op *);
+
+#endif

Re: [RFC] crypto: Add support for the Geode AES engine

[Evgeniy Polyakov]

On Wed, Sep 27, 2006 at 01:31:47PM -0600, Jordan Crouse (jordan.crouse@amd.com) wrote:
> Hey all -
> 
> I've had a poor excuse for a crypto driver for the Geode LX AES hardware
> sitting in my Geode GIT tree (and -mm) for some time now.  I sent out the 
> original version some time ago (June) and I got some very smart 
> recommendations for changes, but other events transpired, and I didn't do 
> a very good job of following up.
> 
> So now, in the interest of getting this out of everybody's hair and on
> the path to mainstreaming, I'm sending out a newly updated version of the 
> driver for comments.  This should be fully updated for all the new API
> changes in the crypto GIT tree.  Once this is acceptable, I would like to get
> this pulled into the crypto tree.

As far as I can see, register access is not protected, how can your
driver handle the case when dm-crypt and IPsec simultaneously try to
encrypt/decrypt some data, it can happen even around 
preemt_disable/enable calls and actually crypto processing can happen 
in interrupt context too (although it is not the best thing to do).

You added timeout for the broken hardware condition, I think it is
better to return some error from _crypt() in that case, and, btw, that
name is not very good choice.

> Thanks,
> Jordan
> -- 
> Jordan Crouse
> Senior Linux Engineer
> Advanced Micro Devices, Inc.
> <www.amd.com/embeddedprocessors>


-- 
	Evgeniy Polyakov

Re: crypto: Add support for the Geode AES engine

[Jordan Crouse]

> As far as I can see, register access is not protected, how can your
> driver handle the case when dm-crypt and IPsec simultaneously try to
> encrypt/decrypt some data, it can happen even around 
> preemt_disable/enable calls and actually crypto processing can happen 
> in interrupt context too (although it is not the best thing to do).

I had the mutex in there, but I took it out based on our previous
conversations, which probably was a little rash.  If CRYPTO_TFM_REQ_MAY_SLEEP
is still a valid flag to check, I could use that along with a spin lock of
some sort.   I'll think about this a bit more.

> You added timeout for the broken hardware condition, I think it is
> better to return some error from _crypt() in that case, and, btw, that
> name is not very good choice.

I normally use _<blah> for static functions within my module - it helps
me remember which commands might be important to others and which are just
utility functions for my own use and abuse.  I'll change the name.

Thanks,
Jordan

-- 
Jordan Crouse
Senior Linux Engineer
Advanced Micro Devices, Inc.
<www.amd.com/embeddedprocessors>

crypto: Add support for the Geode AES engine (v2)

[Jordan Crouse]

[-- Attachment #1: Type: text/plain, Size: 1054 bytes --]

> As far as I can see, register access is not protected, how can your
> driver handle the case when dm-crypt and IPsec simultaneously try to
> encrypt/decrypt some data, it can happen even around 
> preemt_disable/enable calls and actually crypto processing can happen 
> in interrupt context too (although it is not the best thing to do).

I was sitting there trying to architect some grand scheme, and then it
occured to me that we should just turn off the interrupts all together
in the critical area.  Its not optimal for performance, but it
avoids lots of crazy if statements.

> You added timeout for the broken hardware condition, I think it is
> better to return some error from _crypt() in that case, and, btw, that
> name is not very good choice.

Fixed the name and I return the error.  I don't propagate it through the
whole chain, because quite frankly, if the crypto engine fails, then its
a good bet the processor is on fire. :)

Jordan
-- 
Jordan Crouse
Senior Linux Engineer
Advanced Micro Devices, Inc.
<www.amd.com/embeddedprocessors>

[-- Attachment #2: geode-lx-crypto.patch --]
[-- Type: text/plain, Size: 13759 bytes --]

diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index adb5541..e816535 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -51,4 +51,17 @@ config CRYPTO_DEV_PADLOCK_SHA
 	  If unsure say M. The compiled module will be
 	  called padlock-sha.ko
 
+config CRYPTO_DEV_GEODE
+	tristate "Support for the Geode LX AES engine"
+	depends on CRYPTO && X86_32
+	select CRYPTO_ALGAPI
+	select CRYPTO_BLKCIPHER
+	default m
+	help
+	  Say 'Y' here to use the AMD Geode LX processor on-board AES
+	  engine for the CryptoAPI AES alogrithm.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called geode-aes.
+
 endmenu
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 4c3d0ec..6059cf8 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK) += padlock.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
+obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c
new file mode 100644
index 0000000..73b33ba
--- /dev/null
+++ b/drivers/crypto/geode-aes.c
@@ -0,0 +1,481 @@
+/* CryptoAPI interface for the Geode LX hardware AES encryption module
+ * Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/crypto.h>
+#include <crypto/algapi.h>
+
+#include <asm/io.h>
+#include <asm/delay.h>
+
+#include "geode-aes.h"
+
+/* Register definitions */
+
+#define AES_CTRLA_REG  0x0000
+
+#define AES_CTRL_START     0x01
+#define AES_CTRL_DECRYPT   0x00
+#define AES_CTRL_ENCRYPT   0x02
+#define AES_CTRL_WRKEY     0x04
+#define AES_CTRL_DCA       0x08
+#define AES_CTRL_SCA       0x10
+#define AES_CTRL_CBC       0x20
+
+#define AES_INTR_REG  0x0008
+
+#define AES_INTRA_PENDING (1 << 16)
+#define AES_INTRB_PENDING (1 << 17)
+
+#define AES_INTR_PENDING  (AES_INTRA_PENDING | AES_INTRB_PENDING)
+#define AES_INTR_MASK     0x07
+
+#define AES_SOURCEA_REG   0x0010
+#define AES_DSTA_REG      0x0014
+#define AES_LENA_REG      0x0018
+#define AES_WRITEKEY0_REG 0x0030
+#define AES_WRITEIV0_REG  0x0040
+
+/*  A very large counter that is used to gracefully bail out of an
+ *  operation in case of trouble
+ */
+
+#define AES_OP_TIMEOUT    0x50000
+
+/* Useful macros */
+
+#define SET_KEY(key) _writefield(AES_WRITEKEY0_REG, key)
+#define SET_IV(iv)   _writefield(AES_WRITEIV0_REG, iv)
+#define GET_IV(iv)   _readfield(AES_WRITEIV0_REG, iv)
+#define AWRITE(val, reg) (iowrite32(val, _iobase + reg))
+#define AREAD(reg)  (ioread32(_iobase + reg))
+
+/* Static structures */
+
+static void __iomem * _iobase;
+
+/* Write a 128 bit field (either a writable key or IV) */
+static inline void
+_writefield(u32 offset, void *value)
+{
+	int i;
+	for(i = 0; i < 4; i++)
+		AWRITE(((u32 *) value)[i], offset + (i * 4));
+}
+
+/* Read a 128 bit field (either a writable key or IV) */
+static inline void
+_readfield(u32 offset, void *value)
+{
+	int i;
+	for(i = 0; i < 4; i++)
+		((u32 *) value)[i] = AREAD(offset + (i * 4));
+}
+
+static int
+do_crypt(void *src, void *dst, int len, u32 flags)
+{
+	u32 status;
+	u32 counter = AES_OP_TIMEOUT;
+
+	AWRITE(virt_to_phys(src), AES_SOURCEA_REG);
+	AWRITE(virt_to_phys(dst), AES_DSTA_REG);
+	AWRITE(len,  AES_LENA_REG);
+
+	/* Start the operation */ 
+	AWRITE(AES_CTRL_START | flags, AES_CTRLA_REG);
+
+	do 
+		status = AREAD(AES_INTR_REG);
+	while(!(status & AES_INTRA_PENDING) && --counter);
+
+	/* Clear the event */
+	AWRITE((status & 0xFF) | AES_INTRA_PENDING, AES_INTR_REG);
+	return counter ? 0 : 1;
+}
+
+unsigned int
+geode_aes_crypt(struct geode_aes_op *op)
+{
+	u32 flags = 0;
+	int iflags;
+
+	if (op->len == 0 || op->src == op->dst)
+		return 0;
+
+	if (op->flags & AES_FLAGS_COHERENT)
+		flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
+
+	if (op->dir == AES_DIR_ENCRYPT)
+		flags |= AES_CTRL_ENCRYPT;
+
+	/* Start the critical section */
+
+	spin_lock_irqsave(&lock, iflags);
+
+	if (op->mode == AES_MODE_CBC) {
+		flags |= AES_CTRL_CBC;
+		SET_IV(op->iv);
+	}
+
+	if (op->flags & AES_FLAGS_USRKEY) {
+		flags |= AES_CTRL_WRKEY;
+		SET_KEY(op->key);
+	}
+
+	do_crypt(op->src, op->dst, op->len, flags);
+
+	if (op->mode == AES_MODE_CBC)
+		GET_IV(op->iv);
+
+	spin_lock_irqrestore(&lock, iflags);
+
+	return op->len;
+}
+
+/* CRYPTO-API Functions */
+
+#define blk_ctx(tfm) ((struct geode_aes_op *) crypto_blkcipher_ctx(tfm))
+#define ctx(tfm) ((struct geode_aes_op *) crypto_tfm_ctx(tfm))
+
+static int
+geode_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int len)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if (len != AES_KEY_LENGTH) {
+		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+
+	memcpy(op->key, key, len);
+	return 0;
+}
+
+static void
+geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if ((out == NULL) || (in == NULL))
+		return;
+
+	op->src = (void *) in;
+	op->dst = (void *) out;
+	op->mode = AES_MODE_ECB;
+	op->flags = 0;
+	op->len = AES_MIN_BLOCK_SIZE;
+	op->dir = AES_DIR_ENCRYPT;
+
+	geode_aes_crypt(op);
+}
+
+
+static void
+geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if ((out == NULL) || (in == NULL))
+		return;
+
+	op->src = (void *) in;
+	op->dst = (void *) out;
+	op->mode = AES_MODE_ECB;
+	op->flags = 0;
+	op->len = AES_MIN_BLOCK_SIZE;
+	op->dir = AES_DIR_DECRYPT;
+
+	geode_aes_crypt(op);
+}
+
+
+static struct crypto_alg geode_alg = {
+	.cra_name               =       "aes",
+	.cra_driver_name	=       "geode-aes-128",
+	.cra_priority           =       300,
+	.cra_alignmask          =       15,
+	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_alg.cra_list),
+	.cra_u			=	{
+		.cipher = {
+			.cia_min_keysize	=  AES_KEY_LENGTH,
+			.cia_max_keysize	=  AES_KEY_LENGTH,
+			.cia_setkey		=  geode_setkey,
+			.cia_encrypt		=  geode_encrypt,
+			.cia_decrypt		=  geode_decrypt
+		}
+	}
+};
+
+static int
+geode_cbc_decrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_CBC;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_DECRYPT;
+
+		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
+
+		ret = geode_aes_crypt(op);
+
+		memcpy(walk.iv, op->iv, AES_IV_LENGTH);
+		nbytes -= ret;
+
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static int
+geode_cbc_encrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_CBC;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_ENCRYPT;
+
+		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static struct crypto_alg geode_cbc_alg = {
+	.cra_name		=	"cbc(aes)",
+	.cra_driver_name	=	"cbc-aes-geode-128",
+	.cra_priority		=	400,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_alignmask		=	15,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_cbc_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_KEY_LENGTH,
+			.max_keysize		=	AES_KEY_LENGTH,
+			.setkey			=	geode_setkey,
+			.encrypt		=	geode_cbc_encrypt,
+			.decrypt		=	geode_cbc_decrypt,
+		}
+	}
+};
+
+static int
+geode_ecb_decrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_ECB;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_DECRYPT;
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static int
+geode_ecb_encrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_ECB;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_ENCRYPT;
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		ret =  blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static struct crypto_alg geode_ecb_alg = {
+	.cra_name		=	"ecb(aes)",
+	.cra_driver_name	=	"ecb-aes-geode-128",
+	.cra_priority		=	400,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_alignmask		=	15,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_ecb_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_KEY_LENGTH,
+			.max_keysize		=	AES_KEY_LENGTH,
+			.setkey			=	geode_setkey,
+			.encrypt		=	geode_ecb_encrypt,
+			.decrypt		=	geode_ecb_decrypt,
+		}
+	}
+};
+
+static void
+geode_aes_remove(struct pci_dev *dev)
+{
+	crypto_unregister_alg(&geode_alg);
+	crypto_unregister_alg(&geode_ecb_alg);
+	crypto_unregister_alg(&geode_cbc_alg);
+
+	pci_iounmap(dev, _iobase);
+	_iobase = NULL;
+
+	pci_release_regions(dev);
+	pci_disable_device(dev);
+}
+
+
+static int
+geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	int ret;
+
+	if ((ret = pci_enable_device(dev)))
+		return ret;
+
+	if ((ret = pci_request_regions(dev, "geode-aes-128")))
+		goto eenable;
+
+	_iobase = pci_iomap(dev, 0, 0);
+
+	if (_iobase == NULL) {
+		ret = -ENOMEM;
+		goto erequest;
+	}
+
+	/* Clear any pending activity */
+	AWRITE(AES_INTR_PENDING | AES_INTR_MASK, AES_INTR_REG);
+
+	if ((ret = crypto_register_alg(&geode_alg)))
+		goto eiomap;
+
+	if ((ret = crypto_register_alg(&geode_ecb_alg)))
+		goto ealg;
+
+	if ((ret = crypto_register_alg(&geode_cbc_alg)))
+		goto eecb;
+
+	printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n");
+	return 0;
+
+ eecb:
+	crypto_unregister_alg(&geode_ecb_alg);
+
+ ealg:
+	crypto_unregister_alg(&geode_alg);
+
+ eiomap:
+	pci_iounmap(dev, _iobase);
+
+ erequest:
+	pci_release_regions(dev);
+
+ eenable:
+	pci_disable_device(dev);
+
+	printk(KERN_ERR "geode-aes:  GEODE AES initialization failed.\n");
+	return ret;
+}
+
+struct pci_device_id geode_aes_tbl[] = {
+	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, PCI_ANY_ID, PCI_ANY_ID} ,
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
+
+static struct pci_driver geode_aes_driver = {
+	name:      "Geode LX AES",
+	id_table:  geode_aes_tbl,
+	probe:     geode_aes_probe,
+	remove:    __devexit_p(geode_aes_remove)
+};
+
+static int __devinit
+geode_aes_init(void)
+{
+	return pci_module_init(&geode_aes_driver);
+}
+
+static void __devexit
+geode_aes_exit(void)
+{
+	pci_unregister_driver(&geode_aes_driver);
+}
+
+MODULE_AUTHOR("Advanced Micro Devices, Inc.");
+MODULE_DESCRIPTION("Geode LX Hardware AES driver");
+MODULE_LICENSE("GPL");
+
+module_init(geode_aes_init);
+module_exit(geode_aes_exit);
diff --git a/drivers/crypto/geode-aes.h b/drivers/crypto/geode-aes.h
new file mode 100644
index 0000000..97f9eee
--- /dev/null
+++ b/drivers/crypto/geode-aes.h
@@ -0,0 +1,42 @@
+/* Copyright (C) 2003-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef _GEODE_AES_H_
+#define _GEODE_AES_H_
+
+#define AES_KEY_LENGTH 16
+#define AES_IV_LENGTH  16
+
+#define AES_MIN_BLOCK_SIZE 16
+
+#define AES_MODE_ECB 0
+#define AES_MODE_CBC 1
+
+#define AES_DIR_DECRYPT 0
+#define AES_DIR_ENCRYPT 1
+
+#define AES_FLAGS_USRKEY   (1 << 0)
+#define AES_FLAGS_COHERENT (1 << 1)
+
+struct geode_aes_op {
+
+  void *src;
+  void *dst;
+
+  u32 mode;
+  u32 dir;
+  u32 flags;
+  int len;
+
+  u8 key[AES_KEY_LENGTH];
+  u8 iv[AES_IV_LENGTH];
+};
+
+unsigned int geode_aes_crypt(struct geode_aes_op *);
+
+#endif

Re: crypto: Add support for the Geode AES engine (v3)

[Jordan Crouse]

[-- Attachment #1: Type: text/plain, Size: 204 bytes --]

Sigh - forgot to do an 'stg refresh' before generating the diff.  This one
actually compiles.

Jordan
-- 
Jordan Crouse
Senior Linux Engineer
Advanced Micro Devices, Inc.
<www.amd.com/embeddedprocessors>

[-- Attachment #2: geode-lx-crypto.patch --]
[-- Type: text/plain, Size: 14023 bytes --]

[PATCH] crypto:  Add support for the Geode LX AES hardware

This adds support for the AES engine on the Geode LX processor.

Signed-off-by: Jordan Crouse <jordan.crouse@amd.com>

diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index adb5541..e816535 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -51,4 +51,17 @@ config CRYPTO_DEV_PADLOCK_SHA
 	  If unsure say M. The compiled module will be
 	  called padlock-sha.ko
 
+config CRYPTO_DEV_GEODE
+	tristate "Support for the Geode LX AES engine"
+	depends on CRYPTO && X86_32
+	select CRYPTO_ALGAPI
+	select CRYPTO_BLKCIPHER
+	default m
+	help
+	  Say 'Y' here to use the AMD Geode LX processor on-board AES
+	  engine for the CryptoAPI AES alogrithm.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called geode-aes.
+
 endmenu
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 4c3d0ec..6059cf8 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK) += padlock.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
+obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c
new file mode 100644
index 0000000..1fbc41d
--- /dev/null
+++ b/drivers/crypto/geode-aes.c
@@ -0,0 +1,486 @@
+/* CryptoAPI interface for the Geode LX hardware AES encryption module
+ * Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/crypto.h>
+#include <linux/spinlock.h>
+#include <crypto/algapi.h>
+
+#include <asm/io.h>
+#include <asm/delay.h>
+
+#include "geode-aes.h"
+
+/* Register definitions */
+
+#define AES_CTRLA_REG  0x0000
+
+#define AES_CTRL_START     0x01
+#define AES_CTRL_DECRYPT   0x00
+#define AES_CTRL_ENCRYPT   0x02
+#define AES_CTRL_WRKEY     0x04
+#define AES_CTRL_DCA       0x08
+#define AES_CTRL_SCA       0x10
+#define AES_CTRL_CBC       0x20
+
+#define AES_INTR_REG  0x0008
+
+#define AES_INTRA_PENDING (1 << 16)
+#define AES_INTRB_PENDING (1 << 17)
+
+#define AES_INTR_PENDING  (AES_INTRA_PENDING | AES_INTRB_PENDING)
+#define AES_INTR_MASK     0x07
+
+#define AES_SOURCEA_REG   0x0010
+#define AES_DSTA_REG      0x0014
+#define AES_LENA_REG      0x0018
+#define AES_WRITEKEY0_REG 0x0030
+#define AES_WRITEIV0_REG  0x0040
+
+/*  A very large counter that is used to gracefully bail out of an
+ *  operation in case of trouble
+ */
+
+#define AES_OP_TIMEOUT    0x50000
+
+/* Useful macros */
+
+#define SET_KEY(key) _writefield(AES_WRITEKEY0_REG, key)
+#define SET_IV(iv)   _writefield(AES_WRITEIV0_REG, iv)
+#define GET_IV(iv)   _readfield(AES_WRITEIV0_REG, iv)
+#define AWRITE(val, reg) (iowrite32(val, _iobase + reg))
+#define AREAD(reg)  (ioread32(_iobase + reg))
+
+/* Static structures */
+
+static void __iomem * _iobase;
+static spinlock_t lock;
+
+/* Write a 128 bit field (either a writable key or IV) */
+static inline void
+_writefield(u32 offset, void *value)
+{
+	int i;
+	for(i = 0; i < 4; i++)
+		AWRITE(((u32 *) value)[i], offset + (i * 4));
+}
+
+/* Read a 128 bit field (either a writable key or IV) */
+static inline void
+_readfield(u32 offset, void *value)
+{
+	int i;
+	for(i = 0; i < 4; i++)
+		((u32 *) value)[i] = AREAD(offset + (i * 4));
+}
+
+static int
+do_crypt(void *src, void *dst, int len, u32 flags)
+{
+	u32 status;
+	u32 counter = AES_OP_TIMEOUT;
+
+	AWRITE(virt_to_phys(src), AES_SOURCEA_REG);
+	AWRITE(virt_to_phys(dst), AES_DSTA_REG);
+	AWRITE(len,  AES_LENA_REG);
+
+	/* Start the operation */ 
+	AWRITE(AES_CTRL_START | flags, AES_CTRLA_REG);
+
+	do 
+		status = AREAD(AES_INTR_REG);
+	while(!(status & AES_INTRA_PENDING) && --counter);
+
+	/* Clear the event */
+	AWRITE((status & 0xFF) | AES_INTRA_PENDING, AES_INTR_REG);
+	return counter ? 0 : 1;
+}
+
+unsigned int
+geode_aes_crypt(struct geode_aes_op *op)
+{
+
+	u32 flags = 0;
+	int iflags;
+
+	if (op->len == 0 || op->src == op->dst)
+		return 0;
+
+	if (op->flags & AES_FLAGS_COHERENT)
+		flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
+
+	if (op->dir == AES_DIR_ENCRYPT)
+		flags |= AES_CTRL_ENCRYPT;
+
+	/* Start the critical section */
+
+	spin_lock_irqsave(&lock, iflags);
+
+	if (op->mode == AES_MODE_CBC) {
+		flags |= AES_CTRL_CBC;
+		SET_IV(op->iv);
+	}
+
+	if (op->flags & AES_FLAGS_USRKEY) {
+		flags |= AES_CTRL_WRKEY;
+		SET_KEY(op->key);
+	}
+
+	do_crypt(op->src, op->dst, op->len, flags);
+
+	if (op->mode == AES_MODE_CBC)
+		GET_IV(op->iv);
+
+	spin_unlock_irqrestore(&lock, iflags);
+
+	return op->len;
+}
+
+/* CRYPTO-API Functions */
+
+#define blk_ctx(tfm) ((struct geode_aes_op *) crypto_blkcipher_ctx(tfm))
+#define ctx(tfm) ((struct geode_aes_op *) crypto_tfm_ctx(tfm))
+
+static int
+geode_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int len)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if (len != AES_KEY_LENGTH) {
+		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+
+	memcpy(op->key, key, len);
+	return 0;
+}
+
+static void
+geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if ((out == NULL) || (in == NULL))
+		return;
+
+	op->src = (void *) in;
+	op->dst = (void *) out;
+	op->mode = AES_MODE_ECB;
+	op->flags = 0;
+	op->len = AES_MIN_BLOCK_SIZE;
+	op->dir = AES_DIR_ENCRYPT;
+
+	geode_aes_crypt(op);
+}
+
+
+static void
+geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct geode_aes_op *op = ctx(tfm);
+
+	if ((out == NULL) || (in == NULL))
+		return;
+
+	op->src = (void *) in;
+	op->dst = (void *) out;
+	op->mode = AES_MODE_ECB;
+	op->flags = 0;
+	op->len = AES_MIN_BLOCK_SIZE;
+	op->dir = AES_DIR_DECRYPT;
+
+	geode_aes_crypt(op);
+}
+
+
+static struct crypto_alg geode_alg = {
+	.cra_name               =       "aes",
+	.cra_driver_name	=       "geode-aes-128",
+	.cra_priority           =       300,
+	.cra_alignmask          =       15,
+	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_alg.cra_list),
+	.cra_u			=	{
+		.cipher = {
+			.cia_min_keysize	=  AES_KEY_LENGTH,
+			.cia_max_keysize	=  AES_KEY_LENGTH,
+			.cia_setkey		=  geode_setkey,
+			.cia_encrypt		=  geode_encrypt,
+			.cia_decrypt		=  geode_decrypt
+		}
+	}
+};
+
+static int
+geode_cbc_decrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_CBC;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_DECRYPT;
+
+		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
+
+		ret = geode_aes_crypt(op);
+
+		memcpy(walk.iv, op->iv, AES_IV_LENGTH);
+		nbytes -= ret;
+
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static int
+geode_cbc_encrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_CBC;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_ENCRYPT;
+
+		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static struct crypto_alg geode_cbc_alg = {
+	.cra_name		=	"cbc(aes)",
+	.cra_driver_name	=	"cbc-aes-geode-128",
+	.cra_priority		=	400,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_alignmask		=	15,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_cbc_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_KEY_LENGTH,
+			.max_keysize		=	AES_KEY_LENGTH,
+			.setkey			=	geode_setkey,
+			.encrypt		=	geode_cbc_encrypt,
+			.decrypt		=	geode_cbc_decrypt,
+		}
+	}
+};
+
+static int
+geode_ecb_decrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_ECB;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_DECRYPT;
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static int
+geode_ecb_encrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = blk_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = (void *) walk.src.virt.addr,
+		op->dst = (void *) walk.dst.virt.addr;
+		op->mode = AES_MODE_ECB;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_ENCRYPT;
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		ret =  blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static struct crypto_alg geode_ecb_alg = {
+	.cra_name		=	"ecb(aes)",
+	.cra_driver_name	=	"ecb-aes-geode-128",
+	.cra_priority		=	400,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_alignmask		=	15,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_ecb_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_KEY_LENGTH,
+			.max_keysize		=	AES_KEY_LENGTH,
+			.setkey			=	geode_setkey,
+			.encrypt		=	geode_ecb_encrypt,
+			.decrypt		=	geode_ecb_decrypt,
+		}
+	}
+};
+
+static void
+geode_aes_remove(struct pci_dev *dev)
+{
+	crypto_unregister_alg(&geode_alg);
+	crypto_unregister_alg(&geode_ecb_alg);
+	crypto_unregister_alg(&geode_cbc_alg);
+
+	pci_iounmap(dev, _iobase);
+	_iobase = NULL;
+
+	pci_release_regions(dev);
+	pci_disable_device(dev);
+}
+
+
+static int
+geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	int ret;
+
+	if ((ret = pci_enable_device(dev)))
+		return ret;
+
+	if ((ret = pci_request_regions(dev, "geode-aes-128")))
+		goto eenable;
+
+	_iobase = pci_iomap(dev, 0, 0);
+
+	if (_iobase == NULL) {
+		ret = -ENOMEM;
+		goto erequest;
+	}
+
+	spin_lock_init(&lock);
+
+	/* Clear any pending activity */
+	AWRITE(AES_INTR_PENDING | AES_INTR_MASK, AES_INTR_REG);
+
+	if ((ret = crypto_register_alg(&geode_alg)))
+		goto eiomap;
+
+	if ((ret = crypto_register_alg(&geode_ecb_alg)))
+		goto ealg;
+
+	if ((ret = crypto_register_alg(&geode_cbc_alg)))
+		goto eecb;
+
+	printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n");
+	return 0;
+
+ eecb:
+	crypto_unregister_alg(&geode_ecb_alg);
+
+ ealg:
+	crypto_unregister_alg(&geode_alg);
+
+ eiomap:
+	pci_iounmap(dev, _iobase);
+
+ erequest:
+	pci_release_regions(dev);
+
+ eenable:
+	pci_disable_device(dev);
+
+	printk(KERN_ERR "geode-aes:  GEODE AES initialization failed.\n");
+	return ret;
+}
+
+struct pci_device_id geode_aes_tbl[] = {
+	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, PCI_ANY_ID, PCI_ANY_ID} ,
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
+
+static struct pci_driver geode_aes_driver = {
+	name:      "Geode LX AES",
+	id_table:  geode_aes_tbl,
+	probe:     geode_aes_probe,
+	remove:    __devexit_p(geode_aes_remove)
+};
+
+static int __devinit
+geode_aes_init(void)
+{
+	return pci_module_init(&geode_aes_driver);
+}
+
+static void __devexit
+geode_aes_exit(void)
+{
+	pci_unregister_driver(&geode_aes_driver);
+}
+
+MODULE_AUTHOR("Advanced Micro Devices, Inc.");
+MODULE_DESCRIPTION("Geode LX Hardware AES driver");
+MODULE_LICENSE("GPL");
+
+module_init(geode_aes_init);
+module_exit(geode_aes_exit);
diff --git a/drivers/crypto/geode-aes.h b/drivers/crypto/geode-aes.h
new file mode 100644
index 0000000..97f9eee
--- /dev/null
+++ b/drivers/crypto/geode-aes.h
@@ -0,0 +1,42 @@
+/* Copyright (C) 2003-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef _GEODE_AES_H_
+#define _GEODE_AES_H_
+
+#define AES_KEY_LENGTH 16
+#define AES_IV_LENGTH  16
+
+#define AES_MIN_BLOCK_SIZE 16
+
+#define AES_MODE_ECB 0
+#define AES_MODE_CBC 1
+
+#define AES_DIR_DECRYPT 0
+#define AES_DIR_ENCRYPT 1
+
+#define AES_FLAGS_USRKEY   (1 << 0)
+#define AES_FLAGS_COHERENT (1 << 1)
+
+struct geode_aes_op {
+
+  void *src;
+  void *dst;
+
+  u32 mode;
+  u32 dir;
+  u32 flags;
+  int len;
+
+  u8 key[AES_KEY_LENGTH];
+  u8 iv[AES_IV_LENGTH];
+};
+
+unsigned int geode_aes_crypt(struct geode_aes_op *);
+
+#endif

Re: crypto: Add support for the Geode AES engine (v2)

[Andrew Morton]

On Thu, 28 Sep 2006 15:47:50 -0600
"Jordan Crouse" <jordan.crouse@amd.com> wrote:

> > As far as I can see, register access is not protected, how can your
> > driver handle the case when dm-crypt and IPsec simultaneously try to
> > encrypt/decrypt some data, it can happen even around 
> > preemt_disable/enable calls and actually crypto processing can happen 
> > in interrupt context too (although it is not the best thing to do).
> 
> I was sitting there trying to architect some grand scheme, and then it
> occured to me that we should just turn off the interrupts all together
> in the critical area.  Its not optimal for performance, but it
> avoids lots of crazy if statements.
> 
> > You added timeout for the broken hardware condition, I think it is
> > better to return some error from _crypt() in that case, and, btw, that
> > name is not very good choice.
> 
> Fixed the name and I return the error.  I don't propagate it through the
> whole chain, because quite frankly, if the crypto engine fails, then its
> a good bet the processor is on fire. :)
> 
> ...
>



> diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
> index adb5541..e816535 100644
> --- a/drivers/crypto/Kconfig
> +++ b/drivers/crypto/Kconfig
> @@ -51,4 +51,17 @@ config CRYPTO_DEV_PADLOCK_SHA
>  	  If unsure say M. The compiled module will be
>  	  called padlock-sha.ko
>  
> +config CRYPTO_DEV_GEODE
> +	tristate "Support for the Geode LX AES engine"
> +	depends on CRYPTO && X86_32

Does it depend on GEODE in some fashion too?

> +	select CRYPTO_ALGAPI
> +	select CRYPTO_BLKCIPHER
> +	default m
> +	help
> +	  Say 'Y' here to use the AMD Geode LX processor on-board AES
> +	  engine for the CryptoAPI AES alogrithm.
> +
> +	  To compile this driver as a module, choose M here: the module
> +	  will be called geode-aes.
> +
>  endmenu
> 
> ...
>
> +
> +/* Useful macros */
> +
> +#define SET_KEY(key) _writefield(AES_WRITEKEY0_REG, key)
> +#define SET_IV(iv)   _writefield(AES_WRITEIV0_REG, iv)
> +#define GET_IV(iv)   _readfield(AES_WRITEIV0_REG, iv)
> +#define AWRITE(val, reg) (iowrite32(val, _iobase + reg))
> +#define AREAD(reg)  (ioread32(_iobase + reg))

lower-case static-inlines are nicer...

Or just remove them and open-code it all.

> +/* Static structures */
> +
> +static void __iomem * _iobase;
> +
> +/* Write a 128 bit field (either a writable key or IV) */
> +static inline void
> +_writefield(u32 offset, void *value)
> +{
> +	int i;
> +	for(i = 0; i < 4; i++)
> +		AWRITE(((u32 *) value)[i], offset + (i * 4));
> +}
>
> +/* Read a 128 bit field (either a writable key or IV) */
> +static inline void
> +_readfield(u32 offset, void *value)
> +{
> +	int i;
> +	for(i = 0; i < 4; i++)
> +		((u32 *) value)[i] = AREAD(offset + (i * 4));
> +}
> +
> ...
>
> +unsigned int
> +geode_aes_crypt(struct geode_aes_op *op)
> +{
> +	u32 flags = 0;
> +	int iflags;
> +
> +	if (op->len == 0 || op->src == op->dst)
> +		return 0;
> +
> +	if (op->flags & AES_FLAGS_COHERENT)
> +		flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
> +
> +	if (op->dir == AES_DIR_ENCRYPT)
> +		flags |= AES_CTRL_ENCRYPT;
> +
> +	/* Start the critical section */
> +
> +	spin_lock_irqsave(&lock, iflags);
> +
> +	if (op->mode == AES_MODE_CBC) {
> +		flags |= AES_CTRL_CBC;
> +		SET_IV(op->iv);
> +	}
> +
> +	if (op->flags & AES_FLAGS_USRKEY) {
> +		flags |= AES_CTRL_WRKEY;
> +		SET_KEY(op->key);
> +	}
> +
> +	do_crypt(op->src, op->dst, op->len, flags);
> +
> +	if (op->mode == AES_MODE_CBC)
> +		GET_IV(op->iv);
> +
> +	spin_lock_irqrestore(&lock, iflags);
> +
> +	return op->len;
> +}

Running do_crypt() under spin_lock_irqsave() seems a bit sad from a latency
POV.

> +/* CRYPTO-API Functions */
> +
> +#define blk_ctx(tfm) ((struct geode_aes_op *) crypto_blkcipher_ctx(tfm))
> +#define ctx(tfm) ((struct geode_aes_op *) crypto_tfm_ctx(tfm))

Both crypto_blkcipher_ctx() and crypto_tfm_ctx() return a nice void*. 
There's no need for the typecast and hence there's really no need for these
macros at all.  Simply open-code the crypto_blkcipher_ctx() and
crypto_tfm_ctx() calls.

> +static int
> +geode_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int len)
> +{
> +	struct geode_aes_op *op = ctx(tfm);
> +
> +	if (len != AES_KEY_LENGTH) {
> +		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
> +		return -EINVAL;
> +	}
> +
> +	memcpy(op->key, key, len);
> +	return 0;
> +}
> +
> +static void
> +geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
> +{
> +	struct geode_aes_op *op = ctx(tfm);
> +
> +	if ((out == NULL) || (in == NULL))
> +		return;
> +
> +	op->src = (void *) in;
> +	op->dst = (void *) out;

Unneeded casts

> +	op->mode = AES_MODE_ECB;
> +	op->flags = 0;
> +	op->len = AES_MIN_BLOCK_SIZE;
> +	op->dir = AES_DIR_ENCRYPT;
> +
> +	geode_aes_crypt(op);
> +}
> +
> +
> +static void
> +geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
> +{
> +	struct geode_aes_op *op = ctx(tfm);
> +
> +	if ((out == NULL) || (in == NULL))
> +		return;
> +
> +	op->src = (void *) in;
> +	op->dst = (void *) out;

Unneeded casts

> +static int
> +geode_cbc_decrypt(struct blkcipher_desc *desc,
> +		  struct scatterlist *dst, struct scatterlist *src,
> +		  unsigned int nbytes)
> +{
> +	struct geode_aes_op *op = blk_ctx(desc->tfm);
> +	struct blkcipher_walk walk;
> +	int err, ret;
> +
> +	blkcipher_walk_init(&walk, dst, src, nbytes);
> +	err = blkcipher_walk_virt(desc, &walk);
> +
> +	while((nbytes = walk.nbytes)) {
> +		op->src = (void *) walk.src.virt.addr,
> +		op->dst = (void *) walk.dst.virt.addr;

Unneeded cast

> +		op->mode = AES_MODE_CBC;
> +		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
> +		op->dir = AES_DIR_DECRYPT;
> +
> +		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
> +
> +		ret = geode_aes_crypt(op);
> +
> +		memcpy(walk.iv, op->iv, AES_IV_LENGTH);
> +		nbytes -= ret;
> +
> +		err = blkcipher_walk_done(desc, &walk, nbytes);
> +	}
> +
> +	return err;
> +}
>
> ...
>

> +		op->src = (void *) walk.src.virt.addr,
> +		op->dst = (void *) walk.dst.virt.addr;

Unneeded cast.  Please review whole patchset.

> +struct pci_device_id geode_aes_tbl[] = {
> +	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, PCI_ANY_ID, PCI_ANY_ID} ,
> +	{ 0, }
> +};

static?

> +static struct pci_driver geode_aes_driver = {
> +	name:      "Geode LX AES",
> +	id_table:  geode_aes_tbl,
> +	probe:     geode_aes_probe,
> +	remove:    __devexit_p(geode_aes_remove)
> +};

Please use `.name = value'

> +static int __devinit
> +geode_aes_init(void)
> +{
> +	return pci_module_init(&geode_aes_driver);
> +}
> +
> +static void __devexit
> +geode_aes_exit(void)
> +{
> +	pci_unregister_driver(&geode_aes_driver);
> +}

These should be __init and __exit, I think?

> +struct geode_aes_op {
> +
> +  void *src;
> +  void *dst;
> +
> +  u32 mode;
> +  u32 dir;
> +  u32 flags;
> +  int len;
> +
> +  u8 key[AES_KEY_LENGTH];
> +  u8 iv[AES_IV_LENGTH];
> +};

tabs, please.

Re: crypto: Add support for the Geode AES engine (v4)

[Jordan Crouse]

[-- Attachment #1: Type: text/plain, Size: 1384 bytes --]

> Does it depend on GEODE in some fashion too?

Its only available on the Geode platform, but there is no code it needs 
that is only exposed by CONFIG_MGEODE_LX elsewhere in the kernel.  This is
just appears as a regular, run of the mill PCI device.

> > +	spin_lock_irqsave(&lock, iflags);
> > +
> > +	if (op->mode == AES_MODE_CBC) {
> > +		flags |= AES_CTRL_CBC;
> > +		SET_IV(op->iv);
> > +	}
> > +
> > +	if (op->flags & AES_FLAGS_USRKEY) {
> > +		flags |= AES_CTRL_WRKEY;
> > +		SET_KEY(op->key);
> > +	}
> > +
> > +	do_crypt(op->src, op->dst, op->len, flags);
> > +
> > +	if (op->mode == AES_MODE_CBC)
> > +		GET_IV(op->iv);
> > +
> > +	spin_lock_irqrestore(&lock, iflags);
> > +
> > +	return op->len;
> > +}
> 
> Running do_crypt() under spin_lock_irqsave() seems a bit sad from a latency
> POV.

Indeed - but the problem is that the key and/or the IV need to stay
unmodified until the operation completes.  The datasheet calls this 
out specifically:

"This register should not be changed during an AES encryption or decryption 
operation."

So not only do we need to protect ourselves while changing the fields, we
also need to make sure they remain untouched until the whole operation is
completed. 

Changes made and new version follows. Thanks for the comments.

Jordan
-- 
Jordan Crouse
Senior Linux Engineer
Advanced Micro Devices, Inc.
<www.amd.com/embeddedprocessors>

[-- Attachment #2: geode-lx-crypto.patch --]
[-- Type: text/plain, Size: 13700 bytes --]

[PATCH] crypto:  Add support for the Geode LX AES hardware

Add a driver to support the AES hardware on the Geode LX processor.

Signed-off-by: Jordan Crouse <jordan.crouse@amd.com>
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index adb5541..e816535 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -51,4 +51,17 @@ config CRYPTO_DEV_PADLOCK_SHA
 	  If unsure say M. The compiled module will be
 	  called padlock-sha.ko
 
+config CRYPTO_DEV_GEODE
+	tristate "Support for the Geode LX AES engine"
+	depends on CRYPTO && X86_32
+	select CRYPTO_ALGAPI
+	select CRYPTO_BLKCIPHER
+	default m
+	help
+	  Say 'Y' here to use the AMD Geode LX processor on-board AES
+	  engine for the CryptoAPI AES alogrithm.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called geode-aes.
+
 endmenu
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 4c3d0ec..6059cf8 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK) += padlock.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
+obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c
new file mode 100644
index 0000000..da2d35d
--- /dev/null
+++ b/drivers/crypto/geode-aes.c
@@ -0,0 +1,474 @@
+ /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/crypto.h>
+#include <linux/spinlock.h>
+#include <crypto/algapi.h>
+
+#include <asm/io.h>
+#include <asm/delay.h>
+
+#include "geode-aes.h"
+
+/* Register definitions */
+
+#define AES_CTRLA_REG  0x0000
+
+#define AES_CTRL_START     0x01
+#define AES_CTRL_DECRYPT   0x00
+#define AES_CTRL_ENCRYPT   0x02
+#define AES_CTRL_WRKEY     0x04
+#define AES_CTRL_DCA       0x08
+#define AES_CTRL_SCA       0x10
+#define AES_CTRL_CBC       0x20
+
+#define AES_INTR_REG  0x0008
+
+#define AES_INTRA_PENDING (1 << 16)
+#define AES_INTRB_PENDING (1 << 17)
+
+#define AES_INTR_PENDING  (AES_INTRA_PENDING | AES_INTRB_PENDING)
+#define AES_INTR_MASK     0x07
+
+#define AES_SOURCEA_REG   0x0010
+#define AES_DSTA_REG      0x0014
+#define AES_LENA_REG      0x0018
+#define AES_WRITEKEY0_REG 0x0030
+#define AES_WRITEIV0_REG  0x0040
+
+/*  A very large counter that is used to gracefully bail out of an
+ *  operation in case of trouble
+ */
+
+#define AES_OP_TIMEOUT    0x50000
+
+/* Static structures */
+
+static void __iomem * _iobase;
+static spinlock_t lock;
+
+/* Write a 128 bit field (either a writable key or IV) */
+static inline void
+_writefield(u32 offset, void *value)
+{
+	int i;
+	for(i = 0; i < 4; i++)
+		iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
+}
+
+/* Read a 128 bit field (either a writable key or IV) */
+static inline void
+_readfield(u32 offset, void *value)
+{
+	int i;
+	for(i = 0; i < 4; i++)
+		((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
+}
+
+static int
+do_crypt(void *src, void *dst, int len, u32 flags)
+{
+	u32 status;
+	u32 counter = AES_OP_TIMEOUT;
+
+	iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG);
+	iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG);
+	iowrite32(len,  _iobase + AES_LENA_REG);
+
+	/* Start the operation */
+	iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG);
+
+	do
+		status = ioread32(_iobase + AES_INTR_REG);
+	while(!(status & AES_INTRA_PENDING) && --counter);
+
+	/* Clear the event */
+	iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG);
+	return counter ? 0 : 1;
+}
+
+unsigned int
+geode_aes_crypt(struct geode_aes_op *op)
+{
+
+	u32 flags = 0;
+	int iflags;
+
+	if (op->len == 0 || op->src == op->dst)
+		return 0;
+
+	if (op->flags & AES_FLAGS_COHERENT)
+		flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
+
+	if (op->dir == AES_DIR_ENCRYPT)
+		flags |= AES_CTRL_ENCRYPT;
+
+	/* Start the critical section */
+
+	spin_lock_irqsave(&lock, iflags);
+
+	if (op->mode == AES_MODE_CBC) {
+		flags |= AES_CTRL_CBC;
+		_writefield(AES_WRITEIV0_REG, op->iv);
+	}
+
+	if (op->flags & AES_FLAGS_USRKEY) {
+		flags |= AES_CTRL_WRKEY;
+		_writefield(AES_WRITEKEY0_REG, op->key);
+	}
+
+	do_crypt(op->src, op->dst, op->len, flags);
+
+	if (op->mode == AES_MODE_CBC)
+		_readfield(AES_WRITEIV0_REG, op->iv);
+
+	spin_unlock_irqrestore(&lock, iflags);
+
+	return op->len;
+}
+
+/* CRYPTO-API Functions */
+
+static int
+geode_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int len)
+{
+	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+	if (len != AES_KEY_LENGTH) {
+		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+
+	memcpy(op->key, key, len);
+	return 0;
+}
+
+static void
+geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+	if ((out == NULL) || (in == NULL))
+		return;
+
+	op->src = (void *) in;
+	op->dst = (void *) out;
+	op->mode = AES_MODE_ECB;
+	op->flags = 0;
+	op->len = AES_MIN_BLOCK_SIZE;
+	op->dir = AES_DIR_ENCRYPT;
+
+	geode_aes_crypt(op);
+}
+
+
+static void
+geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+	if ((out == NULL) || (in == NULL))
+		return;
+
+	op->src = (void *) in;
+	op->dst = (void *) out;
+	op->mode = AES_MODE_ECB;
+	op->flags = 0;
+	op->len = AES_MIN_BLOCK_SIZE;
+	op->dir = AES_DIR_DECRYPT;
+
+	geode_aes_crypt(op);
+}
+
+
+static struct crypto_alg geode_alg = {
+	.cra_name               =       "aes",
+	.cra_driver_name	=       "geode-aes-128",
+	.cra_priority           =       300,
+	.cra_alignmask          =       15,
+	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_alg.cra_list),
+	.cra_u			=	{
+		.cipher = {
+			.cia_min_keysize	=  AES_KEY_LENGTH,
+			.cia_max_keysize	=  AES_KEY_LENGTH,
+			.cia_setkey		=  geode_setkey,
+			.cia_encrypt		=  geode_encrypt,
+			.cia_decrypt		=  geode_decrypt
+		}
+	}
+};
+
+static int
+geode_cbc_decrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = walk.src.virt.addr,
+		op->dst = walk.dst.virt.addr;
+		op->mode = AES_MODE_CBC;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_DECRYPT;
+
+		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
+
+		ret = geode_aes_crypt(op);
+
+		memcpy(walk.iv, op->iv, AES_IV_LENGTH);
+		nbytes -= ret;
+
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static int
+geode_cbc_encrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = walk.src.virt.addr,
+		op->dst = walk.dst.virt.addr;
+		op->mode = AES_MODE_CBC;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_ENCRYPT;
+
+		memcpy(op->iv, walk.iv, AES_IV_LENGTH);
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static struct crypto_alg geode_cbc_alg = {
+	.cra_name		=	"cbc(aes)",
+	.cra_driver_name	=	"cbc-aes-geode-128",
+	.cra_priority		=	400,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_alignmask		=	15,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_cbc_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_KEY_LENGTH,
+			.max_keysize		=	AES_KEY_LENGTH,
+			.setkey			=	geode_setkey,
+			.encrypt		=	geode_cbc_encrypt,
+			.decrypt		=	geode_cbc_decrypt,
+		}
+	}
+};
+
+static int
+geode_ecb_decrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = walk.src.virt.addr,
+		op->dst = walk.dst.virt.addr;
+		op->mode = AES_MODE_ECB;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_DECRYPT;
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static int
+geode_ecb_encrypt(struct blkcipher_desc *desc,
+		  struct scatterlist *dst, struct scatterlist *src,
+		  unsigned int nbytes)
+{
+	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err, ret;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while((nbytes = walk.nbytes)) {
+		op->src = walk.src.virt.addr,
+		op->dst = walk.dst.virt.addr;
+		op->mode = AES_MODE_ECB;
+		op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+		op->dir = AES_DIR_ENCRYPT;
+
+		ret = geode_aes_crypt(op);
+		nbytes -= ret;
+		ret =  blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static struct crypto_alg geode_ecb_alg = {
+	.cra_name		=	"ecb(aes)",
+	.cra_driver_name	=	"ecb-aes-geode-128",
+	.cra_priority		=	400,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct geode_aes_op),
+	.cra_alignmask		=	15,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(geode_ecb_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_KEY_LENGTH,
+			.max_keysize		=	AES_KEY_LENGTH,
+			.setkey			=	geode_setkey,
+			.encrypt		=	geode_ecb_encrypt,
+			.decrypt		=	geode_ecb_decrypt,
+		}
+	}
+};
+
+static void
+geode_aes_remove(struct pci_dev *dev)
+{
+	crypto_unregister_alg(&geode_alg);
+	crypto_unregister_alg(&geode_ecb_alg);
+	crypto_unregister_alg(&geode_cbc_alg);
+
+	pci_iounmap(dev, _iobase);
+	_iobase = NULL;
+
+	pci_release_regions(dev);
+	pci_disable_device(dev);
+}
+
+
+static int
+geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	int ret;
+
+	if ((ret = pci_enable_device(dev)))
+		return ret;
+
+	if ((ret = pci_request_regions(dev, "geode-aes-128")))
+		goto eenable;
+
+	_iobase = pci_iomap(dev, 0, 0);
+
+	if (_iobase == NULL) {
+		ret = -ENOMEM;
+		goto erequest;
+	}
+
+	spin_lock_init(&lock);
+
+	/* Clear any pending activity */
+	iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG);
+
+	if ((ret = crypto_register_alg(&geode_alg)))
+		goto eiomap;
+
+	if ((ret = crypto_register_alg(&geode_ecb_alg)))
+		goto ealg;
+
+	if ((ret = crypto_register_alg(&geode_cbc_alg)))
+		goto eecb;
+
+	printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n");
+	return 0;
+
+ eecb:
+	crypto_unregister_alg(&geode_ecb_alg);
+
+ ealg:
+	crypto_unregister_alg(&geode_alg);
+
+ eiomap:
+	pci_iounmap(dev, _iobase);
+
+ erequest:
+	pci_release_regions(dev);
+
+ eenable:
+	pci_disable_device(dev);
+
+	printk(KERN_ERR "geode-aes:  GEODE AES initialization failed.\n");
+	return ret;
+}
+
+static struct pci_device_id geode_aes_tbl[] = {
+	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, PCI_ANY_ID, PCI_ANY_ID} ,
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
+
+static struct pci_driver geode_aes_driver = {
+	.name = "Geode LX AES",
+	.id_table = geode_aes_tbl,
+	.probe = geode_aes_probe,
+	.remove = __devexit_p(geode_aes_remove)
+};
+
+static int __init
+geode_aes_init(void)
+{
+	return pci_module_init(&geode_aes_driver);
+}
+
+static void __exit
+geode_aes_exit(void)
+{
+	pci_unregister_driver(&geode_aes_driver);
+}
+
+MODULE_AUTHOR("Advanced Micro Devices, Inc.");
+MODULE_DESCRIPTION("Geode LX Hardware AES driver");
+MODULE_LICENSE("GPL");
+
+module_init(geode_aes_init);
+module_exit(geode_aes_exit);
diff --git a/drivers/crypto/geode-aes.h b/drivers/crypto/geode-aes.h
new file mode 100644
index 0000000..3e3a571
--- /dev/null
+++ b/drivers/crypto/geode-aes.h
@@ -0,0 +1,42 @@
+/* Copyright (C) 2003-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef _GEODE_AES_H_
+#define _GEODE_AES_H_
+
+#define AES_KEY_LENGTH 16
+#define AES_IV_LENGTH  16
+
+#define AES_MIN_BLOCK_SIZE 16
+
+#define AES_MODE_ECB 0
+#define AES_MODE_CBC 1
+
+#define AES_DIR_DECRYPT 0
+#define AES_DIR_ENCRYPT 1
+
+#define AES_FLAGS_USRKEY   (1 << 0)
+#define AES_FLAGS_COHERENT (1 << 1)
+
+struct geode_aes_op {
+
+	void *src;
+	void *dst;
+
+	u32 mode;
+	u32 dir;
+	u32 flags;
+	int len;
+
+	u8 key[AES_KEY_LENGTH];
+	u8 iv[AES_IV_LENGTH];
+};
+
+unsigned int geode_aes_crypt(struct geode_aes_op *);
+
+#endif

Re: crypto: Add support for the Geode AES engine (v4)

[Andrew Morton]

On Thu, 28 Sep 2006 17:17:39 -0600
"Jordan Crouse" <jordan.crouse@amd.com> wrote:

> Add a driver to support the AES hardware on the Geode LX processor.

So... what are you planning on doing with this?  Ask Linus to pull
that git tree, or what?

Re: crypto: Add support for the Geode AES engine (v4)

[Jordan Crouse]

On 28/09/06 16:54 -0700, Andrew Morton wrote:
> On Thu, 28 Sep 2006 17:17:39 -0600
> "Jordan Crouse" <jordan.crouse@amd.com> wrote:
> 
> > Add a driver to support the AES hardware on the Geode LX processor.
> 
> So... what are you planning on doing with this?  Ask Linus to pull
> that git tree, or what?

Good question - I guess I'm not sure what the correct protocol is here.  
I should have done this long ago, so I was ready for the 2.6.19 window. 
I also should have gotten it to the crypto guys first, before putting it
in the Geode tree. I guess I never really expected it to be in 2.6.19,
I just wanted to get this cleaned up and shipped off.  It was mocking me
sitting there slowly bit rotting.

As it stands, I think the right move is to get it into the crypto tree to
patiently await 2.6.20.   

Jordan

-- 
Jordan Crouse
Senior Linux Engineer
Advanced Micro Devices, Inc.
<www.amd.com/embeddedprocessors>

Re: crypto: Add support for the Geode AES engine

[Evgeniy Polyakov]

On Thu, Sep 28, 2006 at 10:31:15AM -0600, Jordan Crouse (jordan.crouse@amd.com) wrote:
> > As far as I can see, register access is not protected, how can your
> > driver handle the case when dm-crypt and IPsec simultaneously try to
> > encrypt/decrypt some data, it can happen even around 
> > preemt_disable/enable calls and actually crypto processing can happen 
> > in interrupt context too (although it is not the best thing to do).
> 
> I had the mutex in there, but I took it out based on our previous
> conversations, which probably was a little rash.  If CRYPTO_TFM_REQ_MAY_SLEEP
> is still a valid flag to check, I could use that along with a spin lock of
> some sort.   I'll think about this a bit more.

If it is set, then you can safely sleep there, but if it is not, it does
not always mean that sleeping is forbidden.

-- 
	Evgeniy Polyakov

Re: crypto: Add support for the Geode AES engine (v4)

[Herbert Xu]

On Thu, Sep 28, 2006 at 11:17:39PM +0000, Jordan Crouse wrote:
> 
> Changes made and new version follows. Thanks for the comments.

Patch applied.  Thanks everybody.

Cheers,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt