From: Herbert Xu <herbert@gondor.apana.org.au>
To: Linus Torvalds <torvalds@linux-foundation.org>,
"David S. Miller" <davem@davemloft.net>,
"Jason A. Donenfeld" <Jason@zx2c4.com>,
Eric Biggers <ebiggers@kernel.org>,
Ard Biesheuvel <ard.biesheuvel@linaro.org>,
Linux Crypto Mailing List <linux-crypto@vger.kernel.org>,
linux-fscrypt@vger.kernel.org,
linux-arm-kernel@lists.infradead.org,
LKML <linux-kernel@vger.kernel.org>,
Paul Crowley <paulcrowley@google.com>,
Greg Kaiser <gkaiser@google.com>,
Samuel Neves <samuel.c.p.neves@gmail.com>,
Tomer Ashur <tomer.ashur@esat.kuleuven.be>,
Martin Willi <martin@strongswan.org>
Subject: [PATCH 7/17] crypto: poly1305 - Export core functions without crypto API
Date: Fri, 22 Mar 2019 14:29:44 +0800 [thread overview]
Message-ID: <E1h7DgW-0001I5-Il@gondobar> (raw)
In-Reply-To: 20190322062740.nrwfx2rvmt7lzotj@gondor.apana.org.au
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 9661 bytes --]
This patch exports the raw poly1305 functions, including the generic
as well as the x86 accelerated version. This allows them to be
used without going through the crypto API.
In order to ensure that zinc can link to the requisite functions,
this function removes the failure mode from the x86 accelerated
glue code so that the modules will always load, even if the hardware
is not available. In that case, the crypto API functions would not
be registered.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
arch/x86/crypto/poly1305_glue.c | 86 ++++++++++++++++++++--------------------
crypto/poly1305_generic.c | 16 +------
include/crypto/poly1305.h | 42 ++++++++++++++++++-
3 files changed, 87 insertions(+), 57 deletions(-)
diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c
index 88cc01506c84..7a70b0f42c5c 100644
--- a/arch/x86/crypto/poly1305_glue.c
+++ b/arch/x86/crypto/poly1305_glue.c
@@ -20,15 +20,7 @@
struct poly1305_simd_desc_ctx {
struct poly1305_desc_ctx base;
- /* derived key u set? */
- bool uset;
-#ifdef CONFIG_AS_AVX2
- /* derived keys r^3, r^4 set? */
- bool wset;
-#endif
- /* derived Poly1305 key r^2 */
- u32 u[5];
- /* ... silently appended r^3 and r^4 when using AVX2 */
+ struct poly1305_simd_xtra x;
};
asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src,
@@ -41,14 +33,11 @@ asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r,
static bool poly1305_use_avx2;
#endif
-static int poly1305_simd_init(struct shash_desc *desc)
+static int poly1305_simd_init2(struct shash_desc *desc)
{
struct poly1305_simd_desc_ctx *sctx = shash_desc_ctx(desc);
- sctx->uset = false;
-#ifdef CONFIG_AS_AVX2
- sctx->wset = false;
-#endif
+ poly1305_simd_init(&sctx->x);
return crypto_poly1305_init(desc);
}
@@ -64,60 +53,70 @@ static void poly1305_simd_mult(u32 *a, const u32 *b)
poly1305_block_sse2(a, m, b, 1);
}
-static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
- const u8 *src, unsigned int srclen)
+unsigned int poly1305_simd_blocks(struct poly1305_state *state,
+ const struct poly1305_key *key,
+ struct poly1305_simd_xtra *sctx,
+ const void *src, unsigned int srclen)
{
- struct poly1305_simd_desc_ctx *sctx;
- unsigned int blocks, datalen;
-
- BUILD_BUG_ON(offsetof(struct poly1305_simd_desc_ctx, base));
- sctx = container_of(dctx, struct poly1305_simd_desc_ctx, base);
-
- if (unlikely(!dctx->sset)) {
- datalen = crypto_poly1305_setdesckey(dctx, src, srclen);
- src += srclen - datalen;
- srclen = datalen;
- }
+ unsigned int blocks;
#ifdef CONFIG_AS_AVX2
if (poly1305_use_avx2 && srclen >= POLY1305_BLOCK_SIZE * 4) {
if (unlikely(!sctx->wset)) {
if (!sctx->uset) {
- memcpy(sctx->u, dctx->r.r, sizeof(sctx->u));
- poly1305_simd_mult(sctx->u, dctx->r.r);
+ memcpy(sctx->u, key->r, sizeof(sctx->u));
+ poly1305_simd_mult(sctx->u, key->r);
sctx->uset = true;
}
memcpy(sctx->u + 5, sctx->u, sizeof(sctx->u));
- poly1305_simd_mult(sctx->u + 5, dctx->r.r);
+ poly1305_simd_mult(sctx->u + 5, key->r);
memcpy(sctx->u + 10, sctx->u + 5, sizeof(sctx->u));
- poly1305_simd_mult(sctx->u + 10, dctx->r.r);
+ poly1305_simd_mult(sctx->u + 10, key->r);
sctx->wset = true;
}
blocks = srclen / (POLY1305_BLOCK_SIZE * 4);
- poly1305_4block_avx2(dctx->h.h, src, dctx->r.r, blocks,
- sctx->u);
+ poly1305_4block_avx2(state->h, src, key->r, blocks, sctx->u);
src += POLY1305_BLOCK_SIZE * 4 * blocks;
srclen -= POLY1305_BLOCK_SIZE * 4 * blocks;
}
#endif
if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) {
if (unlikely(!sctx->uset)) {
- memcpy(sctx->u, dctx->r.r, sizeof(sctx->u));
- poly1305_simd_mult(sctx->u, dctx->r.r);
+ memcpy(sctx->u, key->r, sizeof(sctx->u));
+ poly1305_simd_mult(sctx->u, key->r);
sctx->uset = true;
}
blocks = srclen / (POLY1305_BLOCK_SIZE * 2);
- poly1305_2block_sse2(dctx->h.h, src, dctx->r.r, blocks,
+ poly1305_2block_sse2(state->h, src, key->r, blocks,
sctx->u);
src += POLY1305_BLOCK_SIZE * 2 * blocks;
srclen -= POLY1305_BLOCK_SIZE * 2 * blocks;
}
if (srclen >= POLY1305_BLOCK_SIZE) {
- poly1305_block_sse2(dctx->h.h, src, dctx->r.r, 1);
+ poly1305_block_sse2(state->h, src, key->r, 1);
srclen -= POLY1305_BLOCK_SIZE;
}
return srclen;
}
+EXPORT_SYMBOL_GPL(poly1305_simd_blocks);
+
+static unsigned int poly1305_simd_blocks2(struct poly1305_desc_ctx *dctx,
+ const u8 *src, unsigned int srclen)
+{
+ struct poly1305_simd_desc_ctx *sctx;
+ unsigned int datalen;
+
+ BUILD_BUG_ON(offsetof(struct poly1305_simd_desc_ctx, base));
+ sctx = container_of(dctx, struct poly1305_simd_desc_ctx, base);
+
+ if (unlikely(!dctx->sset)) {
+ datalen = crypto_poly1305_setdesckey(dctx, src, srclen);
+ src += srclen - datalen;
+ srclen = datalen;
+ }
+
+ return poly1305_simd_blocks(&dctx->h, &dctx->r, &sctx->x, src, srclen);
+}
static int poly1305_simd_update(struct shash_desc *desc,
const u8 *src, unsigned int srclen)
@@ -139,14 +138,14 @@ static int poly1305_simd_update(struct shash_desc *desc,
dctx->buflen += bytes;
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
- poly1305_simd_blocks(dctx, dctx->buf,
- POLY1305_BLOCK_SIZE);
+ poly1305_simd_blocks2(dctx, dctx->buf,
+ POLY1305_BLOCK_SIZE);
dctx->buflen = 0;
}
}
if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
- bytes = poly1305_simd_blocks(dctx, src, srclen);
+ bytes = poly1305_simd_blocks2(dctx, src, srclen);
src += srclen - bytes;
srclen = bytes;
}
@@ -163,7 +162,7 @@ static int poly1305_simd_update(struct shash_desc *desc,
static struct shash_alg alg = {
.digestsize = POLY1305_DIGEST_SIZE,
- .init = poly1305_simd_init,
+ .init = poly1305_simd_init2,
.update = poly1305_simd_update,
.final = crypto_poly1305_final,
.descsize = sizeof(struct poly1305_simd_desc_ctx),
@@ -179,7 +178,7 @@ static struct shash_alg alg = {
static int __init poly1305_simd_mod_init(void)
{
if (!boot_cpu_has(X86_FEATURE_XMM2))
- return -ENODEV;
+ return 0;
#ifdef CONFIG_AS_AVX2
poly1305_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) &&
@@ -194,6 +193,9 @@ static int __init poly1305_simd_mod_init(void)
static void __exit poly1305_simd_mod_exit(void)
{
+ if (!boot_cpu_has(X86_FEATURE_XMM2))
+ return;
+
crypto_unregister_shash(&alg);
}
diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c
index 2a06874204e8..2fc44fce66ca 100644
--- a/crypto/poly1305_generic.c
+++ b/crypto/poly1305_generic.c
@@ -87,10 +87,9 @@ unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx,
}
EXPORT_SYMBOL_GPL(crypto_poly1305_setdesckey);
-static void poly1305_blocks_internal(struct poly1305_state *state,
- const struct poly1305_key *key,
- const void *src, unsigned int nblocks,
- u32 hibit)
+void poly1305_blocks_internal(struct poly1305_state *state,
+ const struct poly1305_key *key, const void *src,
+ unsigned int nblocks, u32 hibit)
{
u32 r0, r1, r2, r3, r4;
u32 s1, s2, s3, s4;
@@ -154,14 +153,7 @@ static void poly1305_blocks_internal(struct poly1305_state *state,
state->h[3] = h3;
state->h[4] = h4;
}
-
-void poly1305_core_blocks(struct poly1305_state *state,
- const struct poly1305_key *key,
- const void *src, unsigned int nblocks)
-{
- poly1305_blocks_internal(state, key, src, nblocks, 1 << 24);
-}
-EXPORT_SYMBOL_GPL(poly1305_core_blocks);
+EXPORT_SYMBOL_GPL(poly1305_blocks_internal);
static void poly1305_blocks(struct poly1305_desc_ctx *dctx,
const u8 *src, unsigned int srclen, u32 hibit)
diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h
index 34317ed2071e..4cfb8a46785b 100644
--- a/include/crypto/poly1305.h
+++ b/include/crypto/poly1305.h
@@ -38,6 +38,20 @@ struct poly1305_desc_ctx {
bool sset;
};
+struct poly1305_simd_xtra {
+ /* derived key u set? */
+ bool uset;
+#ifdef CONFIG_AS_AVX2
+ /* derived keys r^3, r^4 set? */
+ bool wset;
+#endif
+ /* derived Poly1305 key r^2 */
+ u32 u[5];
+ /* ... silently appended r^3 and r^4 when using AVX2 */
+};
+
+struct shash_desc;
+
/*
* Poly1305 core functions. These implement the ε-almost-∆-universal hash
* function underlying the Poly1305 MAC, i.e. they don't add an encrypted nonce
@@ -48,9 +62,18 @@ static inline void poly1305_core_init(struct poly1305_state *state)
{
memset(state->h, 0, sizeof(state->h));
}
-void poly1305_core_blocks(struct poly1305_state *state,
- const struct poly1305_key *key,
- const void *src, unsigned int nblocks);
+
+void poly1305_blocks_internal(struct poly1305_state *state,
+ const struct poly1305_key *key, const void *src,
+ unsigned int nblocks, u32 hibit);
+
+static inline void poly1305_core_blocks(struct poly1305_state *state,
+ const struct poly1305_key *key,
+ const void *src, unsigned int nblocks)
+{
+ poly1305_blocks_internal(state, key, src, nblocks, 1 << 24);
+}
+
void poly1305_core_emit(const struct poly1305_state *state, void *dst);
/* Crypto API helper functions for the Poly1305 MAC */
@@ -61,4 +84,17 @@ int crypto_poly1305_update(struct shash_desc *desc,
const u8 *src, unsigned int srclen);
int crypto_poly1305_final(struct shash_desc *desc, u8 *dst);
+static inline void poly1305_simd_init(struct poly1305_simd_xtra *x)
+{
+ x->uset = false;
+#ifdef CONFIG_AS_AVX2
+ x->wset = false;
+#endif
+}
+
+unsigned int poly1305_simd_blocks(struct poly1305_state *state,
+ const struct poly1305_key *key,
+ struct poly1305_simd_xtra *sctx,
+ const void *src, unsigned int srclen);
+
#endif
next prev parent reply other threads:[~2019-03-22 6:31 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-03-22 6:27 [PATCH 0/17] Add zinc using existing algorithm implementations Herbert Xu
2019-03-22 6:29 ` [PATCH 1/17] asm: simd context helper API Herbert Xu
2019-03-22 6:29 ` [PATCH 2/17] crypto: chacha20 - Export chacha20 functions without crypto API Herbert Xu
2019-03-22 6:29 ` [PATCH 3/17] zinc: introduce minimal cryptography library Herbert Xu
2019-03-22 6:29 ` [PATCH 4/17] zinc: Add generic C implementation of chacha20 and self-test Herbert Xu
2019-03-22 6:29 ` [PATCH 5/17] zinc: Add x86 accelerated ChaCha20 Herbert Xu
2019-03-22 6:29 ` [PATCH 6/17] zinc: Add arm accelerated chacha20 Herbert Xu
2019-03-22 6:29 ` Herbert Xu [this message]
2019-03-22 6:29 ` [PATCH 8/17] zinc: Add generic C implementation of poly1305 and self-test Herbert Xu
2019-03-22 6:29 ` [PATCH 9/17] zinc: Add x86 accelerated poly1305 Herbert Xu
2019-03-22 6:29 ` [PATCH 10/17] zinc: ChaCha20Poly1305 construction and selftest Herbert Xu
2019-03-22 6:29 ` [PATCH 11/17] zinc: BLAKE2s generic C implementation " Herbert Xu
2019-03-22 6:29 ` [PATCH 12/17] zinc: BLAKE2s x86_64 implementation Herbert Xu
2019-03-22 6:29 ` [PATCH 13/17] zinc: Curve25519 generic C implementations and selftest Herbert Xu
2019-03-22 6:29 ` [PATCH 14/17] zinc: Curve25519 x86_64 implementation Herbert Xu
2019-03-22 6:29 ` [PATCH 15/17] zinc: import Bernstein and Schwabe's Curve25519 ARM implementation Herbert Xu
2019-03-22 6:29 ` [PATCH 16/17] zinc: " Herbert Xu
2019-03-22 6:29 ` [PATCH 17/17] security/keys: rewrite big_key crypto to use Zinc Herbert Xu
2019-03-22 6:41 ` [PATCH 0/17] Add zinc using existing algorithm implementations Jason A. Donenfeld
2019-03-22 7:56 ` Ard Biesheuvel
2019-03-22 8:10 ` Jason A. Donenfeld
2019-03-22 17:48 ` Linus Torvalds
2019-03-25 9:10 ` Pascal Van Leeuwen
2019-03-26 9:46 ` Riku Voipio
2019-04-09 16:14 ` Pascal Van Leeuwen
2019-03-25 10:43 ` Ard Biesheuvel
2019-03-25 10:45 ` Jason A. Donenfeld
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=E1h7DgW-0001I5-Il@gondobar \
--to=herbert@gondor.apana.org.au \
--cc=Jason@zx2c4.com \
--cc=ard.biesheuvel@linaro.org \
--cc=davem@davemloft.net \
--cc=ebiggers@kernel.org \
--cc=gkaiser@google.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-fscrypt@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=martin@strongswan.org \
--cc=paulcrowley@google.com \
--cc=samuel.c.p.neves@gmail.com \
--cc=tomer.ashur@esat.kuleuven.be \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).