From: Dmitry Safonov <dima@arista.com>
To: linux-kernel@vger.kernel.org
Cc: Dmitry Safonov <0x7f454c46@gmail.com>,
Dmitry Safonov <dima@arista.com>,
Andy Lutomirski <luto@amacapital.net>,
Ard Biesheuvel <ardb@kernel.org>,
David Ahern <dsahern@kernel.org>,
"David S. Miller" <davem@davemloft.net>,
Eric Biggers <ebiggers@kernel.org>,
Eric Dumazet <edumazet@google.com>,
Francesco Ruggeri <fruggeri@arista.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
Hideaki YOSHIFUJI <yoshfuji@linux-ipv6.org>,
Jakub Kicinski <kuba@kernel.org>,
Leonard Crestez <cdleonard@gmail.com>,
Paolo Abeni <pabeni@redhat.com>,
Salam Noureddine <noureddine@arista.com>,
netdev@vger.kernel.org, linux-crypto@vger.kernel.org
Subject: [PATCH 1/6] crypto: Introduce crypto_pool
Date: Tue, 26 Jul 2022 21:15:55 +0100 [thread overview]
Message-ID: <20220726201600.1715505-2-dima@arista.com> (raw)
In-Reply-To: <20220726201600.1715505-1-dima@arista.com>
Introduce a per-CPU pool of async crypto requests that can be used
in bh-disabled contexts (designed with net RX/TX softirqs as users in
mind). Allocation can sleep and is a slow-path.
Initial implementation has only ahash as a backend and a fix-sized array
of possible algorithms used in parallel.
Signed-off-by: Dmitry Safonov <dima@arista.com>
---
crypto/Kconfig | 6 +
crypto/Makefile | 1 +
crypto/crypto_pool.c | 287 ++++++++++++++++++++++++++++++++++++++++++
include/crypto/pool.h | 34 +++++
4 files changed, 328 insertions(+)
create mode 100644 crypto/crypto_pool.c
create mode 100644 include/crypto/pool.h
diff --git a/crypto/Kconfig b/crypto/Kconfig
index bb427a835e44..aeddaa3dcc77 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -2128,6 +2128,12 @@ config CRYPTO_STATS
config CRYPTO_HASH_INFO
bool
+config CRYPTO_POOL
+ tristate "Per-CPU crypto pool"
+ default n
+ help
+ Per-CPU pool of crypto requests ready for usage in atomic contexts.
+
source "drivers/crypto/Kconfig"
source "crypto/asymmetric_keys/Kconfig"
source "certs/Kconfig"
diff --git a/crypto/Makefile b/crypto/Makefile
index 167c004dbf4f..6d1d9801b76b 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRYPTO_ACOMP2) += crypto_acompress.o
cryptomgr-y := algboss.o testmgr.o
obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o
+obj-$(CONFIG_CRYPTO_POOL) += crypto_pool.o
obj-$(CONFIG_CRYPTO_USER) += crypto_user.o
crypto_user-y := crypto_user_base.o
crypto_user-$(CONFIG_CRYPTO_STATS) += crypto_user_stat.o
diff --git a/crypto/crypto_pool.c b/crypto/crypto_pool.c
new file mode 100644
index 000000000000..c668c02499b7
--- /dev/null
+++ b/crypto/crypto_pool.c
@@ -0,0 +1,287 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <crypto/pool.h>
+#include <linux/kref.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/percpu.h>
+#include <linux/workqueue.h>
+
+static unsigned long scratch_size = DEFAULT_CRYPTO_POOL_SCRATCH_SZ;
+static DEFINE_PER_CPU(void *, crypto_pool_scratch);
+
+struct crypto_pool_entry {
+ struct ahash_request * __percpu *req;
+ const char *alg;
+ struct kref kref;
+ bool needs_key;
+};
+
+#define CPOOL_SIZE (PAGE_SIZE/sizeof(struct crypto_pool_entry))
+static struct crypto_pool_entry cpool[CPOOL_SIZE];
+static int last_allocated;
+static DEFINE_MUTEX(cpool_mutex);
+
+static int crypto_pool_scratch_alloc(void)
+{
+ int cpu;
+
+ lockdep_assert_held(&cpool_mutex);
+
+ for_each_possible_cpu(cpu) {
+ void *scratch = per_cpu(crypto_pool_scratch, cpu);
+
+ if (scratch)
+ continue;
+
+ scratch = kmalloc_node(scratch_size, GFP_KERNEL,
+ cpu_to_node(cpu));
+ if (!scratch)
+ return -ENOMEM;
+ per_cpu(crypto_pool_scratch, cpu) = scratch;
+ }
+ return 0;
+}
+
+static void crypto_pool_scratch_free(void)
+{
+ int cpu;
+
+ lockdep_assert_held(&cpool_mutex);
+
+ for_each_possible_cpu(cpu) {
+ void *scratch = per_cpu(crypto_pool_scratch, cpu);
+
+ if (!scratch)
+ continue;
+ per_cpu(crypto_pool_scratch, cpu) = NULL;
+ kfree(scratch);
+ }
+}
+
+static int __cpool_alloc_ahash(struct crypto_pool_entry *e, const char *alg)
+{
+ struct crypto_ahash *hash;
+ int cpu, ret = -ENOMEM;
+
+ e->alg = kstrdup(alg, GFP_KERNEL);
+ if (!e->alg)
+ return -ENOMEM;
+
+ e->req = alloc_percpu(struct ahash_request *);
+ if (!e->req)
+ goto out_free_alg;
+
+ hash = crypto_alloc_ahash(alg, 0, CRYPTO_ALG_ASYNC);
+ if (IS_ERR(hash)) {
+ ret = PTR_ERR(hash);
+ goto out_free_req;
+ }
+
+ /* If hash has .setkey(), allocate ahash per-cpu, not only request */
+ e->needs_key = crypto_ahash_get_flags(hash) & CRYPTO_TFM_NEED_KEY;
+
+ for_each_possible_cpu(cpu) {
+ struct ahash_request *req;
+
+ if (!hash)
+ hash = crypto_alloc_ahash(alg, 0, CRYPTO_ALG_ASYNC);
+ if (IS_ERR(hash))
+ goto out_free;
+
+ req = ahash_request_alloc(hash, GFP_KERNEL);
+ if (!req)
+ goto out_free;
+
+ ahash_request_set_callback(req, 0, NULL, NULL);
+
+ *per_cpu_ptr(e->req, cpu) = req;
+
+ if (e->needs_key)
+ hash = NULL;
+ }
+ kref_init(&e->kref);
+ return 0;
+
+out_free:
+ if (!IS_ERR_OR_NULL(hash) && e->needs_key)
+ crypto_free_ahash(hash);
+
+ for_each_possible_cpu(cpu) {
+ if (*per_cpu_ptr(e->req, cpu) == NULL)
+ break;
+ hash = crypto_ahash_reqtfm(*per_cpu_ptr(e->req, cpu));
+ ahash_request_free(*per_cpu_ptr(e->req, cpu));
+ if (e->needs_key) {
+ crypto_free_ahash(hash);
+ hash = NULL;
+ }
+ }
+
+ if (hash)
+ crypto_free_ahash(hash);
+out_free_req:
+ free_percpu(e->req);
+out_free_alg:
+ kfree(e->alg);
+ e->alg = NULL;
+ return ret;
+}
+
+/**
+ * crypto_pool_alloc_ahash - allocates pool for ahash requests
+ * @alg: name of async hash algorithm
+ */
+int crypto_pool_alloc_ahash(const char *alg)
+{
+ unsigned int i;
+ int err;
+
+ /* slow-path */
+ mutex_lock(&cpool_mutex);
+ err = crypto_pool_scratch_alloc();
+ if (err)
+ goto out;
+
+ for (i = 0; i < last_allocated; i++) {
+ if (cpool[i].alg && !strcmp(cpool[i].alg, alg)) {
+ kref_get(&cpool[i].kref);
+ goto out;
+ }
+ }
+
+ for (i = 0; i < last_allocated; i++) {
+ if (!cpool[i].alg)
+ break;
+ }
+ if (i >= CPOOL_SIZE) {
+ err = -ENOSPC;
+ goto out;
+ }
+
+ err = __cpool_alloc_ahash(&cpool[i], alg);
+ if (!err && last_allocated <= i)
+ last_allocated++;
+out:
+ mutex_unlock(&cpool_mutex);
+ return err ?: (int)i;
+}
+EXPORT_SYMBOL_GPL(crypto_pool_alloc_ahash);
+
+static void __cpool_free_entry(struct crypto_pool_entry *e)
+{
+ struct crypto_ahash *hash = NULL;
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ if (*per_cpu_ptr(e->req, cpu) == NULL)
+ continue;
+
+ hash = crypto_ahash_reqtfm(*per_cpu_ptr(e->req, cpu));
+ ahash_request_free(*per_cpu_ptr(e->req, cpu));
+ if (e->needs_key) {
+ crypto_free_ahash(hash);
+ hash = NULL;
+ }
+ }
+ if (hash)
+ crypto_free_ahash(hash);
+ free_percpu(e->req);
+ kfree(e->alg);
+ memset(e, 0, sizeof(*e));
+}
+
+static void cpool_cleanup_work_cb(struct work_struct *work)
+{
+ unsigned int i;
+ bool free_scratch = true;
+
+ mutex_lock(&cpool_mutex);
+ for (i = 0; i < last_allocated; i++) {
+ if (kref_read(&cpool[i].kref) > 0) {
+ free_scratch = false;
+ continue;
+ }
+ if (!cpool[i].alg)
+ continue;
+ __cpool_free_entry(&cpool[i]);
+ }
+ if (free_scratch)
+ crypto_pool_scratch_free();
+ mutex_unlock(&cpool_mutex);
+}
+
+static DECLARE_WORK(cpool_cleanup_work, cpool_cleanup_work_cb);
+static void cpool_schedule_cleanup(struct kref *kref)
+{
+ schedule_work(&cpool_cleanup_work);
+}
+
+/**
+ * crypto_pool_release - decreases number of users for a pool. If it was
+ * the last user of the pool, releases any memory that was consumed.
+ * @id: crypto_pool that was previously allocated by crypto_pool_alloc_ahash()
+ */
+void crypto_pool_release(unsigned int id)
+{
+ if (WARN_ON_ONCE(id > last_allocated || !cpool[id].alg))
+ return;
+
+ /* slow-path */
+ kref_put(&cpool[id].kref, cpool_schedule_cleanup);
+}
+EXPORT_SYMBOL_GPL(crypto_pool_release);
+
+/**
+ * crypto_pool_add - increases number of users (refcounter) for a pool
+ * @id: crypto_pool that was previously allocated by crypto_pool_alloc_ahash()
+ */
+void crypto_pool_add(unsigned int id)
+{
+ if (WARN_ON_ONCE(id > last_allocated || !cpool[id].alg))
+ return;
+ kref_get(&cpool[id].kref);
+}
+EXPORT_SYMBOL_GPL(crypto_pool_add);
+
+/**
+ * crypto_pool_get - disable bh and start using crypto_pool
+ * @id: crypto_pool that was previously allocated by crypto_pool_alloc_ahash()
+ * @c: returned crypto_pool for usage (uninitialized on failure)
+ */
+int crypto_pool_get(unsigned int id, struct crypto_pool *c)
+{
+ struct crypto_pool_ahash *ret = (struct crypto_pool_ahash *)c;
+
+ local_bh_disable();
+ if (WARN_ON_ONCE(id > last_allocated || !cpool[id].alg)) {
+ local_bh_enable();
+ return -EINVAL;
+ }
+ ret->req = *this_cpu_ptr(cpool[id].req);
+ ret->base.scratch = this_cpu_read(crypto_pool_scratch);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_pool_get);
+
+/**
+ * crypto_pool_algo - return algorithm of crypto_pool
+ * @id: crypto_pool that was previously allocated by crypto_pool_alloc_ahash()
+ * @buf: buffer to return name of algorithm
+ * @buf_len: size of @buf
+ */
+size_t crypto_pool_algo(unsigned int id, char *buf, size_t buf_len)
+{
+ size_t ret = 0;
+
+ /* slow-path */
+ mutex_lock(&cpool_mutex);
+ if (cpool[id].alg)
+ ret = strscpy(buf, cpool[id].alg, buf_len);
+ mutex_unlock(&cpool_mutex);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_pool_algo);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Per-CPU pool of crypto requests");
diff --git a/include/crypto/pool.h b/include/crypto/pool.h
new file mode 100644
index 000000000000..2c61aa45faff
--- /dev/null
+++ b/include/crypto/pool.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _CRYPTO_POOL_H
+#define _CRYPTO_POOL_H
+
+#include <crypto/hash.h>
+
+#define DEFAULT_CRYPTO_POOL_SCRATCH_SZ 128
+
+struct crypto_pool {
+ void *scratch;
+};
+
+/*
+ * struct crypto_pool_ahash - per-CPU pool of ahash_requests
+ * @base: common members that can be used by any async crypto ops
+ * @req: pre-allocated ahash request
+ */
+struct crypto_pool_ahash {
+ struct crypto_pool base;
+ struct ahash_request *req;
+};
+
+int crypto_pool_alloc_ahash(const char *alg);
+void crypto_pool_add(unsigned int id);
+void crypto_pool_release(unsigned int id);
+
+int crypto_pool_get(unsigned int id, struct crypto_pool *c);
+static inline void crypto_pool_put(void)
+{
+ local_bh_enable();
+}
+size_t crypto_pool_algo(unsigned int id, char *buf, size_t buf_len);
+
+#endif /* _CRYPTO_POOL_H */
--
2.36.1
next prev parent reply other threads:[~2022-07-26 20:16 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-07-26 20:15 [PATCH 0/6] net/crypto: Introduce crypto_pool Dmitry Safonov
2022-07-26 20:15 ` Dmitry Safonov [this message]
2022-07-26 20:15 ` [PATCH 2/6] crypto_pool: Add crypto_pool_reserve_scratch() Dmitry Safonov
2022-07-26 20:15 ` [PATCH 3/6] net/tcp: Separate tcp_md5sig_info allocation into tcp_md5sig_info_add() Dmitry Safonov
2022-07-26 20:15 ` [PATCH 4/6] net/tcp: Disable TCP-MD5 static key on tcp_md5sig_info destruction Dmitry Safonov
2022-08-14 15:49 ` kernel test robot
2022-08-14 15:49 ` kernel test robot
2022-07-26 20:15 ` [PATCH 5/6] net/tcp: Use crypto_pool for TCP-MD5 Dmitry Safonov
2022-07-26 20:16 ` [PATCH 6/6] net/ipv6: sr: Switch to using crypto_pool Dmitry Safonov
2022-07-27 2:52 ` Jakub Kicinski
2022-08-08 3:40 ` liulongfang
2022-08-23 19:23 ` Dmitry Safonov
2022-07-27 0:17 ` [PATCH 0/6] net/crypto: Introduce crypto_pool Herbert Xu
2022-07-27 15:52 ` Leonard Crestez
2022-07-28 9:26 ` Herbert Xu
2022-07-29 16:13 ` Dmitry Safonov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220726201600.1715505-2-dima@arista.com \
--to=dima@arista.com \
--cc=0x7f454c46@gmail.com \
--cc=ardb@kernel.org \
--cc=cdleonard@gmail.com \
--cc=davem@davemloft.net \
--cc=dsahern@kernel.org \
--cc=ebiggers@kernel.org \
--cc=edumazet@google.com \
--cc=fruggeri@arista.com \
--cc=herbert@gondor.apana.org.au \
--cc=kuba@kernel.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@amacapital.net \
--cc=netdev@vger.kernel.org \
--cc=noureddine@arista.com \
--cc=pabeni@redhat.com \
--cc=yoshfuji@linux-ipv6.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).