From: Robert Elliott <elliott@hpe.com>
To: herbert@gondor.apana.org.au, davem@davemloft.net,
tim.c.chen@linux.intel.com, linux-crypto@vger.kernel.org,
linux-kernel@vger.kernel.org
Cc: Robert Elliott <elliott@hpe.com>
Subject: [RFC PATCH 3/7] crypto: x86/crc - limit FPU preemption
Date: Thu, 6 Oct 2022 17:31:47 -0500 [thread overview]
Message-ID: <20221006223151.22159-4-elliott@hpe.com> (raw)
In-Reply-To: <20221006223151.22159-1-elliott@hpe.com>
As done by the ECB and CBC helpers in arch/x86/crypt/ecb_cbc_helpers.h,
limit the number of bytes processed between kernel_fpu_begin() and
kernel_fpu_end() calls.
Those functions call preempt_disable() and preempt_enable(), so
the CPU core is unavailable for scheduling while running, leading to:
rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: {12-... } 22 jiffies s: 277 root: 0x1/.
Fixes: 78c37d191dd6 ("crypto: crc32 - add crc32 pclmulqdq implementation and wrappers for table implementation")
Fixes: 6a8ce1ef3940 ("crypto: crc32c - Optimize CRC32C calculation with PCLMULQDQ instruction")
Fixes: 0b95a7f85718 ("crypto: crct10dif - Glue code to cast accelerated CRCT10DIF assembly as a crypto transform")
Suggested-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: Robert Elliott <elliott@hpe.com>
---
arch/x86/crypto/crc32-pclmul_glue.c | 18 ++++++++++----
arch/x86/crypto/crc32c-intel_glue.c | 32 ++++++++++++++++++++-----
arch/x86/crypto/crct10dif-pclmul_glue.c | 32 ++++++++++++++++++++-----
3 files changed, 66 insertions(+), 16 deletions(-)
diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c
index 288200fe7b4e..7cf65dc726c4 100644
--- a/arch/x86/crypto/crc32-pclmul_glue.c
+++ b/arch/x86/crypto/crc32-pclmul_glue.c
@@ -49,6 +49,8 @@
#define SCALE_F 16L /* size of xmm register */
#define SCALE_F_MASK (SCALE_F - 1)
+#define FPU_BYTES 4096U /* avoid kernel_fpu_begin/end scheduler/rcu stalls */
+
u32 crc32_pclmul_le_16(unsigned char const *buffer, size_t len, u32 crc32);
static u32 __attribute__((pure))
@@ -57,6 +59,7 @@ static u32 __attribute__((pure))
unsigned int iquotient;
unsigned int iremainder;
unsigned int prealign;
+ unsigned int chunk;
if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !crypto_simd_usable())
return crc32_le(crc, p, len);
@@ -73,12 +76,19 @@ static u32 __attribute__((pure))
iquotient = len & (~SCALE_F_MASK);
iremainder = len & SCALE_F_MASK;
- kernel_fpu_begin();
- crc = crc32_pclmul_le_16(p, iquotient, crc);
- kernel_fpu_end();
+ do {
+ chunk = min(iquotient, FPU_BYTES);
+ iquotient -= chunk;
+
+ kernel_fpu_begin();
+ crc = crc32_pclmul_le_16(p, chunk, crc);
+ kernel_fpu_end();
+
+ p += chunk;
+ } while (iquotient);
if (iremainder)
- crc = crc32_le(crc, p + iquotient, iremainder);
+ crc = crc32_le(crc, p, iremainder);
return crc;
}
diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c
index c5c965b694c6..b277c215f0fb 100644
--- a/arch/x86/crypto/crc32c-intel_glue.c
+++ b/arch/x86/crypto/crc32c-intel_glue.c
@@ -44,6 +44,8 @@
*/
#define CRC32C_PCL_BREAKEVEN 512
+#define FPU_BYTES 4096U /* avoid kernel_fpu_begin/end scheduler/rcu stalls */
+
asmlinkage unsigned int crc_pcl(const u8 *buffer, int len,
unsigned int crc_init);
#endif /* CONFIG_X86_64 */
@@ -155,15 +157,23 @@ static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
u32 *crcp = shash_desc_ctx(desc);
+ unsigned int chunk;
/*
* use faster PCL version if datasize is large enough to
* overcome kernel fpu state save/restore overhead
*/
if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
- kernel_fpu_begin();
- *crcp = crc_pcl(data, len, *crcp);
- kernel_fpu_end();
+ do {
+ chunk = min(len, FPU_BYTES);
+ len -= chunk;
+
+ kernel_fpu_begin();
+ *crcp = crc_pcl(data, chunk, *crcp);
+ kernel_fpu_end();
+
+ data += chunk;
+ } while (len);
} else
*crcp = crc32c_intel_le_hw(*crcp, data, len);
return 0;
@@ -172,10 +182,20 @@ static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
u8 *out)
{
+ unsigned int chunk;
+
if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
- kernel_fpu_begin();
- *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp));
- kernel_fpu_end();
+ do {
+ chunk = min(len, FPU_BYTES);
+ len -= chunk;
+
+ kernel_fpu_begin();
+ *crcp = crc_pcl(data, chunk, *crcp);
+ kernel_fpu_end();
+
+ data += chunk;
+ } while (len);
+ *(__le32 *)out = ~cpu_to_le32(*crcp);
} else
*(__le32 *)out =
~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c
index 7c5a32282d51..bcd362df6b62 100644
--- a/arch/x86/crypto/crct10dif-pclmul_glue.c
+++ b/arch/x86/crypto/crct10dif-pclmul_glue.c
@@ -36,6 +36,8 @@
#include <asm/cpu_device_id.h>
#include <asm/simd.h>
+#define FPU_BYTES 4096U /* avoid kernel_fpu_begin/end scheduler/rcu stalls */
+
asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len);
struct chksum_desc_ctx {
@@ -55,11 +57,19 @@ static int chksum_update(struct shash_desc *desc, const u8 *data,
unsigned int length)
{
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+ unsigned int chunk;
if (length >= 16 && crypto_simd_usable()) {
- kernel_fpu_begin();
- ctx->crc = crc_t10dif_pcl(ctx->crc, data, length);
- kernel_fpu_end();
+ do {
+ chunk = min(length, FPU_BYTES);
+ length -= chunk;
+
+ kernel_fpu_begin();
+ ctx->crc = crc_t10dif_pcl(ctx->crc, data, chunk);
+ kernel_fpu_end();
+
+ data += chunk;
+ } while (length);
} else
ctx->crc = crc_t10dif_generic(ctx->crc, data, length);
return 0;
@@ -75,10 +85,20 @@ static int chksum_final(struct shash_desc *desc, u8 *out)
static int __chksum_finup(__u16 crc, const u8 *data, unsigned int len, u8 *out)
{
+ unsigned int chunk;
+
if (len >= 16 && crypto_simd_usable()) {
- kernel_fpu_begin();
- *(__u16 *)out = crc_t10dif_pcl(crc, data, len);
- kernel_fpu_end();
+ do {
+ chunk = min(len, FPU_BYTES);
+ len -= chunk;
+
+ kernel_fpu_begin();
+ crc = crc_t10dif_pcl(crc, data, chunk);
+ kernel_fpu_end();
+
+ data += chunk;
+ } while (len);
+ *(__u16 *)out = crc;
} else
*(__u16 *)out = crc_t10dif_generic(crc, data, len);
return 0;
--
2.37.3
next prev parent reply other threads:[~2022-10-06 22:37 UTC|newest]
Thread overview: 126+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-10-06 22:31 [RFC PATCH 0/7] crypto: x86 - fix RCU stalls Robert Elliott
2022-10-06 22:31 ` [RFC PATCH 1/7] rcu: correct CONFIG_EXT_RCU_CPU_STALL_TIMEOUT descriptions Robert Elliott
2022-10-06 22:31 ` [RFC PATCH 2/7] crypto: x86/sha - limit FPU preemption Robert Elliott
2022-10-06 22:31 ` Robert Elliott [this message]
2022-10-06 22:31 ` [RFC PATCH 4/7] crypto: x86/sm3 " Robert Elliott
2022-10-06 22:31 ` [RFC PATCH 5/7] crypto: x86/ghash - restructure FPU context saving Robert Elliott
2022-10-06 22:31 ` [RFC PATCH 6/7] crypto: x86/ghash - limit FPU preemption Robert Elliott
2022-10-06 22:31 ` [RFC PATCH 7/7] crypto: x86 - use common macro for FPU limit Robert Elliott
2022-10-12 21:59 ` [PATCH v2 00/19] crypto: x86 - fix RCU stalls Robert Elliott
2022-10-12 21:59 ` [PATCH v2 01/19] crypto: tcrypt - test crc32 Robert Elliott
2022-10-12 21:59 ` [PATCH v2 02/19] crypto: tcrypt - test nhpoly1305 Robert Elliott
2022-10-12 21:59 ` [PATCH v2 03/19] crypto: tcrypt - reschedule during cycles speed tests Robert Elliott
2022-10-12 21:59 ` [PATCH v2 04/19] crypto: x86/sha - limit FPU preemption Robert Elliott
2022-10-13 0:41 ` Jason A. Donenfeld
2022-10-13 21:50 ` Elliott, Robert (Servers)
2022-10-14 11:01 ` David Laight
2022-10-13 5:57 ` Eric Biggers
2022-10-13 6:04 ` Herbert Xu
2022-10-13 6:08 ` Eric Biggers
2022-10-13 7:50 ` Herbert Xu
2022-10-13 22:41 ` :Re: " Elliott, Robert (Servers)
2022-10-12 21:59 ` [PATCH v2 05/19] crypto: x86/crc " Robert Elliott
2022-10-13 2:00 ` Herbert Xu
2022-10-13 22:34 ` Elliott, Robert (Servers)
2022-10-14 4:02 ` David Laight
2022-10-24 2:03 ` kernel test robot
2022-10-12 21:59 ` [PATCH v2 06/19] crypto: x86/sm3 " Robert Elliott
2022-10-12 21:59 ` [PATCH v2 07/19] crypto: x86/ghash - restructure FPU context saving Robert Elliott
2022-10-12 21:59 ` [PATCH v2 08/19] crypto: x86/ghash - limit FPU preemption Robert Elliott
2022-10-13 6:03 ` Eric Biggers
2022-10-13 22:52 ` Elliott, Robert (Servers)
2022-10-12 21:59 ` [PATCH v2 09/19] crypto: x86 - use common macro for FPU limit Robert Elliott
2022-10-13 0:35 ` Jason A. Donenfeld
2022-10-13 21:48 ` Elliott, Robert (Servers)
2022-10-14 1:26 ` Jason A. Donenfeld
2022-10-18 0:06 ` Elliott, Robert (Servers)
2022-10-12 21:59 ` [PATCH v2 10/19] crypto: x86/sha1, sha256 - load based on CPU features Robert Elliott
2022-10-12 21:59 ` [PATCH v2 11/19] crypto: x86/crc " Robert Elliott
2022-10-12 21:59 ` [PATCH v2 12/19] crypto: x86/sm3 " Robert Elliott
2022-10-12 21:59 ` [PATCH v2 13/19] crypto: x86/ghash " Robert Elliott
2022-10-12 21:59 ` [PATCH v2 14/19] crypto: x86 " Robert Elliott
2022-10-14 14:26 ` Elliott, Robert (Servers)
2022-10-12 21:59 ` [PATCH v2 15/19] crypto: x86 - add pr_fmt to all modules Robert Elliott
2022-10-12 21:59 ` [PATCH v2 16/19] crypto: x86 - print CPU optimized loaded messages Robert Elliott
2022-10-13 0:40 ` Jason A. Donenfeld
2022-10-13 13:47 ` kernel test robot
2022-10-13 13:48 ` kernel test robot
2022-10-12 21:59 ` [PATCH v2 17/19] crypto: x86 - standardize suboptimal prints Robert Elliott
2022-10-13 0:38 ` Jason A. Donenfeld
2022-10-12 21:59 ` [PATCH v2 18/19] crypto: x86 - standardize not loaded prints Robert Elliott
2022-10-13 0:42 ` Jason A. Donenfeld
2022-10-13 22:20 ` Elliott, Robert (Servers)
2022-11-10 22:06 ` Elliott, Robert (Servers)
2022-10-12 21:59 ` [PATCH v2 19/19] crypto: x86/sha - register only the best function Robert Elliott
2022-10-13 6:07 ` Eric Biggers
2022-10-13 7:52 ` Herbert Xu
2022-10-13 22:59 ` Elliott, Robert (Servers)
2022-10-14 8:22 ` Herbert Xu
2022-11-01 21:34 ` [PATCH v2 00/19] crypto: x86 - fix RCU stalls Elliott, Robert (Servers)
2022-11-03 4:27 ` [PATCH v3 00/17] crypt: " Robert Elliott
2022-11-03 4:27 ` [PATCH v3 01/17] crypto: tcrypt - test crc32 Robert Elliott
2022-11-03 4:27 ` [PATCH v3 02/17] crypto: tcrypt - test nhpoly1305 Robert Elliott
2022-11-03 4:27 ` [PATCH v3 03/17] crypto: tcrypt - reschedule during cycles speed tests Robert Elliott
2022-11-03 4:27 ` [PATCH v3 04/17] crypto: x86/sha - limit FPU preemption Robert Elliott
2022-11-03 4:27 ` [PATCH v3 05/17] crypto: x86/crc " Robert Elliott
2022-11-03 4:27 ` [PATCH v3 06/17] crypto: x86/sm3 " Robert Elliott
2022-11-03 4:27 ` [PATCH v3 07/17] crypto: x86/ghash - use u8 rather than char Robert Elliott
2022-11-03 4:27 ` [PATCH v3 08/17] crypto: x86/ghash - restructure FPU context saving Robert Elliott
2022-11-03 4:27 ` [PATCH v3 09/17] crypto: x86/ghash - limit FPU preemption Robert Elliott
2022-11-03 4:27 ` [PATCH v3 10/17] crypto: x86/*poly* " Robert Elliott
2022-11-03 4:27 ` [PATCH v3 11/17] crypto: x86/sha - register all variations Robert Elliott
2022-11-03 9:26 ` kernel test robot
2022-11-03 4:27 ` [PATCH v3 12/17] crypto: x86/sha - minimize time in FPU context Robert Elliott
2022-11-03 4:27 ` [PATCH v3 13/17] crypto: x86/sha1, sha256 - load based on CPU features Robert Elliott
2022-11-03 4:27 ` [PATCH v3 14/17] crypto: x86/crc " Robert Elliott
2022-11-03 4:27 ` [PATCH v3 15/17] crypto: x86/sm3 " Robert Elliott
2022-11-03 4:27 ` [PATCH v3 16/17] crypto: x86/ghash,polyval " Robert Elliott
2022-11-03 4:27 ` [PATCH v3 17/17] crypto: x86/nhpoly1305, poly1305 " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 00/24] crypto: fix RCU stalls Robert Elliott
2022-11-16 4:13 ` [PATCH v4 01/24] crypto: tcrypt - test crc32 Robert Elliott
2022-11-16 4:13 ` [PATCH v4 02/24] crypto: tcrypt - test nhpoly1305 Robert Elliott
2022-11-16 4:13 ` [PATCH v4 03/24] crypto: tcrypt - reschedule during cycles speed tests Robert Elliott
2022-11-16 4:13 ` [PATCH v4 04/24] crypto: x86/sha - limit FPU preemption Robert Elliott
2022-11-16 4:13 ` [PATCH v4 05/24] crypto: x86/crc " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 06/24] crypto: x86/sm3 " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 07/24] crypto: x86/ghash - use u8 rather than char Robert Elliott
2022-11-16 4:13 ` [PATCH v4 08/24] crypto: x86/ghash - restructure FPU context saving Robert Elliott
2022-11-16 4:13 ` [PATCH v4 09/24] crypto: x86/ghash - limit FPU preemption Robert Elliott
2022-11-16 4:13 ` [PATCH v4 10/24] crypto: x86/poly " Robert Elliott
2022-11-16 11:13 ` Jason A. Donenfeld
2022-11-22 5:06 ` Elliott, Robert (Servers)
2022-11-22 9:07 ` David Laight
2022-11-25 8:40 ` Herbert Xu
2022-11-25 8:59 ` Ard Biesheuvel
2022-11-25 9:03 ` Herbert Xu
2022-11-28 16:57 ` Elliott, Robert (Servers)
2022-11-28 18:48 ` Elliott, Robert (Servers)
2022-12-02 6:21 ` Elliott, Robert (Servers)
2022-12-02 9:25 ` Herbert Xu
2022-12-02 16:15 ` Elliott, Robert (Servers)
2022-12-06 4:27 ` Herbert Xu
2022-12-06 14:03 ` Peter Lafreniere
2022-12-06 14:44 ` David Laight
2022-12-06 23:06 ` Peter Lafreniere
2022-12-10 0:34 ` Elliott, Robert (Servers)
2022-12-16 22:12 ` Elliott, Robert (Servers)
2022-11-16 4:13 ` [PATCH v4 11/24] crypto: x86/aegis " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 12/24] crypto: x86/sha - register all variations Robert Elliott
2022-11-16 4:13 ` [PATCH v4 13/24] crypto: x86/sha - minimize time in FPU context Robert Elliott
2022-11-16 4:13 ` [PATCH v4 14/24] crypto: x86/sha - load based on CPU features Robert Elliott
2022-11-16 4:13 ` [PATCH v4 15/24] crypto: x86/crc " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 16/24] crypto: x86/sm3 " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 17/24] crypto: x86/poly " Robert Elliott
2022-11-16 11:19 ` Jason A. Donenfeld
2022-11-16 4:13 ` [PATCH v4 18/24] crypto: x86/ghash " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 19/24] crypto: x86/aesni - avoid type conversions Robert Elliott
2022-11-16 4:13 ` [PATCH v4 20/24] crypto: x86/ciphers - load based on CPU features Robert Elliott
2022-11-16 11:30 ` Jason A. Donenfeld
2022-11-16 4:13 ` [PATCH v4 21/24] crypto: x86 - report used CPU features via module parameters Robert Elliott
2022-11-16 11:26 ` Jason A. Donenfeld
2022-11-16 4:13 ` [PATCH v4 22/24] crypto: x86 - report missing " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 23/24] crypto: x86 - report suboptimal CPUs " Robert Elliott
2022-11-16 4:13 ` [PATCH v4 24/24] crypto: x86 - standarize module descriptions Robert Elliott
2022-11-17 3:58 ` [PATCH v4 00/24] crypto: fix RCU stalls Herbert Xu
2022-11-17 15:13 ` Elliott, Robert (Servers)
2022-11-17 15:15 ` Jason A. Donenfeld
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221006223151.22159-4-elliott@hpe.com \
--to=elliott@hpe.com \
--cc=davem@davemloft.net \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=tim.c.chen@linux.intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).