From: Matteo Croce <mcroce@linux.microsoft.com> To: linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org, Paul Walmsley <paul.walmsley@sifive.com>, Palmer Dabbelt <palmer@dabbelt.com>, Albert Ou <aou@eecs.berkeley.edu>, Atish Patra <atish.patra@wdc.com>, Emil Renner Berthing <kernel@esmil.dk>, Akira Tsukamoto <akira.tsukamoto@gmail.com>, Drew Fustini <drew@beagleboard.org>, Bin Meng <bmeng.cn@gmail.com>, David Laight <David.Laight@aculab.com>, Guo Ren <guoren@kernel.org> Subject: [PATCH v3 2/3] riscv: optimized memmove Date: Thu, 17 Jun 2021 17:27:53 +0200 [thread overview] Message-ID: <20210617152754.17960-3-mcroce@linux.microsoft.com> (raw) In-Reply-To: <20210617152754.17960-1-mcroce@linux.microsoft.com> From: Matteo Croce <mcroce@microsoft.com> When the destination buffer is before the source one, or when the buffers doesn't overlap, it's safe to use memcpy() instead, which is optimized to use a bigger data size possible. Signed-off-by: Matteo Croce <mcroce@microsoft.com> --- arch/riscv/include/asm/string.h | 6 ++-- arch/riscv/kernel/riscv_ksyms.c | 2 -- arch/riscv/lib/Makefile | 1 - arch/riscv/lib/memmove.S | 64 --------------------------------- arch/riscv/lib/string.c | 23 ++++++++++++ 5 files changed, 26 insertions(+), 70 deletions(-) delete mode 100644 arch/riscv/lib/memmove.S diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 6b5d6fc3eab4..25d9b9078569 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -17,11 +17,11 @@ extern asmlinkage void *__memset(void *, int, size_t); #define __HAVE_ARCH_MEMCPY extern void *memcpy(void *dest, const void *src, size_t count); extern void *__memcpy(void *dest, const void *src, size_t count); +#define __HAVE_ARCH_MEMMOVE +extern void *memmove(void *dest, const void *src, size_t count); +extern void *__memmove(void *dest, const void *src, size_t count); #endif -#define __HAVE_ARCH_MEMMOVE -extern asmlinkage void *memmove(void *, const void *, size_t); -extern asmlinkage void *__memmove(void *, const void *, size_t); /* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) #define memcpy(dst, src, len) __memcpy(dst, src, len) diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c index 3f6d512a5b97..361565c4db7e 100644 --- a/arch/riscv/kernel/riscv_ksyms.c +++ b/arch/riscv/kernel/riscv_ksyms.c @@ -10,6 +10,4 @@ * Assembly functions that may be used (directly or indirectly) by modules */ EXPORT_SYMBOL(memset); -EXPORT_SYMBOL(memmove); EXPORT_SYMBOL(__memset); -EXPORT_SYMBOL(__memmove); diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 2ffe85d4baee..484f5ff7b508 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only lib-y += delay.o lib-y += memset.o -lib-y += memmove.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE) += string.o diff --git a/arch/riscv/lib/memmove.S b/arch/riscv/lib/memmove.S deleted file mode 100644 index 07d1d2152ba5..000000000000 --- a/arch/riscv/lib/memmove.S +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#include <linux/linkage.h> -#include <asm/asm.h> - -ENTRY(__memmove) -WEAK(memmove) - move t0, a0 - move t1, a1 - - beq a0, a1, exit_memcpy - beqz a2, exit_memcpy - srli t2, a2, 0x2 - - slt t3, a0, a1 - beqz t3, do_reverse - - andi a2, a2, 0x3 - li t4, 1 - beqz t2, byte_copy - -word_copy: - lw t3, 0(a1) - addi t2, t2, -1 - addi a1, a1, 4 - sw t3, 0(a0) - addi a0, a0, 4 - bnez t2, word_copy - beqz a2, exit_memcpy - j byte_copy - -do_reverse: - add a0, a0, a2 - add a1, a1, a2 - andi a2, a2, 0x3 - li t4, -1 - beqz t2, reverse_byte_copy - -reverse_word_copy: - addi a1, a1, -4 - addi t2, t2, -1 - lw t3, 0(a1) - addi a0, a0, -4 - sw t3, 0(a0) - bnez t2, reverse_word_copy - beqz a2, exit_memcpy - -reverse_byte_copy: - addi a0, a0, -1 - addi a1, a1, -1 - -byte_copy: - lb t3, 0(a1) - addi a2, a2, -1 - sb t3, 0(a0) - add a1, a1, t4 - add a0, a0, t4 - bnez a2, byte_copy - -exit_memcpy: - move a0, t0 - move a1, t1 - ret -END(__memmove) diff --git a/arch/riscv/lib/string.c b/arch/riscv/lib/string.c index e48a79a045d8..9c7009d43c39 100644 --- a/arch/riscv/lib/string.c +++ b/arch/riscv/lib/string.c @@ -89,3 +89,26 @@ EXPORT_SYMBOL(__memcpy); void *memcpy(void *dest, const void *src, size_t count) __weak __alias(__memcpy); EXPORT_SYMBOL(memcpy); + +/* + * Simply check if the buffer overlaps an call memcpy() in case, + * otherwise do a simple one byte at time backward copy. + */ +void *__memmove(void *dest, const void *src, size_t count) +{ + if (dest < src || src + count <= dest) + return memcpy(dest, src, count); + + if (dest > src) { + const char *s = src + count; + char *tmp = dest + count; + + while (count--) + *--tmp = *--s; + } + return dest; +} +EXPORT_SYMBOL(__memmove); + +void *memmove(void *dest, const void *src, size_t count) __weak __alias(__memmove); +EXPORT_SYMBOL(memmove); -- 2.31.1
WARNING: multiple messages have this Message-ID (diff)
From: Matteo Croce <mcroce@linux.microsoft.com> To: linux-riscv@lists.infradead.org Cc: linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org, Paul Walmsley <paul.walmsley@sifive.com>, Palmer Dabbelt <palmer@dabbelt.com>, Albert Ou <aou@eecs.berkeley.edu>, Atish Patra <atish.patra@wdc.com>, Emil Renner Berthing <kernel@esmil.dk>, Akira Tsukamoto <akira.tsukamoto@gmail.com>, Drew Fustini <drew@beagleboard.org>, Bin Meng <bmeng.cn@gmail.com>, David Laight <David.Laight@aculab.com>, Guo Ren <guoren@kernel.org> Subject: [PATCH v3 2/3] riscv: optimized memmove Date: Thu, 17 Jun 2021 17:27:53 +0200 [thread overview] Message-ID: <20210617152754.17960-3-mcroce@linux.microsoft.com> (raw) In-Reply-To: <20210617152754.17960-1-mcroce@linux.microsoft.com> From: Matteo Croce <mcroce@microsoft.com> When the destination buffer is before the source one, or when the buffers doesn't overlap, it's safe to use memcpy() instead, which is optimized to use a bigger data size possible. Signed-off-by: Matteo Croce <mcroce@microsoft.com> --- arch/riscv/include/asm/string.h | 6 ++-- arch/riscv/kernel/riscv_ksyms.c | 2 -- arch/riscv/lib/Makefile | 1 - arch/riscv/lib/memmove.S | 64 --------------------------------- arch/riscv/lib/string.c | 23 ++++++++++++ 5 files changed, 26 insertions(+), 70 deletions(-) delete mode 100644 arch/riscv/lib/memmove.S diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 6b5d6fc3eab4..25d9b9078569 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -17,11 +17,11 @@ extern asmlinkage void *__memset(void *, int, size_t); #define __HAVE_ARCH_MEMCPY extern void *memcpy(void *dest, const void *src, size_t count); extern void *__memcpy(void *dest, const void *src, size_t count); +#define __HAVE_ARCH_MEMMOVE +extern void *memmove(void *dest, const void *src, size_t count); +extern void *__memmove(void *dest, const void *src, size_t count); #endif -#define __HAVE_ARCH_MEMMOVE -extern asmlinkage void *memmove(void *, const void *, size_t); -extern asmlinkage void *__memmove(void *, const void *, size_t); /* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) #define memcpy(dst, src, len) __memcpy(dst, src, len) diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c index 3f6d512a5b97..361565c4db7e 100644 --- a/arch/riscv/kernel/riscv_ksyms.c +++ b/arch/riscv/kernel/riscv_ksyms.c @@ -10,6 +10,4 @@ * Assembly functions that may be used (directly or indirectly) by modules */ EXPORT_SYMBOL(memset); -EXPORT_SYMBOL(memmove); EXPORT_SYMBOL(__memset); -EXPORT_SYMBOL(__memmove); diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 2ffe85d4baee..484f5ff7b508 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only lib-y += delay.o lib-y += memset.o -lib-y += memmove.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE) += string.o diff --git a/arch/riscv/lib/memmove.S b/arch/riscv/lib/memmove.S deleted file mode 100644 index 07d1d2152ba5..000000000000 --- a/arch/riscv/lib/memmove.S +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#include <linux/linkage.h> -#include <asm/asm.h> - -ENTRY(__memmove) -WEAK(memmove) - move t0, a0 - move t1, a1 - - beq a0, a1, exit_memcpy - beqz a2, exit_memcpy - srli t2, a2, 0x2 - - slt t3, a0, a1 - beqz t3, do_reverse - - andi a2, a2, 0x3 - li t4, 1 - beqz t2, byte_copy - -word_copy: - lw t3, 0(a1) - addi t2, t2, -1 - addi a1, a1, 4 - sw t3, 0(a0) - addi a0, a0, 4 - bnez t2, word_copy - beqz a2, exit_memcpy - j byte_copy - -do_reverse: - add a0, a0, a2 - add a1, a1, a2 - andi a2, a2, 0x3 - li t4, -1 - beqz t2, reverse_byte_copy - -reverse_word_copy: - addi a1, a1, -4 - addi t2, t2, -1 - lw t3, 0(a1) - addi a0, a0, -4 - sw t3, 0(a0) - bnez t2, reverse_word_copy - beqz a2, exit_memcpy - -reverse_byte_copy: - addi a0, a0, -1 - addi a1, a1, -1 - -byte_copy: - lb t3, 0(a1) - addi a2, a2, -1 - sb t3, 0(a0) - add a1, a1, t4 - add a0, a0, t4 - bnez a2, byte_copy - -exit_memcpy: - move a0, t0 - move a1, t1 - ret -END(__memmove) diff --git a/arch/riscv/lib/string.c b/arch/riscv/lib/string.c index e48a79a045d8..9c7009d43c39 100644 --- a/arch/riscv/lib/string.c +++ b/arch/riscv/lib/string.c @@ -89,3 +89,26 @@ EXPORT_SYMBOL(__memcpy); void *memcpy(void *dest, const void *src, size_t count) __weak __alias(__memcpy); EXPORT_SYMBOL(memcpy); + +/* + * Simply check if the buffer overlaps an call memcpy() in case, + * otherwise do a simple one byte at time backward copy. + */ +void *__memmove(void *dest, const void *src, size_t count) +{ + if (dest < src || src + count <= dest) + return memcpy(dest, src, count); + + if (dest > src) { + const char *s = src + count; + char *tmp = dest + count; + + while (count--) + *--tmp = *--s; + } + return dest; +} +EXPORT_SYMBOL(__memmove); + +void *memmove(void *dest, const void *src, size_t count) __weak __alias(__memmove); +EXPORT_SYMBOL(memmove); -- 2.31.1 _______________________________________________ linux-riscv mailing list linux-riscv@lists.infradead.org http://lists.infradead.org/mailman/listinfo/linux-riscv
next prev parent reply other threads:[~2021-06-17 15:28 UTC|newest] Thread overview: 48+ messages / expand[flat|nested] mbox.gz Atom feed top 2021-06-17 15:27 [PATCH v3 0/3] riscv: optimized mem* functions Matteo Croce 2021-06-17 15:27 ` Matteo Croce 2021-06-17 15:27 ` [PATCH v3 1/3] riscv: optimized memcpy Matteo Croce 2021-06-17 15:27 ` Matteo Croce 2021-06-18 14:06 ` kernel test robot 2021-06-18 14:06 ` kernel test robot 2021-06-18 14:06 ` kernel test robot 2021-06-21 14:26 ` Christoph Hellwig 2021-06-21 14:26 ` Christoph Hellwig 2021-06-22 8:19 ` David Laight 2021-06-22 8:19 ` David Laight 2021-06-22 22:53 ` Matteo Croce 2021-06-22 22:53 ` Matteo Croce 2021-06-22 22:00 ` Matteo Croce 2021-06-22 22:00 ` Matteo Croce 2021-06-22 0:14 ` Nick Kossifidis 2021-06-22 0:14 ` Nick Kossifidis 2021-06-22 23:35 ` Matteo Croce 2021-06-22 23:35 ` Matteo Croce 2021-06-23 9:48 ` Nick Kossifidis 2021-06-23 9:48 ` Nick Kossifidis 2021-06-17 15:27 ` Matteo Croce [this message] 2021-06-17 15:27 ` [PATCH v3 2/3] riscv: optimized memmove Matteo Croce 2021-06-21 14:28 ` Christoph Hellwig 2021-06-21 14:28 ` Christoph Hellwig 2021-06-22 0:46 ` Nick Kossifidis 2021-06-22 0:46 ` Nick Kossifidis 2021-06-30 4:40 ` kernel test robot 2021-06-30 4:40 ` kernel test robot 2021-06-30 4:40 ` kernel test robot 2021-06-17 15:27 ` [PATCH v3 3/3] riscv: optimized memset Matteo Croce 2021-06-17 15:27 ` Matteo Croce 2021-06-21 14:32 ` Christoph Hellwig 2021-06-21 14:32 ` Christoph Hellwig 2021-06-22 1:07 ` Nick Kossifidis 2021-06-22 1:07 ` Nick Kossifidis 2021-06-22 8:38 ` David Laight 2021-06-22 8:38 ` David Laight 2021-06-23 1:14 ` Matteo Croce 2021-06-23 1:14 ` Matteo Croce 2021-06-23 9:05 ` David Laight 2021-06-23 9:05 ` David Laight 2021-06-23 0:08 ` Matteo Croce 2021-06-23 0:08 ` Matteo Croce 2021-06-22 1:09 ` [PATCH v3 0/3] riscv: optimized mem* functions Nick Kossifidis 2021-06-22 1:09 ` Nick Kossifidis 2021-06-22 2:39 ` Guo Ren 2021-06-22 2:39 ` Guo Ren
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210617152754.17960-3-mcroce@linux.microsoft.com \ --to=mcroce@linux.microsoft.com \ --cc=David.Laight@aculab.com \ --cc=akira.tsukamoto@gmail.com \ --cc=aou@eecs.berkeley.edu \ --cc=atish.patra@wdc.com \ --cc=bmeng.cn@gmail.com \ --cc=drew@beagleboard.org \ --cc=guoren@kernel.org \ --cc=kernel@esmil.dk \ --cc=linux-arch@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-riscv@lists.infradead.org \ --cc=palmer@dabbelt.com \ --cc=paul.walmsley@sifive.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.