From: guoren@kernel.org To: palmer@rivosinc.com, arnd@arndb.de, mingo@redhat.com, will@kernel.org, longman@redhat.com, boqun.feng@gmail.com Cc: linux-riscv@lists.infradead.org, linux-arch@vger.kernel.org, linux-kernel@vger.kernel.org, Guo Ren <guoren@linux.alibaba.com>, Guo Ren <guoren@kernel.org>, Peter Zijlstra <peterz@infradead.org> Subject: [PATCH V7 3/5] asm-generic: ticket-lock: Move into ticket_spinlock.h Date: Tue, 28 Jun 2022 04:17:05 -0400 [thread overview] Message-ID: <20220628081707.1997728-4-guoren@kernel.org> (raw) In-Reply-To: <20220628081707.1997728-1-guoren@kernel.org> From: Guo Ren <guoren@linux.alibaba.com> Move ticket-lock definition into an independent file. It's a preparation patch for the following combo spinlock. Signed-off-by: Guo Ren <guoren@linux.alibaba.com> Signed-off-by: Guo Ren <guoren@kernel.org> Cc: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Arnd Bergmann <arnd@arndb.de> Cc: Palmer Dabbelt <palmer@rivosinc.com> --- include/asm-generic/spinlock.h | 44 ++----------- include/asm-generic/ticket_spinlock.h | 92 +++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 37 deletions(-) create mode 100644 include/asm-generic/ticket_spinlock.h diff --git a/include/asm-generic/spinlock.h b/include/asm-generic/spinlock.h index 4caeb8cebe53..f41dc7c2b900 100644 --- a/include/asm-generic/spinlock.h +++ b/include/asm-generic/spinlock.h @@ -27,66 +27,36 @@ #ifndef __ASM_GENERIC_SPINLOCK_H #define __ASM_GENERIC_SPINLOCK_H -#include <linux/atomic.h> -#include <asm-generic/spinlock_types.h> +#include <asm-generic/ticket_spinlock.h> static __always_inline void arch_spin_lock(arch_spinlock_t *lock) { - u32 val = atomic_fetch_add(1<<16, &lock->val); - u16 ticket = val >> 16; - - if (ticket == (u16)val) - return; - - /* - * atomic_cond_read_acquire() is RCpc, but rather than defining a - * custom cond_read_rcsc() here we just emit a full fence. We only - * need the prior reads before subsequent writes ordering from - * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we - * have no outstanding writes due to the atomic_fetch_add() the extra - * orderings are free. - */ - atomic_cond_read_acquire(&lock->val, ticket == (u16)VAL); - smp_mb(); + ticket_spin_lock(lock); } static __always_inline bool arch_spin_trylock(arch_spinlock_t *lock) { - u32 old = atomic_read(&lock->val); - - if ((old >> 16) != (old & 0xffff)) - return false; - - return atomic_try_cmpxchg(&lock->val, &old, old + (1<<16)); /* SC, for RCsc */ + return ticket_spin_trylock(lock); } static __always_inline void arch_spin_unlock(arch_spinlock_t *lock) { - u16 *ptr = (u16 *)lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN); - u32 val = atomic_read(&lock->val); - - smp_store_release(ptr, (u16)val + 1); + ticket_spin_unlock(lock); } static __always_inline int arch_spin_is_locked(arch_spinlock_t *lock) { - u32 val = atomic_read(&lock->val); - - return ((val >> 16) != (val & 0xffff)); + return ticket_spin_is_locked(lock); } static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock) { - u32 val = atomic_read(&lock->val); - - return (s16)((val >> 16) - (val & 0xffff)) > 1; + return ticket_spin_is_contended(lock); } static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) { - u32 val = lock.val.counter; - - return ((val >> 16) == (val & 0xffff)); + return ticket_spin_value_unlocked(lock); } #include <asm/qrwlock.h> diff --git a/include/asm-generic/ticket_spinlock.h b/include/asm-generic/ticket_spinlock.h new file mode 100644 index 000000000000..83e769398eea --- /dev/null +++ b/include/asm-generic/ticket_spinlock.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * 'Generic' ticket-lock implementation. + * + * It relies on atomic_fetch_add() having well defined forward progress + * guarantees under contention. If your architecture cannot provide this, stick + * to a test-and-set lock. + * + * It also relies on atomic_fetch_add() being safe vs smp_store_release() on a + * sub-word of the value. This is generally true for anything LL/SC although + * you'd be hard pressed to find anything useful in architecture specifications + * about this. If your architecture cannot do this you might be better off with + * a test-and-set. + * + * It further assumes atomic_*_release() + atomic_*_acquire() is RCpc and hence + * uses atomic_fetch_add() which is RCsc to create an RCsc hot path, along with + * a full fence after the spin to upgrade the otherwise-RCpc + * atomic_cond_read_acquire(). + * + * The implementation uses smp_cond_load_acquire() to spin, so if the + * architecture has WFE like instructions to sleep instead of poll for word + * modifications be sure to implement that (see ARM64 for example). + * + */ + +#ifndef __ASM_GENERIC_TICKET_SPINLOCK_H +#define __ASM_GENERIC_TICKET_SPINLOCK_H + +#include <linux/atomic.h> +#include <asm-generic/spinlock_types.h> + +static __always_inline void ticket_spin_lock(arch_spinlock_t *lock) +{ + u32 val = atomic_fetch_add(1<<16, &lock->val); + u16 ticket = val >> 16; + + if (ticket == (u16)val) + return; + + /* + * atomic_cond_read_acquire() is RCpc, but rather than defining a + * custom cond_read_rcsc() here we just emit a full fence. We only + * need the prior reads before subsequent writes ordering from + * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we + * have no outstanding writes due to the atomic_fetch_add() the extra + * orderings are free. + */ + atomic_cond_read_acquire(&lock->val, ticket == (u16)VAL); + smp_mb(); +} + +static __always_inline bool ticket_spin_trylock(arch_spinlock_t *lock) +{ + u32 old = atomic_read(&lock->val); + + if ((old >> 16) != (old & 0xffff)) + return false; + + return atomic_try_cmpxchg(&lock->val, &old, old + (1<<16)); /* SC, for RCsc */ +} + +static __always_inline void ticket_spin_unlock(arch_spinlock_t *lock) +{ + u16 *ptr = (u16 *)lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN); + u32 val = atomic_read(&lock->val); + + smp_store_release(ptr, (u16)val + 1); +} + +static __always_inline int ticket_spin_is_locked(arch_spinlock_t *lock) +{ + u32 val = atomic_read(&lock->val); + + return ((val >> 16) != (val & 0xffff)); +} + +static __always_inline int ticket_spin_is_contended(arch_spinlock_t *lock) +{ + u32 val = atomic_read(&lock->val); + + return (s16)((val >> 16) - (val & 0xffff)) > 1; +} + +static __always_inline int ticket_spin_value_unlocked(arch_spinlock_t lock) +{ + u32 val = lock.val.counter; + + return ((val >> 16) == (val & 0xffff)); +} + +#endif /* __ASM_GENERIC_TICKET_SPINLOCK_H */ -- 2.36.1
WARNING: multiple messages have this Message-ID (diff)
From: guoren@kernel.org To: palmer@rivosinc.com, arnd@arndb.de, mingo@redhat.com, will@kernel.org, longman@redhat.com, boqun.feng@gmail.com Cc: linux-riscv@lists.infradead.org, linux-arch@vger.kernel.org, linux-kernel@vger.kernel.org, Guo Ren <guoren@linux.alibaba.com>, Guo Ren <guoren@kernel.org>, Peter Zijlstra <peterz@infradead.org> Subject: [PATCH V7 3/5] asm-generic: ticket-lock: Move into ticket_spinlock.h Date: Tue, 28 Jun 2022 04:17:05 -0400 [thread overview] Message-ID: <20220628081707.1997728-4-guoren@kernel.org> (raw) In-Reply-To: <20220628081707.1997728-1-guoren@kernel.org> From: Guo Ren <guoren@linux.alibaba.com> Move ticket-lock definition into an independent file. It's a preparation patch for the following combo spinlock. Signed-off-by: Guo Ren <guoren@linux.alibaba.com> Signed-off-by: Guo Ren <guoren@kernel.org> Cc: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Arnd Bergmann <arnd@arndb.de> Cc: Palmer Dabbelt <palmer@rivosinc.com> --- include/asm-generic/spinlock.h | 44 ++----------- include/asm-generic/ticket_spinlock.h | 92 +++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 37 deletions(-) create mode 100644 include/asm-generic/ticket_spinlock.h diff --git a/include/asm-generic/spinlock.h b/include/asm-generic/spinlock.h index 4caeb8cebe53..f41dc7c2b900 100644 --- a/include/asm-generic/spinlock.h +++ b/include/asm-generic/spinlock.h @@ -27,66 +27,36 @@ #ifndef __ASM_GENERIC_SPINLOCK_H #define __ASM_GENERIC_SPINLOCK_H -#include <linux/atomic.h> -#include <asm-generic/spinlock_types.h> +#include <asm-generic/ticket_spinlock.h> static __always_inline void arch_spin_lock(arch_spinlock_t *lock) { - u32 val = atomic_fetch_add(1<<16, &lock->val); - u16 ticket = val >> 16; - - if (ticket == (u16)val) - return; - - /* - * atomic_cond_read_acquire() is RCpc, but rather than defining a - * custom cond_read_rcsc() here we just emit a full fence. We only - * need the prior reads before subsequent writes ordering from - * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we - * have no outstanding writes due to the atomic_fetch_add() the extra - * orderings are free. - */ - atomic_cond_read_acquire(&lock->val, ticket == (u16)VAL); - smp_mb(); + ticket_spin_lock(lock); } static __always_inline bool arch_spin_trylock(arch_spinlock_t *lock) { - u32 old = atomic_read(&lock->val); - - if ((old >> 16) != (old & 0xffff)) - return false; - - return atomic_try_cmpxchg(&lock->val, &old, old + (1<<16)); /* SC, for RCsc */ + return ticket_spin_trylock(lock); } static __always_inline void arch_spin_unlock(arch_spinlock_t *lock) { - u16 *ptr = (u16 *)lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN); - u32 val = atomic_read(&lock->val); - - smp_store_release(ptr, (u16)val + 1); + ticket_spin_unlock(lock); } static __always_inline int arch_spin_is_locked(arch_spinlock_t *lock) { - u32 val = atomic_read(&lock->val); - - return ((val >> 16) != (val & 0xffff)); + return ticket_spin_is_locked(lock); } static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock) { - u32 val = atomic_read(&lock->val); - - return (s16)((val >> 16) - (val & 0xffff)) > 1; + return ticket_spin_is_contended(lock); } static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) { - u32 val = lock.val.counter; - - return ((val >> 16) == (val & 0xffff)); + return ticket_spin_value_unlocked(lock); } #include <asm/qrwlock.h> diff --git a/include/asm-generic/ticket_spinlock.h b/include/asm-generic/ticket_spinlock.h new file mode 100644 index 000000000000..83e769398eea --- /dev/null +++ b/include/asm-generic/ticket_spinlock.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * 'Generic' ticket-lock implementation. + * + * It relies on atomic_fetch_add() having well defined forward progress + * guarantees under contention. If your architecture cannot provide this, stick + * to a test-and-set lock. + * + * It also relies on atomic_fetch_add() being safe vs smp_store_release() on a + * sub-word of the value. This is generally true for anything LL/SC although + * you'd be hard pressed to find anything useful in architecture specifications + * about this. If your architecture cannot do this you might be better off with + * a test-and-set. + * + * It further assumes atomic_*_release() + atomic_*_acquire() is RCpc and hence + * uses atomic_fetch_add() which is RCsc to create an RCsc hot path, along with + * a full fence after the spin to upgrade the otherwise-RCpc + * atomic_cond_read_acquire(). + * + * The implementation uses smp_cond_load_acquire() to spin, so if the + * architecture has WFE like instructions to sleep instead of poll for word + * modifications be sure to implement that (see ARM64 for example). + * + */ + +#ifndef __ASM_GENERIC_TICKET_SPINLOCK_H +#define __ASM_GENERIC_TICKET_SPINLOCK_H + +#include <linux/atomic.h> +#include <asm-generic/spinlock_types.h> + +static __always_inline void ticket_spin_lock(arch_spinlock_t *lock) +{ + u32 val = atomic_fetch_add(1<<16, &lock->val); + u16 ticket = val >> 16; + + if (ticket == (u16)val) + return; + + /* + * atomic_cond_read_acquire() is RCpc, but rather than defining a + * custom cond_read_rcsc() here we just emit a full fence. We only + * need the prior reads before subsequent writes ordering from + * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we + * have no outstanding writes due to the atomic_fetch_add() the extra + * orderings are free. + */ + atomic_cond_read_acquire(&lock->val, ticket == (u16)VAL); + smp_mb(); +} + +static __always_inline bool ticket_spin_trylock(arch_spinlock_t *lock) +{ + u32 old = atomic_read(&lock->val); + + if ((old >> 16) != (old & 0xffff)) + return false; + + return atomic_try_cmpxchg(&lock->val, &old, old + (1<<16)); /* SC, for RCsc */ +} + +static __always_inline void ticket_spin_unlock(arch_spinlock_t *lock) +{ + u16 *ptr = (u16 *)lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN); + u32 val = atomic_read(&lock->val); + + smp_store_release(ptr, (u16)val + 1); +} + +static __always_inline int ticket_spin_is_locked(arch_spinlock_t *lock) +{ + u32 val = atomic_read(&lock->val); + + return ((val >> 16) != (val & 0xffff)); +} + +static __always_inline int ticket_spin_is_contended(arch_spinlock_t *lock) +{ + u32 val = atomic_read(&lock->val); + + return (s16)((val >> 16) - (val & 0xffff)) > 1; +} + +static __always_inline int ticket_spin_value_unlocked(arch_spinlock_t lock) +{ + u32 val = lock.val.counter; + + return ((val >> 16) == (val & 0xffff)); +} + +#endif /* __ASM_GENERIC_TICKET_SPINLOCK_H */ -- 2.36.1 _______________________________________________ linux-riscv mailing list linux-riscv@lists.infradead.org http://lists.infradead.org/mailman/listinfo/linux-riscv
next prev parent reply other threads:[~2022-06-28 8:20 UTC|newest] Thread overview: 48+ messages / expand[flat|nested] mbox.gz Atom feed top 2022-06-28 8:17 [PATCH V7 0/5] riscv: Add qspinlock support with combo style guoren 2022-06-28 8:17 ` guoren 2022-06-28 8:17 ` [PATCH V7 1/5] asm-generic: ticket-lock: Remove unnecessary atomic_read guoren 2022-06-28 8:17 ` guoren 2022-06-28 18:05 ` Waiman Long 2022-06-28 18:05 ` Waiman Long 2022-06-29 2:12 ` Guo Ren 2022-06-29 2:12 ` Guo Ren 2022-06-29 8:27 ` David Laight 2022-06-29 8:27 ` David Laight 2022-07-01 15:18 ` Guo Ren 2022-07-01 15:18 ` Guo Ren 2022-07-04 9:52 ` Peter Zijlstra 2022-07-04 9:52 ` Peter Zijlstra 2022-07-04 11:10 ` Guo Ren 2022-07-04 11:10 ` Guo Ren 2022-06-28 8:17 ` [PATCH V7 2/5] asm-generic: ticket-lock: Use the same struct definitions with qspinlock guoren 2022-06-28 8:17 ` guoren 2022-06-28 8:17 ` guoren [this message] 2022-06-28 8:17 ` [PATCH V7 3/5] asm-generic: ticket-lock: Move into ticket_spinlock.h guoren 2022-06-28 8:17 ` [PATCH V7 4/5] asm-generic: spinlock: Add combo spinlock (ticket & queued) guoren 2022-06-28 8:17 ` guoren 2022-06-28 18:13 ` Waiman Long 2022-06-28 18:13 ` Waiman Long 2022-06-29 1:17 ` Guo Ren 2022-06-29 1:17 ` Guo Ren 2022-06-29 1:34 ` Waiman Long 2022-06-29 1:34 ` Waiman Long 2022-06-29 2:29 ` Guo Ren 2022-06-29 2:29 ` Guo Ren 2022-06-29 7:08 ` Arnd Bergmann 2022-06-29 7:08 ` Arnd Bergmann 2022-06-29 8:24 ` Guo Ren 2022-06-29 8:24 ` Guo Ren 2022-06-29 8:29 ` Arnd Bergmann 2022-06-29 8:29 ` Arnd Bergmann 2022-07-01 12:18 ` Guo Ren 2022-07-01 12:18 ` Guo Ren 2022-06-29 12:53 ` Waiman Long 2022-06-29 12:53 ` Waiman Long 2022-07-04 9:57 ` Peter Zijlstra 2022-07-04 9:57 ` Peter Zijlstra 2022-07-04 13:13 ` Guo Ren 2022-07-04 13:13 ` Guo Ren 2022-07-04 13:45 ` Peter Zijlstra 2022-07-04 13:45 ` Peter Zijlstra 2022-06-28 8:17 ` [PATCH V7 5/5] riscv: Add qspinlock support guoren 2022-06-28 8:17 ` guoren
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20220628081707.1997728-4-guoren@kernel.org \ --to=guoren@kernel.org \ --cc=arnd@arndb.de \ --cc=boqun.feng@gmail.com \ --cc=guoren@linux.alibaba.com \ --cc=linux-arch@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-riscv@lists.infradead.org \ --cc=longman@redhat.com \ --cc=mingo@redhat.com \ --cc=palmer@rivosinc.com \ --cc=peterz@infradead.org \ --cc=will@kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.