From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S932842AbbI3UDw (ORCPT ); Wed, 30 Sep 2015 16:03:52 -0400 Received: from smtp2.provo.novell.com ([137.65.250.81]:41628 "EHLO smtp2.provo.novell.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754234AbbI3UDp (ORCPT ); Wed, 30 Sep 2015 16:03:45 -0400 From: Davidlohr Bueso To: Peter Zijlstra , Ingo Molnar , Thomas Gleixner Cc: Andrew Morton , Linus Torvalds , Will Deacon , "Paul E.McKenney" , linux-kernel@vger.kernel.org, Davidlohr Bueso , Davidlohr Bueso Subject: [PATCH 2/5] locking/mutex: Use acquire/release semantics Date: Wed, 30 Sep 2015 13:03:12 -0700 Message-Id: <1443643395-17016-3-git-send-email-dave@stgolabs.net> X-Mailer: git-send-email 2.1.4 In-Reply-To: <1443643395-17016-1-git-send-email-dave@stgolabs.net> References: <1443643395-17016-1-git-send-email-dave@stgolabs.net> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org As of 654672d4ba1 (locking/atomics: Add _{acquire|release|relaxed}() variants of some atomic operations) and 6d79ef2d30e (locking, asm-generic: Add _{relaxed|acquire|release}() variants for 'atomic_long_t'), weakly ordered archs can benefit from more relaxed use of barriers when locking and unlocking, instead of regular full barrier semantics. While currently only arm64 supports such optimizations, updating corresponding locking primitives serves for other archs to immediately benefit as well, once the necessary machinery is implemented of course. Signed-off-by: Davidlohr Bueso --- include/asm-generic/mutex-dec.h | 8 ++++---- include/asm-generic/mutex-xchg.h | 10 +++++----- kernel/locking/mutex.c | 9 +++++---- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/asm-generic/mutex-dec.h b/include/asm-generic/mutex-dec.h index d4f9fb4..fd694cf 100644 --- a/include/asm-generic/mutex-dec.h +++ b/include/asm-generic/mutex-dec.h @@ -20,7 +20,7 @@ static inline void __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *)) { - if (unlikely(atomic_dec_return(count) < 0)) + if (unlikely(atomic_dec_return_acquire(count) < 0)) fail_fn(count); } @@ -35,7 +35,7 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *)) static inline int __mutex_fastpath_lock_retval(atomic_t *count) { - if (unlikely(atomic_dec_return(count) < 0)) + if (unlikely(atomic_dec_return_acquire(count) < 0)) return -1; return 0; } @@ -56,7 +56,7 @@ __mutex_fastpath_lock_retval(atomic_t *count) static inline void __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) { - if (unlikely(atomic_inc_return(count) <= 0)) + if (unlikely(atomic_inc_return_release(count) <= 0)) fail_fn(count); } @@ -80,7 +80,7 @@ __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) static inline int __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) { - if (likely(atomic_cmpxchg(count, 1, 0) == 1)) + if (likely(atomic_cmpxchg_acquire(count, 1, 0) == 1)) return 1; return 0; } diff --git a/include/asm-generic/mutex-xchg.h b/include/asm-generic/mutex-xchg.h index f169ec0..a6b4a7b 100644 --- a/include/asm-generic/mutex-xchg.h +++ b/include/asm-generic/mutex-xchg.h @@ -31,7 +31,7 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *)) * to ensure that any waiting tasks are woken up by the * unlock slow path. */ - if (likely(atomic_xchg(count, -1) != 1)) + if (likely(atomic_xchg_acquire(count, -1) != 1)) fail_fn(count); } @@ -46,7 +46,7 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *)) static inline int __mutex_fastpath_lock_retval(atomic_t *count) { - if (unlikely(atomic_xchg(count, 0) != 1)) + if (unlikely(atomic_xchg_acquire(count, 0) != 1)) if (likely(atomic_xchg(count, -1) != 1)) return -1; return 0; @@ -67,7 +67,7 @@ __mutex_fastpath_lock_retval(atomic_t *count) static inline void __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) { - if (unlikely(atomic_xchg(count, 1) != 0)) + if (unlikely(atomic_xchg_release(count, 1) != 0)) fail_fn(count); } @@ -91,7 +91,7 @@ __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) static inline int __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) { - int prev = atomic_xchg(count, 0); + int prev = atomic_xchg_acquire(count, 0); if (unlikely(prev < 0)) { /* @@ -105,7 +105,7 @@ __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) * owner's unlock path needlessly, but that's not a problem * in practice. ] */ - prev = atomic_xchg(count, prev); + prev = atomic_xchg_acquire(count, prev); if (prev < 0) prev = 0; } diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 4cccea6..0551c21 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -277,7 +277,7 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock) static inline bool mutex_try_to_acquire(struct mutex *lock) { return !mutex_is_locked(lock) && - (atomic_cmpxchg(&lock->count, 1, 0) == 1); + (atomic_cmpxchg_acquire(&lock->count, 1, 0) == 1); } /* @@ -529,7 +529,8 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, * Once more, try to acquire the lock. Only try-lock the mutex if * it is unlocked to reduce unnecessary xchg() operations. */ - if (!mutex_is_locked(lock) && (atomic_xchg(&lock->count, 0) == 1)) + if (!mutex_is_locked(lock) && + (atomic_xchg_acquire(&lock->count, 0) == 1)) goto skip_wait; debug_mutex_lock_common(lock, &waiter); @@ -553,7 +554,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, * non-negative in order to avoid unnecessary xchg operations: */ if (atomic_read(&lock->count) >= 0 && - (atomic_xchg(&lock->count, -1) == 1)) + (atomic_xchg_acquire(&lock->count, -1) == 1)) break; /* @@ -867,7 +868,7 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count) spin_lock_mutex(&lock->wait_lock, flags); - prev = atomic_xchg(&lock->count, -1); + prev = atomic_xchg_acquire(&lock->count, -1); if (likely(prev == 1)) { mutex_set_owner(lock); mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); -- 2.1.4