* [PATCH 00/10] atomic.h : standardizing atomic primitives
@ 2007-02-11 19:17 Mathieu Desnoyers
2007-02-11 19:17 ` [PATCH 01/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to alpha Mathieu Desnoyers
` (9 more replies)
0 siblings, 10 replies; 14+ messages in thread
From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw)
To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers
atomic.h : standardizing atomic primitives
It mainly adds support for missing 64 bits cmpxchg and 64 bits atomic add
unless. Therefore, principally 64 bits architectures are targeted by these
patches. It also adds the complete list of atomic operations on the atomic_long
type.
These patches compile fine against :
arm
i686
ia64
m68k
mips
mipsel
x86_64
ppc 405
powerpc 970
s390
sparc
sparc64
I haven't built cross-compilers for other architectures yet.
These patches apply on 2.6.20
Signed-off-by : Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH 01/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to alpha 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 02/10] atomic.h : Complete atomic_long operations in asm-generic Mathieu Desnoyers ` (8 subsequent siblings) 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Add atomic64 cmpxchg, xchg and add_unless to alpha Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-alpha/atomic.h +++ b/include/asm-alpha/atomic.h @@ -175,19 +175,62 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) return result; } -#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +/** + * atomic_add_unless - add unless the number is a given value + * @v: pointer of type atomic_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns non-zero if @v was not @u, and zero otherwise. + */ #define atomic_add_unless(v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) +/** + * atomic64_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns non-zero if @v was not @u, and zero otherwise. + */ +#define atomic64_add_unless(v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) + #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) #define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 02/10] atomic.h : Complete atomic_long operations in asm-generic 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 01/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to alpha Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 03/10] atomic.h : i386 type safety fix Mathieu Desnoyers ` (7 subsequent siblings) 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Complete atomic_long operations in asm-generic Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -9,6 +9,7 @@ */ #include <asm/types.h> +#include <asm/system.h> /* * Suppport for atomic_long_t @@ -66,6 +67,72 @@ static inline void atomic_long_sub(long i, atomic_long_t *l) atomic64_sub(i, v); } +static inline int atomic_long_sub_and_test(long i, atomic_long_t *l) +{ + atomic64_t *v = (atomic64_t *)l; + + return atomic64_sub_and_test(i, v); +} + +static inline int atomic_long_dec_and_test(atomic_long_t *l) +{ + atomic64_t *v = (atomic64_t *)l; + + return atomic64_dec_and_test(v); +} + +static inline int atomic_long_inc_and_test(atomic_long_t *l) +{ + atomic64_t *v = (atomic64_t *)l; + + return atomic64_inc_and_test(v); +} + +static inline int atomic_long_add_negative(long i, atomic_long_t *l) +{ + atomic64_t *v = (atomic64_t *)l; + + return atomic64_add_negative(i, v); +} + +static inline long atomic_long_add_return(long i, atomic_long_t *l) +{ + atomic64_t *v = (atomic64_t *)l; + + return (long)atomic64_add_return(i, v); +} + +static inline long atomic_long_sub_return(long i, atomic_long_t *l) +{ + atomic64_t *v = (atomic64_t *)l; + + return (long)atomic64_sub_return(i, v); +} + +static inline long atomic_long_inc_return(atomic_long_t *l) +{ + atomic64_t *v = (atomic64_t *)l; + + return (long)atomic64_inc_return(v); +} + +static inline long atomic_long_dec_return(atomic_long_t *l) +{ + atomic64_t *v = (atomic64_t *)l; + + return (long)atomic64_dec_return(v); +} + +#define atomic_long_add_unless(l, a, u) \ + atomic64_add_unless((atomic64_t *)(l), (a), (u)) + +#define atomic_long_inc_not_zero(l) atomic64_inc_not_zero((atomic64_t *)(l)) + +#define atomic_long_cmpxchg(l, old, new) \ + (atomic_cmpxchg((atomic64_t *)(l), (old), (new))) +#define atomic_long_xchg(v, new) \ + (atomic_xchg((atomic64_t *)(l), (new))) + #else /* BITS_PER_LONG == 64 */ typedef atomic_t atomic_long_t; @@ -113,6 +180,72 @@ static inline void atomic_long_sub(long i, atomic_long_t *l) atomic_sub(i, v); } +static inline int atomic_long_sub_and_test(long i, atomic_long_t *l) +{ + atomic_t *v = (atomic_t *)l; + + return atomic_sub_and_test(i, v); +} + +static inline int atomic_long_dec_and_test(atomic_long_t *l) +{ + atomic_t *v = (atomic_t *)l; + + return atomic_dec_and_test(v); +} + +static inline int atomic_long_inc_and_test(atomic_long_t *l) +{ + atomic_t *v = (atomic_t *)l; + + return atomic_inc_and_test(v); +} + +static inline int atomic_long_add_negative(long i, atomic_long_t *l) +{ + atomic_t *v = (atomic_t *)l; + + return atomic_add_negative(i, v); +} + +static inline long atomic_long_add_return(long i, atomic_long_t *l) +{ + atomic_t *v = (atomic_t *)l; + + return (long)atomic_add_return(i, v); +} + +static inline long atomic_long_sub_return(long i, atomic_long_t *l) +{ + atomic_t *v = (atomic_t *)l; + + return (long)atomic_sub_return(i, v); +} + +static inline long atomic_long_inc_return(atomic_long_t *l) +{ + atomic_t *v = (atomic_t *)l; + + return (long)atomic_inc_return(v); +} + +static inline long atomic_long_dec_return(atomic_long_t *l) +{ + atomic_t *v = (atomic_t *)l; + + return (long)atomic_dec_return(v); +} + +#define atomic_long_add_unless(l, a, u) \ + atomic_add_unless((atomic_t *)(l), (a), (u)) + +#define atomic_long_inc_not_zero(l) atomic_inc_not_zero((atomic_t *)(l)) + +#define atomic_long_cmpxchg(l, old, new) \ + (atomic_cmpxchg((atomic_t *)(l), (old), (new))) +#define atomic_long_xchg(v, new) \ + (atomic_xchg((atomic_t *)(l), (new))) + #endif /* BITS_PER_LONG == 64 */ #endif /* _ASM_GENERIC_ATOMIC_H */ ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 03/10] atomic.h : i386 type safety fix 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 01/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to alpha Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 02/10] atomic.h : Complete atomic_long operations in asm-generic Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 04/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to ia64 Mathieu Desnoyers ` (6 subsequent siblings) 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : i386 type safety fix This patch removes an explicit cast to an integer type for the result returned by cmpxchg. It is not per se a problem on the i386 architecture, because sizeof(int) == sizeof(long), but whenever this code is cut'n'pasted to a different architecture (which happened at least for x86_64), it would simply accept passing an atomic64_t value as parameter to cmpxchg, xchg and add_unless, having 64 bits inputs casted to 32 bits. Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-i386/atomic.h +++ b/include/asm-i386/atomic.h @@ -207,8 +207,8 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) return atomic_add_return(-i,v); } -#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg(v, new) (xchg(&((v)->counter), (new))) /** * atomic_add_unless - add unless the number is a given value @@ -221,7 +221,7 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) */ #define atomic_add_unless(v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \ ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 04/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to ia64 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers ` (2 preceding siblings ...) 2007-02-11 19:17 ` [PATCH 03/10] atomic.h : i386 type safety fix Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 05/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to mips Mathieu Desnoyers ` (5 subsequent siblings) 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Add atomic64 cmpxchg, xchg and add_unless to ia64 Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-ia64/atomic.h +++ b/include/asm-ia64/atomic.h @@ -88,12 +88,16 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) return new; } -#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define atomic64_cmpxchg(v, old, new) \ + (cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) + #define atomic_add_unless(v, a, u) \ ({ \ - int c, old; \ + __typeof__(v->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \ @@ -107,6 +111,22 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) +#define atomic64_add_unless(v, a, u) \ +({ \ + __typeof__(v->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) + #define atomic_add_return(i,v) \ ({ \ int __ia64_aar_i = (i); \ ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 05/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to mips 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers ` (3 preceding siblings ...) 2007-02-11 19:17 ` [PATCH 04/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to ia64 Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 06/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to parisc Mathieu Desnoyers ` (4 subsequent siblings) 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Add atomic64 cmpxchg, xchg and add_unless to mips Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-mips/atomic.h +++ b/include/asm-mips/atomic.h @@ -291,8 +291,8 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) return result; } -#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg(v, new) (xchg(&((v)->counter), (new))) /** * atomic_add_unless - add unless the number is a given value @@ -305,7 +305,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) */ #define atomic_add_unless(v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \ @@ -651,6 +651,29 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) return result; } +#define atomic64_cmpxchg(v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg(v, new) (xchg(&((v)->counter), (new))) + +/** + * atomic64_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns non-zero if @v was not @u, and zero otherwise. + */ +#define atomic64_add_unless(v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) + #define atomic64_dec_return(v) atomic64_sub_return(1,(v)) #define atomic64_inc_return(v) atomic64_add_return(1,(v)) ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 06/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to parisc 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers ` (4 preceding siblings ...) 2007-02-11 19:17 ` [PATCH 05/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to mips Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 07/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to powerpc Mathieu Desnoyers ` (3 subsequent siblings) 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Add atomic64 cmpxchg, xchg and add_unless to parisc Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-parisc/atomic.h +++ b/include/asm-parisc/atomic.h @@ -6,7 +6,6 @@ #define _ASM_PARISC_ATOMIC_H_ #include <linux/types.h> -#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -163,7 +162,7 @@ static __inline__ int atomic_read(const atomic_t *v) } /* exported interface */ -#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) /** @@ -177,7 +176,7 @@ static __inline__ int atomic_read(const atomic_t *v) */ #define atomic_add_unless(v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \ @@ -270,6 +269,31 @@ atomic64_read(const atomic64_t *v) #define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0) #define atomic64_sub_and_test(i,v) (atomic64_sub_return((i),(v)) == 0) +/* exported interface */ +#define atomic64_cmpxchg(v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) + +/** + * atomic64_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns non-zero if @v was not @u, and zero otherwise. + */ +#define atomic64_add_unless(v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) + + #endif /* __LP64__ */ #include <asm-generic/atomic.h> ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 07/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to powerpc 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers ` (5 preceding siblings ...) 2007-02-11 19:17 ` [PATCH 06/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to parisc Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 08/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to sparc64 Mathieu Desnoyers ` (2 subsequent siblings) 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Add atomic64 cmpxchg, xchg and add_unless to powerpc Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-powerpc/atomic.h +++ b/include/asm-powerpc/atomic.h @@ -165,7 +165,7 @@ static __inline__ int atomic_dec_return(atomic_t *v) return t; } -#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) /** @@ -413,6 +413,42 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) return t; } +#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) + +/** + * atomic64_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns non-zero if @v was not @u, and zero otherwise. + */ +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long t; + + __asm__ __volatile__ ( + LWSYNC_ON_SMP +"1: ldarx %0,0,%1 # atomic_add_unless\n\ + cmpd 0,%0,%3 \n\ + beq- 2f \n\ + add %0,%2,%0 \n" +" stdcx. %0,0,%1 \n\ + bne- 1b \n" + ISYNC_ON_SMP +" subf %0,%2,%0 \n\ +2:" + : "=&r" (t) + : "r" (&v->counter), "r" (a), "r" (u) + : "cc", "memory"); + + return t != u; +} + +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) + #endif /* __powerpc64__ */ #include <asm-generic/atomic.h> --- a/include/asm-powerpc/bitops.h +++ b/include/asm-powerpc/bitops.h @@ -39,7 +39,6 @@ #ifdef __KERNEL__ #include <linux/compiler.h> -#include <asm/atomic.h> #include <asm/asm-compat.h> #include <asm/synch.h> --- a/include/asm-powerpc/system.h +++ b/include/asm-powerpc/system.h @@ -7,7 +7,6 @@ #include <linux/kernel.h> #include <asm/hw_irq.h> -#include <asm/atomic.h> /* * Memory barrier. @@ -226,6 +225,29 @@ __xchg_u32(volatile void *p, unsigned long val) return prev; } +/* + * Atomic exchange + * + * Changes the memory location '*ptr' to be val and returns + * the previous value stored there. + */ +static __inline__ unsigned long +__xchg_u32_local(volatile void *p, unsigned long val) +{ + unsigned long prev; + + __asm__ __volatile__( +"1: lwarx %0,0,%2 \n" + PPC405_ERR77(0,%2) +" stwcx. %3,0,%2 \n\ + bne- 1b" + : "=&r" (prev), "+m" (*(volatile unsigned int *)p) + : "r" (p), "r" (val) + : "cc", "memory"); + + return prev; +} + #ifdef CONFIG_PPC64 static __inline__ unsigned long __xchg_u64(volatile void *p, unsigned long val) @@ -245,6 +267,23 @@ __xchg_u64(volatile void *p, unsigned long val) return prev; } + +static __inline__ unsigned long +__xchg_u64_local(volatile void *p, unsigned long val) +{ + unsigned long prev; + + __asm__ __volatile__( +"1: ldarx %0,0,%2 \n" + PPC405_ERR77(0,%2) +" stdcx. %3,0,%2 \n\ + bne- 1b" + : "=&r" (prev), "+m" (*(volatile unsigned long *)p) + : "r" (p), "r" (val) + : "cc", "memory"); + + return prev; +} #endif /* @@ -268,12 +307,33 @@ __xchg(volatile void *ptr, unsigned long x, unsigned int size) return x; } +static __inline__ unsigned long +__xchg_local(volatile void *ptr, unsigned long x, unsigned int size) +{ + switch (size) { + case 4: + return __xchg_u32_local(ptr, x); +#ifdef CONFIG_PPC64 + case 8: + return __xchg_u64_local(ptr, x); +#endif + } + __xchg_called_with_bad_pointer(); + return x; +} #define xchg(ptr,x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \ }) +#define xchg_local(ptr,x) \ + ({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __xchg_local((ptr), \ + (unsigned long)_x_, sizeof(*(ptr))); \ + }) + #define tas(ptr) (xchg((ptr),1)) /* @@ -305,6 +365,28 @@ __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new) return prev; } +static __inline__ unsigned long +__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old, + unsigned long new) +{ + unsigned int prev; + + __asm__ __volatile__ ( +"1: lwarx %0,0,%2 # __cmpxchg_u32\n\ + cmpw 0,%0,%3\n\ + bne- 2f\n" + PPC405_ERR77(0,%2) +" stwcx. %4,0,%2\n\ + bne- 1b" + "\n\ +2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} + #ifdef CONFIG_PPC64 static __inline__ unsigned long __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new) @@ -327,6 +409,27 @@ __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new) return prev; } + +static __inline__ unsigned long +__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old, + unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: ldarx %0,0,%2 # __cmpxchg_u64\n\ + cmpd 0,%0,%3\n\ + bne- 2f\n\ + stdcx. %4,0,%2\n\ + bne- 1b" + "\n\ +2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} #endif /* This function doesn't exist, so you'll get a linker error @@ -349,6 +452,22 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, return old; } +static __inline__ unsigned long +__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, + unsigned int size) +{ + switch (size) { + case 4: + return __cmpxchg_u32_local(ptr, old, new); +#ifdef CONFIG_PPC64 + case 8: + return __cmpxchg_u64_local(ptr, old, new); +#endif + } + __cmpxchg_called_with_bad_pointer(); + return old; +} + #define cmpxchg(ptr,o,n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ @@ -357,6 +476,15 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, (unsigned long)_n_, sizeof(*(ptr))); \ }) + +#define cmpxchg_local(ptr,o,n) \ + ({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, sizeof(*(ptr))); \ + }) + #ifdef CONFIG_PPC64 /* * We handle most unaligned accesses in hardware. On the other hand ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 08/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to sparc64 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers ` (6 preceding siblings ...) 2007-02-11 19:17 ` [PATCH 07/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to powerpc Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 09/10] atomic.h : Add atomic64_xchg to s390 Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 10/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64 Mathieu Desnoyers 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Add atomic64 cmpxchg, xchg and add_unless to sparc64 Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-sparc64/atomic.h +++ b/include/asm-sparc64/atomic.h @@ -70,12 +70,12 @@ extern int atomic64_sub_ret(int, atomic64_t *); #define atomic_add_negative(i, v) (atomic_add_ret(i, v) < 0) #define atomic64_add_negative(i, v) (atomic64_add_ret(i, v) < 0) -#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) #define atomic_add_unless(v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \ @@ -89,6 +89,26 @@ extern int atomic64_sub_ret(int, atomic64_t *); }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) +#define atomic64_cmpxchg(v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) + +#define atomic64_add_unless(v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + likely(c != (u)); \ +}) +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) + /* Atomic operations are already serializing */ #ifdef CONFIG_SMP #define smp_mb__before_atomic_dec() membar_storeload_loadload(); ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 09/10] atomic.h : Add atomic64_xchg to s390 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers ` (7 preceding siblings ...) 2007-02-11 19:17 ` [PATCH 08/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to sparc64 Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 10/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64 Mathieu Desnoyers 9 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Add atomic64_xchg to s390 Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-s390/atomic.h +++ b/include/asm-s390/atomic.h @@ -215,6 +215,8 @@ static __inline__ void atomic64_set_mask(unsigned long mask, atomic64_t * v) __CSG_LOOP(v, mask, "ogr"); } +#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) + static __inline__ long long atomic64_cmpxchg(atomic64_t *v, long long old, long long new) { ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 10/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers ` (8 preceding siblings ...) 2007-02-11 19:17 ` [PATCH 09/10] atomic.h : Add atomic64_xchg to s390 Mathieu Desnoyers @ 2007-02-11 19:17 ` Mathieu Desnoyers 2007-02-12 11:35 ` Andi Kleen 9 siblings, 1 reply; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-11 19:17 UTC (permalink / raw) To: linux-kernel; +Cc: Andrew Morton, Mathieu Desnoyers atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64 Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> --- a/include/asm-x86_64/atomic.h +++ b/include/asm-x86_64/atomic.h @@ -375,8 +375,8 @@ static __inline__ long atomic64_add_return(long i, atomic64_t *v) long __i = i; __asm__ __volatile__( LOCK_PREFIX "xaddq %0, %1;" - :"=r"(i) - :"m"(v->counter), "0"(i)); + :"+r" (i), "+m" (v->counter) + : : "memory"); return i + __i; } @@ -388,7 +388,10 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) #define atomic64_inc_return(v) (atomic64_add_return(1,v)) #define atomic64_dec_return(v) (atomic64_sub_return(1,v)) -#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) /** @@ -402,7 +405,7 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) */ #define atomic_add_unless(v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \ @@ -416,6 +419,31 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) +/** + * atomic64_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns non-zero if @v was not @u, and zero otherwise. + */ +#define atomic64_add_unless(v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) + /* These are x86-specific, used by some header files */ #define atomic_clear_mask(mask, addr) \ __asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \ ^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 10/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64 2007-02-11 19:17 ` [PATCH 10/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64 Mathieu Desnoyers @ 2007-02-12 11:35 ` Andi Kleen 2007-02-12 16:43 ` Mathieu Desnoyers 0 siblings, 1 reply; 14+ messages in thread From: Andi Kleen @ 2007-02-12 11:35 UTC (permalink / raw) To: Mathieu Desnoyers; +Cc: linux-kernel, Andrew Morton Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> writes: > > /** > @@ -402,7 +405,7 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) > */ > #define atomic_add_unless(v, a, u) \ > ({ \ > - int c, old; \ > + __typeof__((v)->counter) c, old; \ This change seems useless. Please drop. In general you should enforce atomic_t vs atomic64_t types on x86, otherwise people will write code that won't compile on other architectures that have truly different implementations. I would suggest using inlines. -Andi ^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 10/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64 2007-02-12 11:35 ` Andi Kleen @ 2007-02-12 16:43 ` Mathieu Desnoyers 2007-02-12 16:49 ` [PATCH] atomic.h : atomic_add_unless as inline. Remove system.h atomic.h circular dependency Mathieu Desnoyers 0 siblings, 1 reply; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-12 16:43 UTC (permalink / raw) To: Andi Kleen; +Cc: linux-kernel, Andrew Morton * Andi Kleen (ak@suse.de) wrote: > Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> writes: > > > > /** > > @@ -402,7 +405,7 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) > > */ > > #define atomic_add_unless(v, a, u) \ > > ({ \ > > - int c, old; \ > > + __typeof__((v)->counter) c, old; \ > > This change seems useless. Please drop. > > In general you should enforce atomic_t vs atomic64_t types on x86, otherwise > people will write code that won't compile on other architectures > that have truly different implementations. > > I would suggest using inlines. > > -Andi Hi Andi, Yes, I agree this typeof is not needed and more error prone. All the original atomic.h code that uses cmpxchg (which includes the atomic_add_unless) uses defines instead of inline functions, probably to circumvent a circular dependency between system.h and atomic.h on powerpc (which my patch addresses). Therefore, it makes sense to use inline functions that will provide type checking. Digging into the FRV architecture shows me that it is also affected by such a circular dependency. Here is the diff applying this against the rest of my atomic.h patches. Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> diff --git a/include/asm-alpha/atomic.h b/include/asm-alpha/atomic.h index 7b4fba8..f5cb7b8 100644 --- a/include/asm-alpha/atomic.h +++ b/include/asm-alpha/atomic.h @@ -2,6 +2,7 @@ #define _ALPHA_ATOMIC_H #include <asm/barrier.h> +#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -190,20 +191,21 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /** @@ -215,20 +217,21 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic64_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic64_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) diff --git a/include/asm-arm/atomic.h b/include/asm-arm/atomic.h index ea88aa6..b86b000 100644 --- a/include/asm-arm/atomic.h +++ b/include/asm-arm/atomic.h @@ -12,6 +12,7 @@ #define __ASM_ARM_ATOMIC_H #include <linux/compiler.h> +#include <asm/system.h> typedef struct { volatile int counter; } atomic_t; diff --git a/include/asm-arm26/atomic.h b/include/asm-arm26/atomic.h index 97e944f..d6dd423 100644 --- a/include/asm-arm26/atomic.h +++ b/include/asm-arm26/atomic.h @@ -20,7 +20,6 @@ #ifndef __ASM_ARM_ATOMIC_H #define __ASM_ARM_ATOMIC_H - #ifdef CONFIG_SMP #error SMP is NOT supported #endif diff --git a/include/asm-avr32/atomic.h b/include/asm-avr32/atomic.h diff --git a/include/asm-frv/atomic.h b/include/asm-frv/atomic.h index 066386a..d425d8d 100644 --- a/include/asm-frv/atomic.h +++ b/include/asm-frv/atomic.h @@ -16,6 +16,7 @@ #include <linux/types.h> #include <asm/spr-regs.h> +#include <asm/system.h> #ifdef CONFIG_SMP #error not SMP safe @@ -258,85 +259,23 @@ extern uint32_t __xchg_32(uint32_t i, volatile void *v); #define tas(ptr) (xchg((ptr), 1)) -/*****************************************************************************/ -/* - * compare and conditionally exchange value with memory - * - if (*ptr == test) then orig = *ptr; *ptr = test; - * - if (*ptr != test) then orig = *ptr; - */ -#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS - -#define cmpxchg(ptr, test, new) \ -({ \ - __typeof__(ptr) __xg_ptr = (ptr); \ - __typeof__(*(ptr)) __xg_orig, __xg_tmp; \ - __typeof__(*(ptr)) __xg_test = (test); \ - __typeof__(*(ptr)) __xg_new = (new); \ - \ - switch (sizeof(__xg_orig)) { \ - case 4: \ - asm volatile( \ - "0: \n" \ - " orcc gr0,gr0,gr0,icc3 \n" \ - " ckeq icc3,cc7 \n" \ - " ld.p %M0,%1 \n" \ - " orcr cc7,cc7,cc3 \n" \ - " sub%I4cc %1,%4,%2,icc0 \n" \ - " bne icc0,#0,1f \n" \ - " cst.p %3,%M0 ,cc3,#1 \n" \ - " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ - " beq icc3,#0,0b \n" \ - "1: \n" \ - : "+U"(*__xg_ptr), "=&r"(__xg_orig), "=&r"(__xg_tmp) \ - : "r"(__xg_new), "NPr"(__xg_test) \ - : "memory", "cc7", "cc3", "icc3", "icc0" \ - ); \ - break; \ - \ - default: \ - __xg_orig = 0; \ - asm volatile("break"); \ - break; \ - } \ - \ - __xg_orig; \ -}) - -#else - -extern uint32_t __cmpxchg_32(uint32_t *v, uint32_t test, uint32_t new); - -#define cmpxchg(ptr, test, new) \ -({ \ - __typeof__(ptr) __xg_ptr = (ptr); \ - __typeof__(*(ptr)) __xg_orig; \ - __typeof__(*(ptr)) __xg_test = (test); \ - __typeof__(*(ptr)) __xg_new = (new); \ - \ - switch (sizeof(__xg_orig)) { \ - case 4: __xg_orig = __cmpxchg_32(__xg_ptr, __xg_test, __xg_new); break; \ - default: \ - __xg_orig = 0; \ - asm volatile("break"); \ - break; \ - } \ - \ - __xg_orig; \ -}) - -#endif - #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) diff --git a/include/asm-frv/bitops.h b/include/asm-frv/bitops.h diff --git a/include/asm-frv/system.h b/include/asm-frv/system.h index 1166899..be303b3 100644 --- a/include/asm-frv/system.h +++ b/include/asm-frv/system.h @@ -13,7 +13,6 @@ #define _ASM_SYSTEM_H #include <linux/linkage.h> -#include <asm/atomic.h> struct thread_struct; @@ -197,4 +196,73 @@ extern void free_initmem(void); #define arch_align_stack(x) (x) +/*****************************************************************************/ +/* + * compare and conditionally exchange value with memory + * - if (*ptr == test) then orig = *ptr; *ptr = test; + * - if (*ptr != test) then orig = *ptr; + */ +#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS + +#define cmpxchg(ptr, test, new) \ +({ \ + __typeof__(ptr) __xg_ptr = (ptr); \ + __typeof__(*(ptr)) __xg_orig, __xg_tmp; \ + __typeof__(*(ptr)) __xg_test = (test); \ + __typeof__(*(ptr)) __xg_new = (new); \ + \ + switch (sizeof(__xg_orig)) { \ + case 4: \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ld.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " sub%I4cc %1,%4,%2,icc0 \n" \ + " bne icc0,#0,1f \n" \ + " cst.p %3,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + "1: \n" \ + : "+U"(*__xg_ptr), "=&r"(__xg_orig), "=&r"(__xg_tmp) \ + : "r"(__xg_new), "NPr"(__xg_test) \ + : "memory", "cc7", "cc3", "icc3", "icc0" \ + ); \ + break; \ + \ + default: \ + __xg_orig = 0; \ + asm volatile("break"); \ + break; \ + } \ + \ + __xg_orig; \ +}) + +#else + +extern uint32_t __cmpxchg_32(uint32_t *v, uint32_t test, uint32_t new); + +#define cmpxchg(ptr, test, new) \ +({ \ + __typeof__(ptr) __xg_ptr = (ptr); \ + __typeof__(*(ptr)) __xg_orig; \ + __typeof__(*(ptr)) __xg_test = (test); \ + __typeof__(*(ptr)) __xg_new = (new); \ + \ + switch (sizeof(__xg_orig)) { \ + case 4: __xg_orig = __cmpxchg_32(__xg_ptr, __xg_test, __xg_new); break; \ + default: \ + __xg_orig = 0; \ + asm volatile("break"); \ + break; \ + } \ + \ + __xg_orig; \ +}) + +#endif + + #endif /* _ASM_SYSTEM_H */ diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h index 069576f..8cc36d8 100644 --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -9,7 +9,6 @@ */ #include <asm/types.h> -#include <asm/system.h> /* * Suppport for atomic_long_t @@ -123,8 +122,12 @@ static inline long atomic_long_dec_return(atomic_long_t *l) return (long)atomic64_dec_return(v); } -#define atomic_long_add_unless(l, a, u) \ - atomic64_add_unless((atomic64_t *)(l), (a), (u)) +static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u) +{ + atomic64_t *v = (atomic64_t *)l; + + return (long)atomic64_add_unless(v, a, u); +} #define atomic_long_inc_not_zero(l) atomic64_inc_not_zero((atomic64_t *)(l)) @@ -236,8 +239,12 @@ static inline long atomic_long_dec_return(atomic_long_t *l) return (long)atomic_dec_return(v); } -#define atomic_long_add_unless(l, a, u) \ - atomic_add_unless((atomic_t *)(l), (a), (u)) +static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u) +{ + atomic_t *v = (atomic_t *)l; + + return (long)atomic_add_unless(v, a, u); +} #define atomic_long_inc_not_zero(l) atomic_inc_not_zero((atomic_t *)(l)) diff --git a/include/asm-i386/atomic.h b/include/asm-i386/atomic.h index 9bd1050..17e4376 100644 --- a/include/asm-i386/atomic.h +++ b/include/asm-i386/atomic.h @@ -219,20 +219,21 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc_return(v) (atomic_add_return(1,v)) diff --git a/include/asm-ia64/atomic.h b/include/asm-ia64/atomic.h index b16ad23..1fc3b83 100644 --- a/include/asm-ia64/atomic.h +++ b/include/asm-ia64/atomic.h @@ -15,6 +15,7 @@ #include <linux/types.h> #include <asm/intrinsics.h> +#include <asm/system.h> /* * On IA-64, counter must always be volatile to ensure that that the @@ -95,36 +96,38 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) (cmpxchg(&((v)->counter), old, new)) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__(v->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__(v->counter) c, old; \ - c = atomic64_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic64_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #define atomic_add_return(i,v) \ diff --git a/include/asm-m32r/atomic.h b/include/asm-m32r/atomic.h index 873e656..663ac88 100644 --- a/include/asm-m32r/atomic.h +++ b/include/asm-m32r/atomic.h @@ -253,14 +253,21 @@ static __inline__ int atomic_dec_return(atomic_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) static __inline__ void atomic_clear_mask(unsigned long mask, atomic_t *addr) diff --git a/include/asm-m68k/atomic.h b/include/asm-m68k/atomic.h index e2dfbde..a1b64a6 100644 --- a/include/asm-m68k/atomic.h +++ b/include/asm-m68k/atomic.h @@ -2,7 +2,7 @@ #define __ARCH_M68K_ATOMIC__ -#include <asm/system.h> /* local_irq_XXX() */ +#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -170,20 +170,21 @@ static inline void atomic_set_mask(unsigned long mask, unsigned long *v) __asm__ __volatile__("orl %1,%0" : "+m" (*v) : "id" (mask)); } -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /* Atomic operations are already serializing */ diff --git a/include/asm-m68knommu/atomic.h b/include/asm-m68knommu/atomic.h index 18a2e3b..b20e2cb 100644 --- a/include/asm-m68knommu/atomic.h +++ b/include/asm-m68knommu/atomic.h @@ -1,7 +1,7 @@ #ifndef __ARCH_M68KNOMMU_ATOMIC__ #define __ARCH_M68KNOMMU_ATOMIC__ -#include <asm/system.h> /* local_irq_XXX() */ +#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -131,14 +131,21 @@ static inline int atomic_sub_return(int i, atomic_t * v) #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_dec_return(v) atomic_sub_return(1,(v)) diff --git a/include/asm-mips/atomic.h b/include/asm-mips/atomic.h index 01a0303..fa9f742 100644 --- a/include/asm-mips/atomic.h +++ b/include/asm-mips/atomic.h @@ -18,6 +18,7 @@ #include <asm/barrier.h> #include <asm/cpu-features.h> #include <asm/war.h> +#include <asm/system.h> typedef struct { volatile int counter; } atomic_t; @@ -303,14 +304,20 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_dec_return(v) atomic_sub_return(1,(v)) @@ -664,14 +671,21 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #define atomic64_dec_return(v) atomic64_sub_return(1,(v)) diff --git a/include/asm-parisc/atomic.h b/include/asm-parisc/atomic.h index 09404ab..afb7cf5 100644 --- a/include/asm-parisc/atomic.h +++ b/include/asm-parisc/atomic.h @@ -6,6 +6,7 @@ #define _ASM_PARISC_ATOMIC_H_ #include <linux/types.h> +#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -174,14 +175,21 @@ static __inline__ int atomic_read(const atomic_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_add(i,v) ((void)(__atomic_add_return( ((int)i),(v)))) @@ -283,14 +291,21 @@ atomic64_read(const atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic64_read(v); \ - while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) diff --git a/include/asm-powerpc/atomic.h b/include/asm-powerpc/atomic.h index 438a7fc..c44810b 100644 --- a/include/asm-powerpc/atomic.h +++ b/include/asm-powerpc/atomic.h @@ -11,6 +11,7 @@ typedef struct { volatile int counter; } atomic_t; #include <linux/compiler.h> #include <asm/synch.h> #include <asm/asm-compat.h> +#include <asm/system.h> #define ATOMIC_INIT(i) { (i) } diff --git a/include/asm-ppc/system.h b/include/asm-ppc/system.h index 7389435..56abe5e 100644 --- a/include/asm-ppc/system.h +++ b/include/asm-ppc/system.h @@ -6,7 +6,6 @@ #include <linux/kernel.h> -#include <asm/atomic.h> #include <asm/hw_irq.h> /* diff --git a/include/asm-sparc64/atomic.h b/include/asm-sparc64/atomic.h index c3feb3a..3fb4e1f 100644 --- a/include/asm-sparc64/atomic.h +++ b/include/asm-sparc64/atomic.h @@ -9,6 +9,7 @@ #define __ARCH_SPARC64_ATOMIC__ #include <linux/types.h> +#include <asm/system.h> typedef struct { volatile int counter; } atomic_t; typedef struct { volatile __s64 counter; } atomic64_t; @@ -73,40 +74,42 @@ extern int atomic64_sub_ret(int, atomic64_t *); #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - likely(c != (u)); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic64_cmpxchg(v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic64_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic64_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - likely(c != (u)); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) /* Atomic operations are already serializing */ diff --git a/include/asm-x86_64/atomic.h b/include/asm-x86_64/atomic.h index 80e4fdb..19e0c60 100644 --- a/include/asm-x86_64/atomic.h +++ b/include/asm-x86_64/atomic.h @@ -2,6 +2,7 @@ #define __ARCH_X86_64_ATOMIC__ #include <asm/alternative.h> +#include <asm/system.h> /* atomic_t should be 32 bit signed type */ @@ -403,20 +404,21 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /** @@ -428,20 +430,21 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic64_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic64_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) /* These are x86-specific, used by some header files */ diff --git a/include/asm-xtensa/atomic.h b/include/asm-xtensa/atomic.h index d73e0fe..9044303 100644 --- a/include/asm-xtensa/atomic.h +++ b/include/asm-xtensa/atomic.h @@ -234,14 +234,21 @@ static inline int atomic_sub_return(int i, atomic_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -- Mathieu Desnoyers Computer Engineering Ph.D. Candidate, École Polytechnique de Montréal OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68 ^ permalink raw reply related [flat|nested] 14+ messages in thread
* Re: [PATCH] atomic.h : atomic_add_unless as inline. Remove system.h atomic.h circular dependency 2007-02-12 16:43 ` Mathieu Desnoyers @ 2007-02-12 16:49 ` Mathieu Desnoyers 0 siblings, 0 replies; 14+ messages in thread From: Mathieu Desnoyers @ 2007-02-12 16:49 UTC (permalink / raw) To: Andrew Morton; +Cc: linux-kernel, Andi Kleen atomic_add_unless as inline. Remove system.h atomic.h circular dependency. I agree (with Andi Kleen) this typeof is not needed and more error prone. All the original atomic.h code that uses cmpxchg (which includes the atomic_add_unless) uses defines instead of inline functions, probably to circumvent a circular dependency between system.h and atomic.h on powerpc (which my patch addresses). Therefore, it makes sense to use inline functions that will provide type checking. Digging into the FRV architecture shows me that it is also affected by such a circular dependency. Here is the diff applying this against the rest of my atomic.h patches. It applies over the atomic.h standardization patches. Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> diff --git a/include/asm-alpha/atomic.h b/include/asm-alpha/atomic.h index 7b4fba8..f5cb7b8 100644 --- a/include/asm-alpha/atomic.h +++ b/include/asm-alpha/atomic.h @@ -2,6 +2,7 @@ #define _ALPHA_ATOMIC_H #include <asm/barrier.h> +#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -190,20 +191,21 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /** @@ -215,20 +217,21 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic64_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic64_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) diff --git a/include/asm-arm/atomic.h b/include/asm-arm/atomic.h index ea88aa6..b86b000 100644 --- a/include/asm-arm/atomic.h +++ b/include/asm-arm/atomic.h @@ -12,6 +12,7 @@ #define __ASM_ARM_ATOMIC_H #include <linux/compiler.h> +#include <asm/system.h> typedef struct { volatile int counter; } atomic_t; diff --git a/include/asm-arm26/atomic.h b/include/asm-arm26/atomic.h index 97e944f..d6dd423 100644 --- a/include/asm-arm26/atomic.h +++ b/include/asm-arm26/atomic.h @@ -20,7 +20,6 @@ #ifndef __ASM_ARM_ATOMIC_H #define __ASM_ARM_ATOMIC_H - #ifdef CONFIG_SMP #error SMP is NOT supported #endif diff --git a/include/asm-avr32/atomic.h b/include/asm-avr32/atomic.h diff --git a/include/asm-frv/atomic.h b/include/asm-frv/atomic.h index 066386a..d425d8d 100644 --- a/include/asm-frv/atomic.h +++ b/include/asm-frv/atomic.h @@ -16,6 +16,7 @@ #include <linux/types.h> #include <asm/spr-regs.h> +#include <asm/system.h> #ifdef CONFIG_SMP #error not SMP safe @@ -258,85 +259,23 @@ extern uint32_t __xchg_32(uint32_t i, volatile void *v); #define tas(ptr) (xchg((ptr), 1)) -/*****************************************************************************/ -/* - * compare and conditionally exchange value with memory - * - if (*ptr == test) then orig = *ptr; *ptr = test; - * - if (*ptr != test) then orig = *ptr; - */ -#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS - -#define cmpxchg(ptr, test, new) \ -({ \ - __typeof__(ptr) __xg_ptr = (ptr); \ - __typeof__(*(ptr)) __xg_orig, __xg_tmp; \ - __typeof__(*(ptr)) __xg_test = (test); \ - __typeof__(*(ptr)) __xg_new = (new); \ - \ - switch (sizeof(__xg_orig)) { \ - case 4: \ - asm volatile( \ - "0: \n" \ - " orcc gr0,gr0,gr0,icc3 \n" \ - " ckeq icc3,cc7 \n" \ - " ld.p %M0,%1 \n" \ - " orcr cc7,cc7,cc3 \n" \ - " sub%I4cc %1,%4,%2,icc0 \n" \ - " bne icc0,#0,1f \n" \ - " cst.p %3,%M0 ,cc3,#1 \n" \ - " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ - " beq icc3,#0,0b \n" \ - "1: \n" \ - : "+U"(*__xg_ptr), "=&r"(__xg_orig), "=&r"(__xg_tmp) \ - : "r"(__xg_new), "NPr"(__xg_test) \ - : "memory", "cc7", "cc3", "icc3", "icc0" \ - ); \ - break; \ - \ - default: \ - __xg_orig = 0; \ - asm volatile("break"); \ - break; \ - } \ - \ - __xg_orig; \ -}) - -#else - -extern uint32_t __cmpxchg_32(uint32_t *v, uint32_t test, uint32_t new); - -#define cmpxchg(ptr, test, new) \ -({ \ - __typeof__(ptr) __xg_ptr = (ptr); \ - __typeof__(*(ptr)) __xg_orig; \ - __typeof__(*(ptr)) __xg_test = (test); \ - __typeof__(*(ptr)) __xg_new = (new); \ - \ - switch (sizeof(__xg_orig)) { \ - case 4: __xg_orig = __cmpxchg_32(__xg_ptr, __xg_test, __xg_new); break; \ - default: \ - __xg_orig = 0; \ - asm volatile("break"); \ - break; \ - } \ - \ - __xg_orig; \ -}) - -#endif - #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) diff --git a/include/asm-frv/bitops.h b/include/asm-frv/bitops.h diff --git a/include/asm-frv/system.h b/include/asm-frv/system.h index 1166899..be303b3 100644 --- a/include/asm-frv/system.h +++ b/include/asm-frv/system.h @@ -13,7 +13,6 @@ #define _ASM_SYSTEM_H #include <linux/linkage.h> -#include <asm/atomic.h> struct thread_struct; @@ -197,4 +196,73 @@ extern void free_initmem(void); #define arch_align_stack(x) (x) +/*****************************************************************************/ +/* + * compare and conditionally exchange value with memory + * - if (*ptr == test) then orig = *ptr; *ptr = test; + * - if (*ptr != test) then orig = *ptr; + */ +#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS + +#define cmpxchg(ptr, test, new) \ +({ \ + __typeof__(ptr) __xg_ptr = (ptr); \ + __typeof__(*(ptr)) __xg_orig, __xg_tmp; \ + __typeof__(*(ptr)) __xg_test = (test); \ + __typeof__(*(ptr)) __xg_new = (new); \ + \ + switch (sizeof(__xg_orig)) { \ + case 4: \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ld.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " sub%I4cc %1,%4,%2,icc0 \n" \ + " bne icc0,#0,1f \n" \ + " cst.p %3,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + "1: \n" \ + : "+U"(*__xg_ptr), "=&r"(__xg_orig), "=&r"(__xg_tmp) \ + : "r"(__xg_new), "NPr"(__xg_test) \ + : "memory", "cc7", "cc3", "icc3", "icc0" \ + ); \ + break; \ + \ + default: \ + __xg_orig = 0; \ + asm volatile("break"); \ + break; \ + } \ + \ + __xg_orig; \ +}) + +#else + +extern uint32_t __cmpxchg_32(uint32_t *v, uint32_t test, uint32_t new); + +#define cmpxchg(ptr, test, new) \ +({ \ + __typeof__(ptr) __xg_ptr = (ptr); \ + __typeof__(*(ptr)) __xg_orig; \ + __typeof__(*(ptr)) __xg_test = (test); \ + __typeof__(*(ptr)) __xg_new = (new); \ + \ + switch (sizeof(__xg_orig)) { \ + case 4: __xg_orig = __cmpxchg_32(__xg_ptr, __xg_test, __xg_new); break; \ + default: \ + __xg_orig = 0; \ + asm volatile("break"); \ + break; \ + } \ + \ + __xg_orig; \ +}) + +#endif + + #endif /* _ASM_SYSTEM_H */ diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h index 069576f..8cc36d8 100644 --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -9,7 +9,6 @@ */ #include <asm/types.h> -#include <asm/system.h> /* * Suppport for atomic_long_t @@ -123,8 +122,12 @@ static inline long atomic_long_dec_return(atomic_long_t *l) return (long)atomic64_dec_return(v); } -#define atomic_long_add_unless(l, a, u) \ - atomic64_add_unless((atomic64_t *)(l), (a), (u)) +static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u) +{ + atomic64_t *v = (atomic64_t *)l; + + return (long)atomic64_add_unless(v, a, u); +} #define atomic_long_inc_not_zero(l) atomic64_inc_not_zero((atomic64_t *)(l)) @@ -236,8 +239,12 @@ static inline long atomic_long_dec_return(atomic_long_t *l) return (long)atomic_dec_return(v); } -#define atomic_long_add_unless(l, a, u) \ - atomic_add_unless((atomic_t *)(l), (a), (u)) +static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u) +{ + atomic_t *v = (atomic_t *)l; + + return (long)atomic_add_unless(v, a, u); +} #define atomic_long_inc_not_zero(l) atomic_inc_not_zero((atomic_t *)(l)) diff --git a/include/asm-i386/atomic.h b/include/asm-i386/atomic.h index 9bd1050..17e4376 100644 --- a/include/asm-i386/atomic.h +++ b/include/asm-i386/atomic.h @@ -219,20 +219,21 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc_return(v) (atomic_add_return(1,v)) diff --git a/include/asm-ia64/atomic.h b/include/asm-ia64/atomic.h index b16ad23..1fc3b83 100644 --- a/include/asm-ia64/atomic.h +++ b/include/asm-ia64/atomic.h @@ -15,6 +15,7 @@ #include <linux/types.h> #include <asm/intrinsics.h> +#include <asm/system.h> /* * On IA-64, counter must always be volatile to ensure that that the @@ -95,36 +96,38 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) (cmpxchg(&((v)->counter), old, new)) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__(v->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__(v->counter) c, old; \ - c = atomic64_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic64_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #define atomic_add_return(i,v) \ diff --git a/include/asm-m32r/atomic.h b/include/asm-m32r/atomic.h index 873e656..663ac88 100644 --- a/include/asm-m32r/atomic.h +++ b/include/asm-m32r/atomic.h @@ -253,14 +253,21 @@ static __inline__ int atomic_dec_return(atomic_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) static __inline__ void atomic_clear_mask(unsigned long mask, atomic_t *addr) diff --git a/include/asm-m68k/atomic.h b/include/asm-m68k/atomic.h index e2dfbde..a1b64a6 100644 --- a/include/asm-m68k/atomic.h +++ b/include/asm-m68k/atomic.h @@ -2,7 +2,7 @@ #define __ARCH_M68K_ATOMIC__ -#include <asm/system.h> /* local_irq_XXX() */ +#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -170,20 +170,21 @@ static inline void atomic_set_mask(unsigned long mask, unsigned long *v) __asm__ __volatile__("orl %1,%0" : "+m" (*v) : "id" (mask)); } -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /* Atomic operations are already serializing */ diff --git a/include/asm-m68knommu/atomic.h b/include/asm-m68knommu/atomic.h index 18a2e3b..b20e2cb 100644 --- a/include/asm-m68knommu/atomic.h +++ b/include/asm-m68knommu/atomic.h @@ -1,7 +1,7 @@ #ifndef __ARCH_M68KNOMMU_ATOMIC__ #define __ARCH_M68KNOMMU_ATOMIC__ -#include <asm/system.h> /* local_irq_XXX() */ +#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -131,14 +131,21 @@ static inline int atomic_sub_return(int i, atomic_t * v) #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_dec_return(v) atomic_sub_return(1,(v)) diff --git a/include/asm-mips/atomic.h b/include/asm-mips/atomic.h index 01a0303..fa9f742 100644 --- a/include/asm-mips/atomic.h +++ b/include/asm-mips/atomic.h @@ -18,6 +18,7 @@ #include <asm/barrier.h> #include <asm/cpu-features.h> #include <asm/war.h> +#include <asm/system.h> typedef struct { volatile int counter; } atomic_t; @@ -303,14 +304,20 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_dec_return(v) atomic_sub_return(1,(v)) @@ -664,14 +671,21 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #define atomic64_dec_return(v) atomic64_sub_return(1,(v)) diff --git a/include/asm-parisc/atomic.h b/include/asm-parisc/atomic.h index 09404ab..afb7cf5 100644 --- a/include/asm-parisc/atomic.h +++ b/include/asm-parisc/atomic.h @@ -6,6 +6,7 @@ #define _ASM_PARISC_ATOMIC_H_ #include <linux/types.h> +#include <asm/system.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -174,14 +175,21 @@ static __inline__ int atomic_read(const atomic_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_add(i,v) ((void)(__atomic_add_return( ((int)i),(v)))) @@ -283,14 +291,21 @@ atomic64_read(const atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic64_read(v); \ - while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) diff --git a/include/asm-powerpc/atomic.h b/include/asm-powerpc/atomic.h index 438a7fc..c44810b 100644 --- a/include/asm-powerpc/atomic.h +++ b/include/asm-powerpc/atomic.h @@ -11,6 +11,7 @@ typedef struct { volatile int counter; } atomic_t; #include <linux/compiler.h> #include <asm/synch.h> #include <asm/asm-compat.h> +#include <asm/system.h> #define ATOMIC_INIT(i) { (i) } diff --git a/include/asm-ppc/system.h b/include/asm-ppc/system.h index 7389435..56abe5e 100644 --- a/include/asm-ppc/system.h +++ b/include/asm-ppc/system.h @@ -6,7 +6,6 @@ #include <linux/kernel.h> -#include <asm/atomic.h> #include <asm/hw_irq.h> /* diff --git a/include/asm-sparc64/atomic.h b/include/asm-sparc64/atomic.h index c3feb3a..3fb4e1f 100644 --- a/include/asm-sparc64/atomic.h +++ b/include/asm-sparc64/atomic.h @@ -9,6 +9,7 @@ #define __ARCH_SPARC64_ATOMIC__ #include <linux/types.h> +#include <asm/system.h> typedef struct { volatile int counter; } atomic_t; typedef struct { volatile __s64 counter; } atomic64_t; @@ -73,40 +74,42 @@ extern int atomic64_sub_ret(int, atomic64_t *); #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - likely(c != (u)); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic64_cmpxchg(v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic64_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic64_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - likely(c != (u)); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) /* Atomic operations are already serializing */ diff --git a/include/asm-x86_64/atomic.h b/include/asm-x86_64/atomic.h index 80e4fdb..19e0c60 100644 --- a/include/asm-x86_64/atomic.h +++ b/include/asm-x86_64/atomic.h @@ -2,6 +2,7 @@ #define __ARCH_X86_64_ATOMIC__ #include <asm/alternative.h> +#include <asm/system.h> /* atomic_t should be 32 bit signed type */ @@ -403,20 +404,21 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /** @@ -428,20 +430,21 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic64_add_unless(v, a, u) \ -({ \ - __typeof__((v)->counter) c, old; \ - c = atomic64_read(v); \ - for (;;) { \ - if (unlikely(c == (u))) \ - break; \ - old = atomic64_cmpxchg((v), c, c + (a)); \ - if (likely(old == c)) \ - break; \ - c = old; \ - } \ - c != (u); \ -}) +static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long c, old; + c = atomic64_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic64_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) /* These are x86-specific, used by some header files */ diff --git a/include/asm-xtensa/atomic.h b/include/asm-xtensa/atomic.h index d73e0fe..9044303 100644 --- a/include/asm-xtensa/atomic.h +++ b/include/asm-xtensa/atomic.h @@ -234,14 +234,21 @@ static inline int atomic_sub_return(int i, atomic_t * v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#define atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c != (u); \ -}) +static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpxchg((v), c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -- Mathieu Desnoyers Computer Engineering Ph.D. Candidate, École Polytechnique de Montréal OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68 ^ permalink raw reply related [flat|nested] 14+ messages in thread
end of thread, other threads:[~2007-02-12 16:50 UTC | newest] Thread overview: 14+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2007-02-11 19:17 [PATCH 00/10] atomic.h : standardizing atomic primitives Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 01/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to alpha Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 02/10] atomic.h : Complete atomic_long operations in asm-generic Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 03/10] atomic.h : i386 type safety fix Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 04/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to ia64 Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 05/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to mips Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 06/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to parisc Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 07/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to powerpc Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 08/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to sparc64 Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 09/10] atomic.h : Add atomic64_xchg to s390 Mathieu Desnoyers 2007-02-11 19:17 ` [PATCH 10/10] atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64 Mathieu Desnoyers 2007-02-12 11:35 ` Andi Kleen 2007-02-12 16:43 ` Mathieu Desnoyers 2007-02-12 16:49 ` [PATCH] atomic.h : atomic_add_unless as inline. Remove system.h atomic.h circular dependency Mathieu Desnoyers
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).