From: tip-bot for Dmitry Vyukov <tipbot@zytor.com>
To: linux-tip-commits@vger.kernel.org
Cc: mark.rutland@arm.com, aryabinin@virtuozzo.com,
linux-kernel@vger.kernel.org, dvyukov@google.com,
mingo@kernel.org, torvalds@linux-foundation.org,
peterz@infradead.org, hpa@zytor.com, tglx@linutronix.de,
akpm@linux-foundation.org
Subject: [tip:locking/core] locking/atomic/x86: Un-macro-ify atomic ops implementation
Date: Thu, 22 Jun 2017 04:04:21 -0700 [thread overview]
Message-ID: <tip-0f2376eb0ff8851124c876eb81806d7ec1b421d1@git.kernel.org> (raw)
In-Reply-To: <a35b983dd3be937a3cf63c4e2db487de2cdc7b8f.1497690003.git.dvyukov@google.com>
Commit-ID: 0f2376eb0ff8851124c876eb81806d7ec1b421d1
Gitweb: http://git.kernel.org/tip/0f2376eb0ff8851124c876eb81806d7ec1b421d1
Author: Dmitry Vyukov <dvyukov@google.com>
AuthorDate: Sat, 17 Jun 2017 11:15:27 +0200
Committer: Ingo Molnar <mingo@kernel.org>
CommitDate: Thu, 22 Jun 2017 10:19:56 +0200
locking/atomic/x86: Un-macro-ify atomic ops implementation
CPP turns perfectly readable code into a much harder to read syntactic soup.
Ingo suggested to write them out as-is in C and ignore the higher linecount.
Do this.
(As a side effect, plain C functions will be easier to KASAN-instrument as well.)
Suggested-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Dmitry Vyukov <dvyukov@google.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: kasan-dev@googlegroups.com
Cc: will.deacon@arm.com
Link: http://lkml.kernel.org/r/a35b983dd3be937a3cf63c4e2db487de2cdc7b8f.1497690003.git.dvyukov@google.com
[ Beautified the C code some more and twiddled the changelog
to mention the linecount increase and the KASAN benefit. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
arch/x86/include/asm/atomic.h | 69 +++++++++++++++++++++-----------
arch/x86/include/asm/atomic64_32.h | 81 +++++++++++++++++++++++++++-----------
arch/x86/include/asm/atomic64_64.h | 67 ++++++++++++++++++++-----------
3 files changed, 147 insertions(+), 70 deletions(-)
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 33380b8..0874ebd 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -197,35 +197,56 @@ static inline int atomic_xchg(atomic_t *v, int new)
return xchg(&v->counter, new);
}
-#define ATOMIC_OP(op) \
-static inline void atomic_##op(int i, atomic_t *v) \
-{ \
- asm volatile(LOCK_PREFIX #op"l %1,%0" \
- : "+m" (v->counter) \
- : "ir" (i) \
- : "memory"); \
+static inline void atomic_and(int i, atomic_t *v)
+{
+ asm volatile(LOCK_PREFIX "andl %1,%0"
+ : "+m" (v->counter)
+ : "ir" (i)
+ : "memory");
+}
+
+static inline int atomic_fetch_and(int i, atomic_t *v)
+{
+ int val = atomic_read(v);
+
+ do { } while (!atomic_try_cmpxchg(v, &val, val & i));
+
+ return val;
}
-#define ATOMIC_FETCH_OP(op, c_op) \
-static inline int atomic_fetch_##op(int i, atomic_t *v) \
-{ \
- int val = atomic_read(v); \
- do { \
- } while (!atomic_try_cmpxchg(v, &val, val c_op i)); \
- return val; \
+static inline void atomic_or(int i, atomic_t *v)
+{
+ asm volatile(LOCK_PREFIX "orl %1,%0"
+ : "+m" (v->counter)
+ : "ir" (i)
+ : "memory");
}
-#define ATOMIC_OPS(op, c_op) \
- ATOMIC_OP(op) \
- ATOMIC_FETCH_OP(op, c_op)
+static inline int atomic_fetch_or(int i, atomic_t *v)
+{
+ int val = atomic_read(v);
-ATOMIC_OPS(and, &)
-ATOMIC_OPS(or , |)
-ATOMIC_OPS(xor, ^)
+ do { } while (!atomic_try_cmpxchg(v, &val, val | i));
-#undef ATOMIC_OPS
-#undef ATOMIC_FETCH_OP
-#undef ATOMIC_OP
+ return val;
+}
+
+static inline void atomic_xor(int i, atomic_t *v)
+{
+ asm volatile(LOCK_PREFIX "xorl %1,%0"
+ : "+m" (v->counter)
+ : "ir" (i)
+ : "memory");
+}
+
+static inline int atomic_fetch_xor(int i, atomic_t *v)
+{
+ int val = atomic_read(v);
+
+ do { } while (!atomic_try_cmpxchg(v, &val, val ^ i));
+
+ return val;
+}
/**
* __atomic_add_unless - add unless the number is already a given value
@@ -239,10 +260,12 @@ ATOMIC_OPS(xor, ^)
static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
{
int c = atomic_read(v);
+
do {
if (unlikely(c == u))
break;
} while (!atomic_try_cmpxchg(v, &c, c + a));
+
return c;
}
diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h
index 71d7705..9e206f3 100644
--- a/arch/x86/include/asm/atomic64_32.h
+++ b/arch/x86/include/asm/atomic64_32.h
@@ -312,37 +312,70 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v)
#undef alternative_atomic64
#undef __alternative_atomic64
-#define ATOMIC64_OP(op, c_op) \
-static inline void atomic64_##op(long long i, atomic64_t *v) \
-{ \
- long long old, c = 0; \
- while ((old = atomic64_cmpxchg(v, c, c c_op i)) != c) \
- c = old; \
+static inline void atomic64_and(long long i, atomic64_t *v)
+{
+ long long old, c = 0;
+
+ while ((old = atomic64_cmpxchg(v, c, c & i)) != c)
+ c = old;
}
-#define ATOMIC64_FETCH_OP(op, c_op) \
-static inline long long atomic64_fetch_##op(long long i, atomic64_t *v) \
-{ \
- long long old, c = 0; \
- while ((old = atomic64_cmpxchg(v, c, c c_op i)) != c) \
- c = old; \
- return old; \
+static inline long long atomic64_fetch_and(long long i, atomic64_t *v)
+{
+ long long old, c = 0;
+
+ while ((old = atomic64_cmpxchg(v, c, c & i)) != c)
+ c = old;
+
+ return old;
}
-ATOMIC64_FETCH_OP(add, +)
+static inline void atomic64_or(long long i, atomic64_t *v)
+{
+ long long old, c = 0;
-#define atomic64_fetch_sub(i, v) atomic64_fetch_add(-(i), (v))
+ while ((old = atomic64_cmpxchg(v, c, c | i)) != c)
+ c = old;
+}
+
+static inline long long atomic64_fetch_or(long long i, atomic64_t *v)
+{
+ long long old, c = 0;
+
+ while ((old = atomic64_cmpxchg(v, c, c | i)) != c)
+ c = old;
+
+ return old;
+}
-#define ATOMIC64_OPS(op, c_op) \
- ATOMIC64_OP(op, c_op) \
- ATOMIC64_FETCH_OP(op, c_op)
+static inline void atomic64_xor(long long i, atomic64_t *v)
+{
+ long long old, c = 0;
+
+ while ((old = atomic64_cmpxchg(v, c, c ^ i)) != c)
+ c = old;
+}
-ATOMIC64_OPS(and, &)
-ATOMIC64_OPS(or, |)
-ATOMIC64_OPS(xor, ^)
+static inline long long atomic64_fetch_xor(long long i, atomic64_t *v)
+{
+ long long old, c = 0;
+
+ while ((old = atomic64_cmpxchg(v, c, c ^ i)) != c)
+ c = old;
+
+ return old;
+}
-#undef ATOMIC64_OPS
-#undef ATOMIC64_FETCH_OP
-#undef ATOMIC64_OP
+static inline long long atomic64_fetch_add(long long i, atomic64_t *v)
+{
+ long long old, c = 0;
+
+ while ((old = atomic64_cmpxchg(v, c, c + i)) != c)
+ c = old;
+
+ return old;
+}
+
+#define atomic64_fetch_sub(i, v) atomic64_fetch_add(-(i), (v))
#endif /* _ASM_X86_ATOMIC64_32_H */
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index 6189a43..8db8879a 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -226,34 +226,55 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)
return dec;
}
-#define ATOMIC64_OP(op) \
-static inline void atomic64_##op(long i, atomic64_t *v) \
-{ \
- asm volatile(LOCK_PREFIX #op"q %1,%0" \
- : "+m" (v->counter) \
- : "er" (i) \
- : "memory"); \
+static inline void atomic64_and(long i, atomic64_t *v)
+{
+ asm volatile(LOCK_PREFIX "andq %1,%0"
+ : "+m" (v->counter)
+ : "er" (i)
+ : "memory");
}
-#define ATOMIC64_FETCH_OP(op, c_op) \
-static inline long atomic64_fetch_##op(long i, atomic64_t *v) \
-{ \
- long val = atomic64_read(v); \
- do { \
- } while (!atomic64_try_cmpxchg(v, &val, val c_op i)); \
- return val; \
+static inline long atomic64_fetch_and(long i, atomic64_t *v)
+{
+ long val = atomic64_read(v);
+
+ do {
+ } while (!atomic64_try_cmpxchg(v, &val, val & i));
+ return val;
}
-#define ATOMIC64_OPS(op, c_op) \
- ATOMIC64_OP(op) \
- ATOMIC64_FETCH_OP(op, c_op)
+static inline void atomic64_or(long i, atomic64_t *v)
+{
+ asm volatile(LOCK_PREFIX "orq %1,%0"
+ : "+m" (v->counter)
+ : "er" (i)
+ : "memory");
+}
-ATOMIC64_OPS(and, &)
-ATOMIC64_OPS(or, |)
-ATOMIC64_OPS(xor, ^)
+static inline long atomic64_fetch_or(long i, atomic64_t *v)
+{
+ long val = atomic64_read(v);
-#undef ATOMIC64_OPS
-#undef ATOMIC64_FETCH_OP
-#undef ATOMIC64_OP
+ do {
+ } while (!atomic64_try_cmpxchg(v, &val, val | i));
+ return val;
+}
+
+static inline void atomic64_xor(long i, atomic64_t *v)
+{
+ asm volatile(LOCK_PREFIX "xorq %1,%0"
+ : "+m" (v->counter)
+ : "er" (i)
+ : "memory");
+}
+
+static inline long atomic64_fetch_xor(long i, atomic64_t *v)
+{
+ long val = atomic64_read(v);
+
+ do {
+ } while (!atomic64_try_cmpxchg(v, &val, val ^ i));
+ return val;
+}
#endif /* _ASM_X86_ATOMIC64_64_H */
next prev parent reply other threads:[~2017-06-22 11:09 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-06-17 9:15 [PATCH v4 0/7] x86, kasan: add KASAN checks to atomic operations Dmitry Vyukov
2017-06-17 9:15 ` [PATCH v4 1/7] x86: un-macro-ify atomic ops implementation Dmitry Vyukov
2017-06-22 11:04 ` tip-bot for Dmitry Vyukov [this message]
2017-07-25 13:54 ` [tip:locking/core] locking/atomic/x86: Un-macro-ify " tip-bot for Dmitry Vyukov
2017-06-17 9:15 ` [PATCH v4 2/7] x86: use s64* for old arg of atomic64_try_cmpxchg() Dmitry Vyukov
2017-06-22 11:04 ` [tip:locking/core] locking/atomic/x86: Use 's64 *' for 'old' argument " tip-bot for Dmitry Vyukov
2017-07-25 13:55 ` tip-bot for Dmitry Vyukov
2017-06-17 9:15 ` [PATCH v4 3/7] asm-generic: add atomic-instrumented.h Dmitry Vyukov
2017-06-19 10:50 ` Mark Rutland
2017-06-22 11:05 ` [tip:locking/core] locking/atomic: Add asm-generic/atomic-instrumented.h tip-bot for Dmitry Vyukov
2018-03-12 12:23 ` [tip:locking/core] locking/atomic, asm-generic: " tip-bot for Dmitry Vyukov
2017-06-17 9:15 ` [PATCH v4 4/7] x86: switch atomic.h to use atomic-instrumented.h Dmitry Vyukov
2017-06-17 9:15 ` [PATCH v4 5/7] kasan: allow kasan_check_read/write() to accept pointers to volatiles Dmitry Vyukov
2017-06-19 10:50 ` Mark Rutland
2017-06-19 13:11 ` Dmitry Vyukov
2017-06-22 8:25 ` Ingo Molnar
2017-06-22 14:15 ` Dmitry Vyukov
2017-06-17 9:15 ` [PATCH v4 6/7] asm-generic: add KASAN instrumentation to atomic operations Dmitry Vyukov
2017-06-19 10:51 ` Mark Rutland
2017-06-17 9:15 ` [PATCH v4 7/7] asm-generic, x86: add comments for atomic instrumentation Dmitry Vyukov
2017-06-19 10:54 ` Mark Rutland
2018-01-29 17:26 [PATCH v6 0/4] x86, kasan: add KASAN checks to atomic operations Dmitry Vyukov
2018-01-29 17:26 ` [PATCH v6 1/4] locking/atomic: Add asm-generic/atomic-instrumented.h Dmitry Vyukov
2018-01-29 17:26 ` [PATCH v6 2/4] x86: switch atomic.h to use atomic-instrumented.h Dmitry Vyukov
2018-03-12 12:24 ` [tip:locking/core] locking/atomic/x86: Switch " tip-bot for Dmitry Vyukov
2018-01-29 17:26 ` [PATCH v6 3/4] asm-generic: add KASAN instrumentation to atomic operations Dmitry Vyukov
2018-03-12 12:24 ` [tip:locking/core] locking/atomic, asm-generic: Add " tip-bot for Dmitry Vyukov
2018-01-29 17:26 ` [PATCH v6 4/4] asm-generic, x86: add comments for atomic instrumentation Dmitry Vyukov
2018-03-12 12:25 ` [tip:locking/core] locking/atomic, asm-generic, x86: Add " tip-bot for Dmitry Vyukov
2018-01-30 9:23 ` [PATCH v6 0/4] x86, kasan: add KASAN checks to atomic operations Dmitry Vyukov
2018-01-30 9:27 ` Dmitry Vyukov
2018-01-30 15:36 ` Will Deacon
2018-01-31 7:28 ` Ingo Molnar
2018-01-31 8:53 ` Dmitry Vyukov
2018-01-31 16:17 ` Will Deacon
2018-02-07 14:17 ` Dmitry Vyukov
2018-02-20 10:40 ` Dmitry Vyukov
2018-02-26 12:52 ` Dmitry Vyukov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=tip-0f2376eb0ff8851124c876eb81806d7ec1b421d1@git.kernel.org \
--to=tipbot@zytor.com \
--cc=akpm@linux-foundation.org \
--cc=aryabinin@virtuozzo.com \
--cc=dvyukov@google.com \
--cc=hpa@zytor.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-tip-commits@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
--cc=tglx@linutronix.de \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).