From: Mark Rutland <mark.rutland@arm.com>
To: Peter Zijlstra <peterz@infradead.org>
Cc: torvalds@linux-foundation.org, corbet@lwn.net, will@kernel.org,
boqun.feng@gmail.com, catalin.marinas@arm.com, dennis@kernel.org,
tj@kernel.org, cl@linux.com, hca@linux.ibm.com,
gor@linux.ibm.com, agordeev@linux.ibm.com,
borntraeger@linux.ibm.com, svens@linux.ibm.com,
tglx@linutronix.de, mingo@redhat.com, bp@alien8.de,
dave.hansen@linux.intel.com, x86@kernel.org, hpa@zytor.com,
joro@8bytes.org, suravee.suthikulpanit@amd.com,
robin.murphy@arm.com, dwmw2@infradead.org,
baolu.lu@linux.intel.com, Arnd Bergmann <arnd@arndb.de>,
Herbert Xu <herbert@gondor.apana.org.au>,
davem@davemloft.net, penberg@kernel.org, rientjes@google.com,
iamjoonsoo.kim@lge.com, Andrew Morton <akpm@linux-foundation.org>,
vbabka@suse.cz, roman.gushchin@linux.dev, 42.hyeyoo@gmail.com,
linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org,
linux-mm@kvack.org, linux-s390@vger.kernel.org,
iommu@lists.linux.dev, linux-arch@vger.kernel.org,
linux-crypto@vger.kernel.org
Subject: Re: [PATCH v2 04/10] instrumentation: Wire up cmpxchg128()
Date: Fri, 3 Feb 2023 16:55:20 +0000 [thread overview]
Message-ID: <Y908eEiQI5plfGWc@FVFF77S0Q05N> (raw)
In-Reply-To: <20230202152655.433050731@infradead.org>
On Thu, Feb 02, 2023 at 03:50:34PM +0100, Peter Zijlstra wrote:
> Wire up the cmpxchg128 familty in the atomic wrappery scripts.
s/familty/family/ (and s/wrappery/wrapper/ ?)
>
> These provide the generic cmpxchg128 family of functions from the
> arch_ prefixed version, adding explicit instrumentation where needed.
>
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Mark.
> ---
> include/linux/atomic/atomic-arch-fallback.h | 95 +++++++++++++++++++++++++++-
> include/linux/atomic/atomic-instrumented.h | 77 ++++++++++++++++++++++
> scripts/atomic/gen-atomic-fallback.sh | 4 -
> scripts/atomic/gen-atomic-instrumented.sh | 4 -
> 4 files changed, 174 insertions(+), 6 deletions(-)
>
> --- a/include/linux/atomic/atomic-arch-fallback.h
> +++ b/include/linux/atomic/atomic-arch-fallback.h
> @@ -77,6 +77,29 @@
>
> #endif /* arch_cmpxchg64_relaxed */
>
> +#ifndef arch_cmpxchg128_relaxed
> +#define arch_cmpxchg128_acquire arch_cmpxchg128
> +#define arch_cmpxchg128_release arch_cmpxchg128
> +#define arch_cmpxchg128_relaxed arch_cmpxchg128
> +#else /* arch_cmpxchg128_relaxed */
> +
> +#ifndef arch_cmpxchg128_acquire
> +#define arch_cmpxchg128_acquire(...) \
> + __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
> +#endif
> +
> +#ifndef arch_cmpxchg128_release
> +#define arch_cmpxchg128_release(...) \
> + __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
> +#endif
> +
> +#ifndef arch_cmpxchg128
> +#define arch_cmpxchg128(...) \
> + __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
> +#endif
> +
> +#endif /* arch_cmpxchg128_relaxed */
> +
> #ifndef arch_try_cmpxchg_relaxed
> #ifdef arch_try_cmpxchg
> #define arch_try_cmpxchg_acquire arch_try_cmpxchg
> @@ -217,6 +240,76 @@
>
> #endif /* arch_try_cmpxchg64_relaxed */
>
> +#ifndef arch_try_cmpxchg128_relaxed
> +#ifdef arch_try_cmpxchg128
> +#define arch_try_cmpxchg128_acquire arch_try_cmpxchg128
> +#define arch_try_cmpxchg128_release arch_try_cmpxchg128
> +#define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128
> +#endif /* arch_try_cmpxchg128 */
> +
> +#ifndef arch_try_cmpxchg128
> +#define arch_try_cmpxchg128(_ptr, _oldp, _new) \
> +({ \
> + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
> + ___r = arch_cmpxchg128((_ptr), ___o, (_new)); \
> + if (unlikely(___r != ___o)) \
> + *___op = ___r; \
> + likely(___r == ___o); \
> +})
> +#endif /* arch_try_cmpxchg128 */
> +
> +#ifndef arch_try_cmpxchg128_acquire
> +#define arch_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
> +({ \
> + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
> + ___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \
> + if (unlikely(___r != ___o)) \
> + *___op = ___r; \
> + likely(___r == ___o); \
> +})
> +#endif /* arch_try_cmpxchg128_acquire */
> +
> +#ifndef arch_try_cmpxchg128_release
> +#define arch_try_cmpxchg128_release(_ptr, _oldp, _new) \
> +({ \
> + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
> + ___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \
> + if (unlikely(___r != ___o)) \
> + *___op = ___r; \
> + likely(___r == ___o); \
> +})
> +#endif /* arch_try_cmpxchg128_release */
> +
> +#ifndef arch_try_cmpxchg128_relaxed
> +#define arch_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
> +({ \
> + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
> + ___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
> + if (unlikely(___r != ___o)) \
> + *___op = ___r; \
> + likely(___r == ___o); \
> +})
> +#endif /* arch_try_cmpxchg128_relaxed */
> +
> +#else /* arch_try_cmpxchg128_relaxed */
> +
> +#ifndef arch_try_cmpxchg128_acquire
> +#define arch_try_cmpxchg128_acquire(...) \
> + __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
> +#endif
> +
> +#ifndef arch_try_cmpxchg128_release
> +#define arch_try_cmpxchg128_release(...) \
> + __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
> +#endif
> +
> +#ifndef arch_try_cmpxchg128
> +#define arch_try_cmpxchg128(...) \
> + __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
> +#endif
> +
> +#endif /* arch_try_cmpxchg128_relaxed */
> +
> #ifndef arch_atomic_read_acquire
> static __always_inline int
> arch_atomic_read_acquire(const atomic_t *v)
> @@ -2456,4 +2549,4 @@ arch_atomic64_dec_if_positive(atomic64_t
> #endif
>
> #endif /* _LINUX_ATOMIC_FALLBACK_H */
> -// b5e87bdd5ede61470c29f7a7e4de781af3770f09
> +// 46357a526de89c762d30fb238f35a7d5950a670b
> --- a/include/linux/atomic/atomic-instrumented.h
> +++ b/include/linux/atomic/atomic-instrumented.h
> @@ -1968,6 +1968,36 @@ atomic_long_dec_if_positive(atomic_long_
> arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
> })
>
> +#define cmpxchg128(ptr, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + kcsan_mb(); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + arch_cmpxchg128(__ai_ptr, __VA_ARGS__); \
> +})
> +
> +#define cmpxchg128_acquire(ptr, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + arch_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
> +})
> +
> +#define cmpxchg128_release(ptr, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + kcsan_release(); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + arch_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
> +})
> +
> +#define cmpxchg128_relaxed(ptr, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + arch_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
> +})
> +
> #define try_cmpxchg(ptr, oldp, ...) \
> ({ \
> typeof(ptr) __ai_ptr = (ptr); \
> @@ -2044,6 +2074,44 @@ atomic_long_dec_if_positive(atomic_long_
> arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
> })
>
> +#define try_cmpxchg128(ptr, oldp, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + typeof(oldp) __ai_oldp = (oldp); \
> + kcsan_mb(); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
> + arch_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
> +})
> +
> +#define try_cmpxchg128_acquire(ptr, oldp, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + typeof(oldp) __ai_oldp = (oldp); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
> + arch_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
> +})
> +
> +#define try_cmpxchg128_release(ptr, oldp, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + typeof(oldp) __ai_oldp = (oldp); \
> + kcsan_release(); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
> + arch_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
> +})
> +
> +#define try_cmpxchg128_relaxed(ptr, oldp, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + typeof(oldp) __ai_oldp = (oldp); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
> + arch_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
> +})
> +
> #define cmpxchg_local(ptr, ...) \
> ({ \
> typeof(ptr) __ai_ptr = (ptr); \
> @@ -2058,6 +2126,13 @@ atomic_long_dec_if_positive(atomic_long_
> arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
> })
>
> +#define cmpxchg128_local(ptr, ...) \
> +({ \
> + typeof(ptr) __ai_ptr = (ptr); \
> + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
> + arch_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
> +})
> +
> #define sync_cmpxchg(ptr, ...) \
> ({ \
> typeof(ptr) __ai_ptr = (ptr); \
> @@ -2083,4 +2158,4 @@ atomic_long_dec_if_positive(atomic_long_
> })
>
> #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
> -// 764f741eb77a7ad565dc8d99ce2837d5542e8aee
> +// 27320c1ec2bf2878ecb9df3ea4816a7bc0c57a52
> --- a/scripts/atomic/gen-atomic-fallback.sh
> +++ b/scripts/atomic/gen-atomic-fallback.sh
> @@ -217,11 +217,11 @@ cat << EOF
>
> EOF
>
> -for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do
> +for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64" "arch_cmpxchg128"; do
> gen_xchg_fallbacks "${xchg}"
> done
>
> -for cmpxchg in "cmpxchg" "cmpxchg64"; do
> +for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
> gen_try_cmpxchg_fallbacks "${cmpxchg}"
> done
>
> --- a/scripts/atomic/gen-atomic-instrumented.sh
> +++ b/scripts/atomic/gen-atomic-instrumented.sh
> @@ -166,14 +166,14 @@ grep '^[a-z]' "$1" | while read name met
> done
>
>
> -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do
> +for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do
> for order in "" "_acquire" "_release" "_relaxed"; do
> gen_xchg "${xchg}" "${order}" ""
> printf "\n"
> done
> done
>
> -for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do
> +for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg"; do
> gen_xchg "${xchg}" "" ""
> printf "\n"
> done
>
>
next prev parent reply other threads:[~2023-02-03 16:55 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-02-02 14:50 [PATCH v2 00/10] Introduce cmpxchg128() -- aka. the demise of cmpxchg_double() Peter Zijlstra
2023-02-02 14:50 ` [PATCH v2 01/10] cyrpto/b128ops: Remove struct u128 Peter Zijlstra
2023-02-02 20:21 ` H. Peter Anvin
2023-02-03 1:06 ` Herbert Xu
2023-02-02 14:50 ` [PATCH v2 02/10] types: Introduce [us]128 Peter Zijlstra
2023-02-03 3:37 ` Herbert Xu
2023-02-02 14:50 ` [PATCH v2 03/10] arch: Introduce arch_{,try_}_cmpxchg128{,_local}() Peter Zijlstra
2023-02-02 17:04 ` Heiko Carstens
2023-02-03 16:52 ` Mark Rutland
2023-02-02 14:50 ` [PATCH v2 04/10] instrumentation: Wire up cmpxchg128() Peter Zijlstra
2023-02-03 16:55 ` Mark Rutland [this message]
2023-02-02 14:50 ` [PATCH v2 05/10] percpu: Wire up cmpxchg128 Peter Zijlstra
2023-02-02 17:05 ` Heiko Carstens
2023-02-03 17:02 ` Mark Rutland
2023-02-03 17:25 ` Arnd Bergmann
2023-02-06 11:24 ` Peter Zijlstra
2023-02-06 12:14 ` Peter Zijlstra
2023-02-06 12:48 ` Peter Zijlstra
2023-02-06 13:20 ` Arnd Bergmann
2023-02-06 12:19 ` Arnd Bergmann
2023-02-02 14:50 ` [PATCH v2 06/10] x86,amd_iommu: Replace cmpxchg_double() Peter Zijlstra
2023-02-02 14:50 ` [PATCH v2 07/10] x86,intel_iommu: " Peter Zijlstra
2023-02-03 2:51 ` Baolu Lu
2023-02-02 14:50 ` [PATCH v2 08/10] slub: " Peter Zijlstra
2023-02-08 13:31 ` Hyeonggon Yoo
2023-02-02 14:50 ` [PATCH v2 09/10] arch: Remove cmpxchg_double Peter Zijlstra
2023-02-08 13:44 ` Hyeonggon Yoo
2023-02-02 14:50 ` [PATCH v2 10/10] s390/cpum_sf: Convert to cmpxchg128() Peter Zijlstra
2023-02-02 17:05 ` Heiko Carstens
2023-02-02 19:39 ` [PATCH v2 00/10] Introduce cmpxchg128() -- aka. the demise of cmpxchg_double() Linus Torvalds
2023-02-02 22:45 ` David Laight
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=Y908eEiQI5plfGWc@FVFF77S0Q05N \
--to=mark.rutland@arm.com \
--cc=42.hyeyoo@gmail.com \
--cc=agordeev@linux.ibm.com \
--cc=akpm@linux-foundation.org \
--cc=arnd@arndb.de \
--cc=baolu.lu@linux.intel.com \
--cc=boqun.feng@gmail.com \
--cc=borntraeger@linux.ibm.com \
--cc=bp@alien8.de \
--cc=catalin.marinas@arm.com \
--cc=cl@linux.com \
--cc=corbet@lwn.net \
--cc=dave.hansen@linux.intel.com \
--cc=davem@davemloft.net \
--cc=dennis@kernel.org \
--cc=dwmw2@infradead.org \
--cc=gor@linux.ibm.com \
--cc=hca@linux.ibm.com \
--cc=herbert@gondor.apana.org.au \
--cc=hpa@zytor.com \
--cc=iamjoonsoo.kim@lge.com \
--cc=iommu@lists.linux.dev \
--cc=joro@8bytes.org \
--cc=linux-arch@vger.kernel.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-s390@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=penberg@kernel.org \
--cc=peterz@infradead.org \
--cc=rientjes@google.com \
--cc=robin.murphy@arm.com \
--cc=roman.gushchin@linux.dev \
--cc=suravee.suthikulpanit@amd.com \
--cc=svens@linux.ibm.com \
--cc=tglx@linutronix.de \
--cc=tj@kernel.org \
--cc=torvalds@linux-foundation.org \
--cc=vbabka@suse.cz \
--cc=will@kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).