* [PATCH v2] locking: Add __lockfunc to slow path functions
@ 2022-08-10 22:03 Namhyung Kim
2022-08-11 18:19 ` Waiman Long
2022-08-22 9:22 ` [tip: locking/core] " tip-bot2 for Namhyung Kim
0 siblings, 2 replies; 3+ messages in thread
From: Namhyung Kim @ 2022-08-10 22:03 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon; +Cc: Waiman Long, Boqun Feng, LKML
So that we can skip the functions in the perf lock contention and other
places like /proc/PID/wchan.
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
* annotate pv qspinlock functions too
arch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------
kernel/locking/qrwlock.c | 4 ++--
kernel/locking/qspinlock.c | 2 +-
kernel/locking/qspinlock_paravirt.h | 4 ++--
4 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h
index 892fd8c3a6f7..60ece592b220 100644
--- a/arch/x86/include/asm/qspinlock_paravirt.h
+++ b/arch/x86/include/asm/qspinlock_paravirt.h
@@ -12,7 +12,7 @@
*/
#ifdef CONFIG_64BIT
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
#define __pv_queued_spin_unlock __pv_queued_spin_unlock
#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
/*
* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
* which combines the registers saving trunk and the body of the following
- * C code:
+ * C code. Note that it puts the code in the .spinlock.text section which
+ * is equivalent to adding __lockfunc in the C code:
*
- * void __pv_queued_spin_unlock(struct qspinlock *lock)
+ * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
* {
* u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
*
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
* rsi = lockval (second argument)
* rdx = internal variable (set to 0)
*/
-asm (".pushsection .text;"
+asm (".pushsection .spinlock.text;"
".globl " PV_UNLOCK ";"
".type " PV_UNLOCK ", @function;"
".align 4,0x90;"
@@ -65,8 +66,8 @@ asm (".pushsection .text;"
#else /* CONFIG_64BIT */
-extern void __pv_queued_spin_unlock(struct qspinlock *lock);
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
+extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
#endif /* CONFIG_64BIT */
#endif
diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c
index 2e1600906c9f..d2ef312a8611 100644
--- a/kernel/locking/qrwlock.c
+++ b/kernel/locking/qrwlock.c
@@ -18,7 +18,7 @@
* queued_read_lock_slowpath - acquire read lock of a queued rwlock
* @lock: Pointer to queued rwlock structure
*/
-void queued_read_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
{
/*
* Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
* queued_write_lock_slowpath - acquire write lock of a queued rwlock
* @lock : Pointer to queued rwlock structure
*/
-void queued_write_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
{
int cnts;
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 65a9a10caa6f..2b23378775fe 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
* queue : ^--' :
*/
-void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
struct mcs_spinlock *prev, *next, *node;
u32 old, tail;
diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
index e84d21aa0722..6afc249ce697 100644
--- a/kernel/locking/qspinlock_paravirt.h
+++ b/kernel/locking/qspinlock_paravirt.h
@@ -489,7 +489,7 @@ pv_wait_head_or_lock(struct qspinlock *lock, struct mcs_spinlock *node)
* PV versions of the unlock fastpath and slowpath functions to be used
* instead of queued_spin_unlock().
*/
-__visible void
+__visible __lockfunc void
__pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
{
struct pv_node *node;
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
#include <asm/qspinlock_paravirt.h>
#ifndef __pv_queued_spin_unlock
-__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
+__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
{
u8 locked;
--
2.37.1.559.g78731f0fdb-goog
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH v2] locking: Add __lockfunc to slow path functions
2022-08-10 22:03 [PATCH v2] locking: Add __lockfunc to slow path functions Namhyung Kim
@ 2022-08-11 18:19 ` Waiman Long
2022-08-22 9:22 ` [tip: locking/core] " tip-bot2 for Namhyung Kim
1 sibling, 0 replies; 3+ messages in thread
From: Waiman Long @ 2022-08-11 18:19 UTC (permalink / raw)
To: Namhyung Kim, Peter Zijlstra, Ingo Molnar, Will Deacon; +Cc: Boqun Feng, LKML
On 8/10/22 18:03, Namhyung Kim wrote:
> So that we can skip the functions in the perf lock contention and other
> places like /proc/PID/wchan.
>
> Signed-off-by: Namhyung Kim <namhyung@kernel.org>
> ---
> * annotate pv qspinlock functions too
>
> arch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------
> kernel/locking/qrwlock.c | 4 ++--
> kernel/locking/qspinlock.c | 2 +-
> kernel/locking/qspinlock_paravirt.h | 4 ++--
> 4 files changed, 12 insertions(+), 11 deletions(-)
>
> diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h
> index 892fd8c3a6f7..60ece592b220 100644
> --- a/arch/x86/include/asm/qspinlock_paravirt.h
> +++ b/arch/x86/include/asm/qspinlock_paravirt.h
> @@ -12,7 +12,7 @@
> */
> #ifdef CONFIG_64BIT
>
> -PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
> +__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
> #define __pv_queued_spin_unlock __pv_queued_spin_unlock
> #define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
> #define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
> @@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
> /*
> * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
> * which combines the registers saving trunk and the body of the following
> - * C code:
> + * C code. Note that it puts the code in the .spinlock.text section which
> + * is equivalent to adding __lockfunc in the C code:
> *
> - * void __pv_queued_spin_unlock(struct qspinlock *lock)
> + * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
> * {
> * u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
> *
> @@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
> * rsi = lockval (second argument)
> * rdx = internal variable (set to 0)
> */
> -asm (".pushsection .text;"
> +asm (".pushsection .spinlock.text;"
> ".globl " PV_UNLOCK ";"
> ".type " PV_UNLOCK ", @function;"
> ".align 4,0x90;"
> @@ -65,8 +66,8 @@ asm (".pushsection .text;"
>
> #else /* CONFIG_64BIT */
>
> -extern void __pv_queued_spin_unlock(struct qspinlock *lock);
> -PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
> +extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
> +__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
>
> #endif /* CONFIG_64BIT */
> #endif
> diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c
> index 2e1600906c9f..d2ef312a8611 100644
> --- a/kernel/locking/qrwlock.c
> +++ b/kernel/locking/qrwlock.c
> @@ -18,7 +18,7 @@
> * queued_read_lock_slowpath - acquire read lock of a queued rwlock
> * @lock: Pointer to queued rwlock structure
> */
> -void queued_read_lock_slowpath(struct qrwlock *lock)
> +void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
> {
> /*
> * Readers come here when they cannot get the lock without waiting
> @@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
> * queued_write_lock_slowpath - acquire write lock of a queued rwlock
> * @lock : Pointer to queued rwlock structure
> */
> -void queued_write_lock_slowpath(struct qrwlock *lock)
> +void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
> {
> int cnts;
>
> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
> index 65a9a10caa6f..2b23378775fe 100644
> --- a/kernel/locking/qspinlock.c
> +++ b/kernel/locking/qspinlock.c
> @@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
> * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
> * queue : ^--' :
> */
> -void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> +void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> {
> struct mcs_spinlock *prev, *next, *node;
> u32 old, tail;
> diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
> index e84d21aa0722..6afc249ce697 100644
> --- a/kernel/locking/qspinlock_paravirt.h
> +++ b/kernel/locking/qspinlock_paravirt.h
> @@ -489,7 +489,7 @@ pv_wait_head_or_lock(struct qspinlock *lock, struct mcs_spinlock *node)
> * PV versions of the unlock fastpath and slowpath functions to be used
> * instead of queued_spin_unlock().
> */
> -__visible void
> +__visible __lockfunc void
> __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
> {
> struct pv_node *node;
> @@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
> #include <asm/qspinlock_paravirt.h>
>
> #ifndef __pv_queued_spin_unlock
> -__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
> +__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
> {
> u8 locked;
>
Acked-by: Waiman Long <longman@redhat.com>
^ permalink raw reply [flat|nested] 3+ messages in thread
* [tip: locking/core] locking: Add __lockfunc to slow path functions
2022-08-10 22:03 [PATCH v2] locking: Add __lockfunc to slow path functions Namhyung Kim
2022-08-11 18:19 ` Waiman Long
@ 2022-08-22 9:22 ` tip-bot2 for Namhyung Kim
1 sibling, 0 replies; 3+ messages in thread
From: tip-bot2 for Namhyung Kim @ 2022-08-22 9:22 UTC (permalink / raw)
To: linux-tip-commits
Cc: Namhyung Kim, Peter Zijlstra (Intel), Waiman Long, x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 501f7f69bca195da266de83eb2c26c30813fba97
Gitweb: https://git.kernel.org/tip/501f7f69bca195da266de83eb2c26c30813fba97
Author: Namhyung Kim <namhyung@kernel.org>
AuthorDate: Wed, 10 Aug 2022 15:03:46 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Fri, 19 Aug 2022 19:47:51 +02:00
locking: Add __lockfunc to slow path functions
So that we can skip the functions in the perf lock contention and other
places like /proc/PID/wchan.
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Link: https://lore.kernel.org/r/20220810220346.1919485-1-namhyung@kernel.org
---
arch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------
kernel/locking/qrwlock.c | 4 ++--
kernel/locking/qspinlock.c | 2 +-
kernel/locking/qspinlock_paravirt.h | 4 ++--
4 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h
index 892fd8c..60ece59 100644
--- a/arch/x86/include/asm/qspinlock_paravirt.h
+++ b/arch/x86/include/asm/qspinlock_paravirt.h
@@ -12,7 +12,7 @@
*/
#ifdef CONFIG_64BIT
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
#define __pv_queued_spin_unlock __pv_queued_spin_unlock
#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
/*
* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
* which combines the registers saving trunk and the body of the following
- * C code:
+ * C code. Note that it puts the code in the .spinlock.text section which
+ * is equivalent to adding __lockfunc in the C code:
*
- * void __pv_queued_spin_unlock(struct qspinlock *lock)
+ * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
* {
* u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
*
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
* rsi = lockval (second argument)
* rdx = internal variable (set to 0)
*/
-asm (".pushsection .text;"
+asm (".pushsection .spinlock.text;"
".globl " PV_UNLOCK ";"
".type " PV_UNLOCK ", @function;"
".align 4,0x90;"
@@ -65,8 +66,8 @@ asm (".pushsection .text;"
#else /* CONFIG_64BIT */
-extern void __pv_queued_spin_unlock(struct qspinlock *lock);
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
+extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
#endif /* CONFIG_64BIT */
#endif
diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c
index 2e16009..d2ef312 100644
--- a/kernel/locking/qrwlock.c
+++ b/kernel/locking/qrwlock.c
@@ -18,7 +18,7 @@
* queued_read_lock_slowpath - acquire read lock of a queued rwlock
* @lock: Pointer to queued rwlock structure
*/
-void queued_read_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
{
/*
* Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
* queued_write_lock_slowpath - acquire write lock of a queued rwlock
* @lock : Pointer to queued rwlock structure
*/
-void queued_write_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
{
int cnts;
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 65a9a10..2b23378 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
* queue : ^--' :
*/
-void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
struct mcs_spinlock *prev, *next, *node;
u32 old, tail;
diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
index e84d21a..6afc249 100644
--- a/kernel/locking/qspinlock_paravirt.h
+++ b/kernel/locking/qspinlock_paravirt.h
@@ -489,7 +489,7 @@ gotlock:
* PV versions of the unlock fastpath and slowpath functions to be used
* instead of queued_spin_unlock().
*/
-__visible void
+__visible __lockfunc void
__pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
{
struct pv_node *node;
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
#include <asm/qspinlock_paravirt.h>
#ifndef __pv_queued_spin_unlock
-__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
+__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
{
u8 locked;
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2022-08-22 9:23 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-08-10 22:03 [PATCH v2] locking: Add __lockfunc to slow path functions Namhyung Kim
2022-08-11 18:19 ` Waiman Long
2022-08-22 9:22 ` [tip: locking/core] " tip-bot2 for Namhyung Kim
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).