From: Dmitry Safonov <dima@arista.com>
To: linux-kernel@vger.kernel.org
Cc: Dmitry Safonov <0x7f454c46@gmail.com>,
Andrei Vagin <avagin@gmail.com>, Dmitry Safonov <dima@arista.com>,
Adrian Reber <adrian@lisas.de>, Andrei Vagin <avagin@openvz.org>,
Andy Lutomirski <luto@kernel.org>, Arnd Bergmann <arnd@arndb.de>,
Christian Brauner <christian.brauner@ubuntu.com>,
Cyrill Gorcunov <gorcunov@openvz.org>,
"Eric W. Biederman" <ebiederm@xmission.com>,
"H. Peter Anvin" <hpa@zytor.com>, Ingo Molnar <mingo@redhat.com>,
Jann Horn <jannh@google.com>, Jeff Dike <jdike@addtoit.com>,
Oleg Nesterov <oleg@redhat.com>,
Pavel Emelyanov <xemul@virtuozzo.com>,
Shuah Khan <shuah@kernel.org>,
Thomas Gleixner <tglx@linutronix.de>,
Vincenzo Frascino <vincenzo.frascino@arm.com>,
containers@lists.linux-foundation.org, criu@openvz.org,
linux-api@vger.kernel.org, x86@kernel.org
Subject: [PATCHv5 26/37] vdso: Introduce vdso_static_branch_unlikely()
Date: Mon, 29 Jul 2019 22:57:08 +0100 [thread overview]
Message-ID: <20190729215758.28405-27-dima@arista.com> (raw)
In-Reply-To: <20190729215758.28405-1-dima@arista.com>
From: Andrei Vagin <avagin@gmail.com>
As it has been discussed on timens RFC, adding a new conditional branch
`if (inside_time_ns)` on VDSO for all processes is undesirable.
Addressing those problems, there are two versions of VDSO's .so:
for host tasks (without any penalty) and for processes inside of time
namespace with clk_to_ns() that subtracts offsets from host's time.
Introduce vdso_static_branch_unlikely(), which is similar to
static_branch_unlikely(); alias it with timens_static_branch_unlikely()
under CONFIG_TIME_NS.
The timens code in vdso will look like this:
if (timens_static_branch_unlikely()) {
clk_to_ns(clk, ts);
}
The version of vdso which is compiled from sources will never execute
clk_to_ns(). And then we can patch the 'no-op' in the straight-line
codepath with a 'jump' instruction to the out-of-line true branch and
get the timens version of the vdso library.
Signed-off-by: Andrei Vagin <avagin@gmail.com>
Co-developed-by: Dmitry Safonov <dima@arista.com>
Signed-off-by: Dmitry Safonov <dima@arista.com>
---
arch/x86/include/asm/jump_label.h | 14 ++++++++++++++
lib/vdso/gettimeofday.c | 10 ++++++++--
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index 06c3cc22a058..376efb53183b 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -53,6 +53,20 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
return true;
}
+static __always_inline bool vdso_static_branch_unlikely(void)
+{
+ asm_volatile_goto("1:\n\t"
+ ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
+ ".pushsection __jump_table, \"aw\"\n\t"
+ "2: .word 1b - 2b, %l[l_yes] - 2b\n\t"
+ ".popsection\n\t"
+ : : : : l_yes);
+
+ return false;
+l_yes:
+ return true;
+}
+
#else /* __ASSEMBLY__ */
.macro STATIC_JUMP_IF_TRUE target, key, def
diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c
index 7525433f8ba4..605bdb92055d 100644
--- a/lib/vdso/gettimeofday.c
+++ b/lib/vdso/gettimeofday.c
@@ -8,6 +8,7 @@
#include <linux/kernel.h>
#include <linux/hrtimer_defs.h>
#include <linux/timens_offsets.h>
+#include <linux/jump_label.h>
#include <vdso/datapage.h>
#include <vdso/helpers.h>
@@ -43,6 +44,8 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
extern u8 timens_page
__attribute__((visibility("hidden")));
+#define timens_static_branch_unlikely vdso_static_branch_unlikely
+
notrace static __always_inline void clk_to_ns(clockid_t clk, struct __kernel_timespec *ts)
{
struct timens_offsets *timens = (struct timens_offsets *) &timens_page;
@@ -79,6 +82,7 @@ notrace static __always_inline void clk_to_ns(clockid_t clk, struct __kernel_tim
}
#else
notrace static __always_inline void clk_to_ns(clockid_t clk, struct __kernel_timespec *ts) {}
+notrace static __always_inline bool timens_static_branch_unlikely(void) { return false; }
#endif
static int do_hres(const struct vdso_data *vd, clockid_t clk,
@@ -108,7 +112,8 @@ static int do_hres(const struct vdso_data *vd, clockid_t clk,
ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
ts->tv_nsec = ns;
- clk_to_ns(clk, ts);
+ if (timens_static_branch_unlikely())
+ clk_to_ns(clk, ts);
return 0;
}
@@ -125,7 +130,8 @@ static void do_coarse(const struct vdso_data *vd, clockid_t clk,
ts->tv_nsec = vdso_ts->nsec;
} while (unlikely(vdso_read_retry(vd, seq)));
- clk_to_ns(clk, ts);
+ if (timens_static_branch_unlikely())
+ clk_to_ns(clk, ts);
}
static __maybe_unused int
--
2.22.0
next prev parent reply other threads:[~2019-07-29 22:01 UTC|newest]
Thread overview: 68+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-07-29 21:56 [PATCHv5 00/37] kernel: Introduce Time Namespace Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 01/37] ns: " Dmitry Safonov
2019-08-01 5:29 ` Andy Lutomirski
2019-08-01 23:46 ` Dmitry Safonov
2019-08-07 0:24 ` [PATCHv6 " Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 02/37] timens: Add timens_offsets Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 03/37] posix-clocks: Rename the clock_get() into clock_get_timespec() Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 04/37] posix-clocks: Rename *_clock_get() functions into *_clock_get_timespec() Dmitry Safonov
2019-08-07 6:01 ` Thomas Gleixner
2019-07-29 21:56 ` [PATCHv5 05/37] alarmtimer: Rename gettime() callback to get_ktime() Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 06/37] alarmtimer: Provide get_timespec() callback Dmitry Safonov
2019-08-07 6:04 ` Thomas Gleixner
2019-08-08 6:18 ` Andrei Vagin
2019-07-29 21:56 ` [PATCHv5 07/37] posix-clocks: Introduce clock_get_ktime() callback Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 08/37] posix-timers: Use clock_get_ktime() in common_timer_get() Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 09/37] posix-clocks: Introduce CLOCK_MONOTONIC time namespace offsets Dmitry Safonov
2019-08-07 6:07 ` Thomas Gleixner
2019-07-29 21:56 ` [PATCHv5 10/37] posix-clocks: Introduce CLOCK_BOOTTIME time namespace offset Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 11/37] kernel: Add do_timens_ktime_to_host() helper Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 12/37] timerfd: Make timerfd_settime() time namespace aware Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 13/37] posix-timers: Make timer_settime() " Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 14/37] alarmtimer: Make nanosleep " Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 15/37] hrtimers: Prepare hrtimer_nanosleep() for time namespaces Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 16/37] posix-timers: Make clock_nanosleep() time namespace aware Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 17/37] fd/proc: Respect boottime inside time namespace for /proc/uptime Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 18/37] x86/vdso2c: Correct err messages on file opening Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 19/37] x86/vdso2c: Convert iterator to unsigned Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 20/37] x86/vdso/Makefile: Add vobjs32 Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 21/37] x86/vdso: Restrict splitting VVAR VMA Dmitry Safonov
2019-08-01 5:23 ` Andy Lutomirski
2019-07-29 21:57 ` [PATCHv5 22/37] x86/vdso: Rename vdso_image {.data=>.text} Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 23/37] x86/vdso: Add offsets page in vvar Dmitry Safonov
2019-08-01 5:22 ` Andy Lutomirski
2019-07-29 21:57 ` [PATCHv5 24/37] x86/vdso: Allocate timens vdso Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 25/37] x86/vdso: Switch image on setns()/clone() Dmitry Safonov
2019-08-01 5:34 ` Andy Lutomirski
2019-08-01 6:09 ` hpa
2019-08-01 21:39 ` Andy Lutomirski
2019-08-07 0:27 ` [PATCHv6 " Dmitry Safonov
2019-07-29 21:57 ` Dmitry Safonov [this message]
2019-07-29 21:57 ` [PATCHv5 27/37] x86/vdso2c: Process jump tables Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 28/37] x86/vdso: Enable static branches for the timens vdso Dmitry Safonov
2019-08-01 5:21 ` Andy Lutomirski
2019-08-01 6:48 ` Thomas Gleixner
2019-07-29 21:57 ` [PATCHv5 29/37] posix-clocks: Add align for timens_offsets Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 30/37] fs/proc: Introduce /proc/pid/timens_offsets Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 31/37] selftest/timens: Add Time Namespace test for supported clocks Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 32/37] selftest/timens: Add a test for timerfd Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 33/37] selftest/timens: Add a test for clock_nanosleep() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 34/37] selftest/timens: Add procfs selftest Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 35/37] selftest/timens: Add timer offsets test Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 36/37] selftests/timens: Add a simple perf test for clock_gettime() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 37/37] selftest/timens: Check that a right vdso is mapped after fork and exec Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 00/37] kernel: Introduce Time Namespace Dmitry Safonov
2019-07-29 22:07 ` Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 01/37] ns: " Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 02/37] timens: Add timens_offsets Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 03/37] posix-clocks: Rename the clock_get() into clock_get_timespec() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 04/37] posix-clocks: Rename *_clock_get() functions into *_clock_get_timespec() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 05/37] alarmtimer: Rename gettime() callback to get_ktime() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 06/37] alarmtimer: Provide get_timespec() callback Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 07/37] posix-clocks: Introduce clock_get_ktime() callback Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 08/37] posix-timers: Use clock_get_ktime() in common_timer_get() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 09/37] posix-clocks: Introduce CLOCK_MONOTONIC time namespace offsets Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 10/37] posix-clocks: Introduce CLOCK_BOOTTIME time namespace offset Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 11/37] kernel: Add do_timens_ktime_to_host() helper Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 12/37] timerfd: Make timerfd_settime() time namespace aware Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 13/37] posix-timers: Make timer_settime() " Dmitry Safonov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190729215758.28405-27-dima@arista.com \
--to=dima@arista.com \
--cc=0x7f454c46@gmail.com \
--cc=adrian@lisas.de \
--cc=arnd@arndb.de \
--cc=avagin@gmail.com \
--cc=avagin@openvz.org \
--cc=christian.brauner@ubuntu.com \
--cc=containers@lists.linux-foundation.org \
--cc=criu@openvz.org \
--cc=ebiederm@xmission.com \
--cc=gorcunov@openvz.org \
--cc=hpa@zytor.com \
--cc=jannh@google.com \
--cc=jdike@addtoit.com \
--cc=linux-api@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=mingo@redhat.com \
--cc=oleg@redhat.com \
--cc=shuah@kernel.org \
--cc=tglx@linutronix.de \
--cc=vincenzo.frascino@arm.com \
--cc=x86@kernel.org \
--cc=xemul@virtuozzo.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).