From: Dmitry Safonov <dima@arista.com>
To: linux-kernel@vger.kernel.org
Cc: Dmitry Safonov <0x7f454c46@gmail.com>,
Dmitry Safonov <dima@arista.com>, Adrian Reber <adrian@lisas.de>,
Andrei Vagin <avagin@openvz.org>,
Andy Lutomirski <luto@kernel.org>, Arnd Bergmann <arnd@arndb.de>,
Christian Brauner <christian.brauner@ubuntu.com>,
Cyrill Gorcunov <gorcunov@openvz.org>,
"Eric W. Biederman" <ebiederm@xmission.com>,
"H. Peter Anvin" <hpa@zytor.com>, Ingo Molnar <mingo@redhat.com>,
Jann Horn <jannh@google.com>, Jeff Dike <jdike@addtoit.com>,
Oleg Nesterov <oleg@redhat.com>,
Pavel Emelyanov <xemul@virtuozzo.com>,
Shuah Khan <shuah@kernel.org>,
Thomas Gleixner <tglx@linutronix.de>,
Vincenzo Frascino <vincenzo.frascino@arm.com>,
containers@lists.linux-foundation.org, criu@openvz.org,
linux-api@vger.kernel.org, x86@kernel.org
Subject: [PATCHv5 24/37] x86/vdso: Allocate timens vdso
Date: Mon, 29 Jul 2019 22:57:06 +0100 [thread overview]
Message-ID: <20190729215758.28405-25-dima@arista.com> (raw)
In-Reply-To: <20190729215758.28405-1-dima@arista.com>
As it has been discussed on timens RFC, adding a new conditional branch
`if (inside_time_ns)` on VDSO for all processes is undesirable.
It will add a penalty for everybody as branch predictor may mispredict
the jump. Also there are instruction cache lines wasted on cmp/jmp.
Those effects of introducing time namespace are very much unwanted
having in mind how much work have been spent on micro-optimisation
vdso code.
The propose is to allocate a second vdso code with dynamically
patched out (disabled by static_branch) timens code on boot time.
Allocate another vdso and copy original code.
Co-developed-by: Andrei Vagin <avagin@openvz.org>
Signed-off-by: Andrei Vagin <avagin@openvz.org>
Signed-off-by: Dmitry Safonov <dima@arista.com>
---
arch/x86/entry/vdso/vdso2c.h | 2 +-
arch/x86/entry/vdso/vma.c | 113 +++++++++++++++++++++++++++++++++--
arch/x86/include/asm/vdso.h | 9 +--
3 files changed, 114 insertions(+), 10 deletions(-)
diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h
index 7556bb70ed8b..885b988aea19 100644
--- a/arch/x86/entry/vdso/vdso2c.h
+++ b/arch/x86/entry/vdso/vdso2c.h
@@ -157,7 +157,7 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
}
fprintf(outfile, "\n};\n\n");
- fprintf(outfile, "const struct vdso_image %s = {\n", image_name);
+ fprintf(outfile, "struct vdso_image %s __ro_after_init = {\n", image_name);
fprintf(outfile, "\t.text = raw_data,\n");
fprintf(outfile, "\t.size = %lu,\n", mapping_size);
if (alt_sec) {
diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
index 9bd66f84db5e..8a8211fd4cfc 100644
--- a/arch/x86/entry/vdso/vma.c
+++ b/arch/x86/entry/vdso/vma.c
@@ -30,26 +30,128 @@
unsigned int __read_mostly vdso64_enabled = 1;
#endif
-void __init init_vdso_image(const struct vdso_image *image)
+void __init init_vdso_image(struct vdso_image *image)
{
BUG_ON(image->size % PAGE_SIZE != 0);
apply_alternatives((struct alt_instr *)(image->text + image->alt),
(struct alt_instr *)(image->text + image->alt +
image->alt_len));
+#ifdef CONFIG_TIME_NS
+ image->text_timens = vmalloc_32(image->size);
+ if (WARN_ON(image->text_timens == NULL))
+ return;
+
+ memcpy(image->text_timens, image->text, image->size);
+#endif
}
struct linux_binprm;
+#ifdef CONFIG_TIME_NS
+static inline struct timens_offsets *current_timens_offsets(void)
+{
+ return current->nsproxy->time_ns->offsets;
+}
+
+static int vdso_check_timens(struct vm_area_struct *vma, bool *in_timens)
+{
+ struct task_struct *tsk;
+
+ if (likely(vma->vm_mm == current->mm)) {
+ *in_timens = !!current_timens_offsets();
+ return 0;
+ }
+
+ /*
+ * .fault() handler can be called over remote process through
+ * interfaces like /proc/$pid/mem or process_vm_{readv,writev}()
+ * Considering such access to vdso as a slow-path.
+ */
+
+#ifdef CONFIG_MEMCG
+ rcu_read_lock();
+
+ tsk = rcu_dereference(vma->vm_mm->owner);
+ if (tsk) {
+ task_lock(tsk);
+ /*
+ * Shouldn't happen: nsproxy is unset in exit_mm().
+ * Before that exit_mm() holds mmap_sem to set (mm = NULL).
+ * It's impossible to have a fault in task without mm
+ * and mmap_sem is taken during the fault.
+ */
+ if (WARN_ON_ONCE(tsk->nsproxy == NULL)) {
+ task_unlock(tsk);
+ rcu_read_unlock();
+ return -EIO;
+ }
+ *in_timens = !!tsk->nsproxy->time_ns->offsets;
+ task_unlock(tsk);
+ rcu_read_unlock();
+ return 0;
+ }
+ rcu_read_unlock();
+#endif
+
+ read_lock(&tasklist_lock);
+ for_each_process(tsk) {
+ struct task_struct *c;
+
+ if (tsk->flags & PF_KTHREAD)
+ continue;
+ for_each_thread(tsk, c) {
+ if (c->mm == vma->vm_mm)
+ goto found;
+ if (c->mm)
+ break;
+ }
+ }
+ read_unlock(&tasklist_lock);
+ return -ESRCH;
+
+found:
+ task_lock(tsk);
+ read_unlock(&tasklist_lock);
+ *in_timens = !!tsk->nsproxy->time_ns->offsets;
+ task_unlock(tsk);
+
+ return 0;
+}
+#else /* CONFIG_TIME_NS */
+static inline int vdso_check_timens(struct vm_area_struct *vma, bool *in_timens)
+{
+ *in_timens = false;
+ return 0;
+}
+static inline struct timens_offsets *current_timens_offsets(void)
+{
+ return NULL;
+}
+#endif /* CONFIG_TIME_NS */
+
static vm_fault_t vdso_fault(const struct vm_special_mapping *sm,
struct vm_area_struct *vma, struct vm_fault *vmf)
{
const struct vdso_image *image = vma->vm_mm->context.vdso_image;
+ unsigned long offset = vmf->pgoff << PAGE_SHIFT;
+ bool in_timens;
+ int err;
if (!image || (vmf->pgoff << PAGE_SHIFT) >= image->size)
return VM_FAULT_SIGBUS;
- vmf->page = virt_to_page(image->text + (vmf->pgoff << PAGE_SHIFT));
+ err = vdso_check_timens(vma, &in_timens);
+ if (err)
+ return VM_FAULT_SIGBUS;
+
+ WARN_ON_ONCE(in_timens && !image->text_timens);
+
+ if (in_timens && image->text_timens)
+ vmf->page = vmalloc_to_page(image->text_timens + offset);
+ else
+ vmf->page = virt_to_page(image->text + offset);
+
get_page(vmf->page);
return 0;
}
@@ -138,13 +240,14 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
return vmf_insert_pfn(vma, vmf->address,
vmalloc_to_pfn(tsc_pg));
} else if (sym_offset == image->sym_timens_page) {
- struct time_namespace *ns = current->nsproxy->time_ns;
+ /* We can fault only in current context for VM_PFNMAP mapping */
+ struct timens_offsets *offsets = current_timens_offsets();
unsigned long pfn;
- if (!ns->offsets)
+ if (!offsets)
pfn = page_to_pfn(ZERO_PAGE(0));
else
- pfn = page_to_pfn(virt_to_page(ns->offsets));
+ pfn = page_to_pfn(virt_to_page(offsets));
return vmf_insert_pfn(vma, vmf->address, pfn);
}
diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h
index 9d420c545607..03f468c63a24 100644
--- a/arch/x86/include/asm/vdso.h
+++ b/arch/x86/include/asm/vdso.h
@@ -12,6 +12,7 @@
struct vdso_image {
void *text;
+ void *text_timens;
unsigned long size; /* Always a multiple of PAGE_SIZE */
unsigned long alt, alt_len;
@@ -30,18 +31,18 @@ struct vdso_image {
};
#ifdef CONFIG_X86_64
-extern const struct vdso_image vdso_image_64;
+extern struct vdso_image vdso_image_64;
#endif
#ifdef CONFIG_X86_X32
-extern const struct vdso_image vdso_image_x32;
+extern struct vdso_image vdso_image_x32;
#endif
#if defined CONFIG_X86_32 || defined CONFIG_COMPAT
-extern const struct vdso_image vdso_image_32;
+extern struct vdso_image vdso_image_32;
#endif
-extern void __init init_vdso_image(const struct vdso_image *image);
+extern void __init init_vdso_image(struct vdso_image *image);
extern int map_vdso_once(const struct vdso_image *image, unsigned long addr);
--
2.22.0
next prev parent reply other threads:[~2019-07-29 22:01 UTC|newest]
Thread overview: 68+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-07-29 21:56 [PATCHv5 00/37] kernel: Introduce Time Namespace Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 01/37] ns: " Dmitry Safonov
2019-08-01 5:29 ` Andy Lutomirski
2019-08-01 23:46 ` Dmitry Safonov
2019-08-07 0:24 ` [PATCHv6 " Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 02/37] timens: Add timens_offsets Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 03/37] posix-clocks: Rename the clock_get() into clock_get_timespec() Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 04/37] posix-clocks: Rename *_clock_get() functions into *_clock_get_timespec() Dmitry Safonov
2019-08-07 6:01 ` Thomas Gleixner
2019-07-29 21:56 ` [PATCHv5 05/37] alarmtimer: Rename gettime() callback to get_ktime() Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 06/37] alarmtimer: Provide get_timespec() callback Dmitry Safonov
2019-08-07 6:04 ` Thomas Gleixner
2019-08-08 6:18 ` Andrei Vagin
2019-07-29 21:56 ` [PATCHv5 07/37] posix-clocks: Introduce clock_get_ktime() callback Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 08/37] posix-timers: Use clock_get_ktime() in common_timer_get() Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 09/37] posix-clocks: Introduce CLOCK_MONOTONIC time namespace offsets Dmitry Safonov
2019-08-07 6:07 ` Thomas Gleixner
2019-07-29 21:56 ` [PATCHv5 10/37] posix-clocks: Introduce CLOCK_BOOTTIME time namespace offset Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 11/37] kernel: Add do_timens_ktime_to_host() helper Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 12/37] timerfd: Make timerfd_settime() time namespace aware Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 13/37] posix-timers: Make timer_settime() " Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 14/37] alarmtimer: Make nanosleep " Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 15/37] hrtimers: Prepare hrtimer_nanosleep() for time namespaces Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 16/37] posix-timers: Make clock_nanosleep() time namespace aware Dmitry Safonov
2019-07-29 21:56 ` [PATCHv5 17/37] fd/proc: Respect boottime inside time namespace for /proc/uptime Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 18/37] x86/vdso2c: Correct err messages on file opening Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 19/37] x86/vdso2c: Convert iterator to unsigned Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 20/37] x86/vdso/Makefile: Add vobjs32 Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 21/37] x86/vdso: Restrict splitting VVAR VMA Dmitry Safonov
2019-08-01 5:23 ` Andy Lutomirski
2019-07-29 21:57 ` [PATCHv5 22/37] x86/vdso: Rename vdso_image {.data=>.text} Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 23/37] x86/vdso: Add offsets page in vvar Dmitry Safonov
2019-08-01 5:22 ` Andy Lutomirski
2019-07-29 21:57 ` Dmitry Safonov [this message]
2019-07-29 21:57 ` [PATCHv5 25/37] x86/vdso: Switch image on setns()/clone() Dmitry Safonov
2019-08-01 5:34 ` Andy Lutomirski
2019-08-01 6:09 ` hpa
2019-08-01 21:39 ` Andy Lutomirski
2019-08-07 0:27 ` [PATCHv6 " Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 26/37] vdso: Introduce vdso_static_branch_unlikely() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 27/37] x86/vdso2c: Process jump tables Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 28/37] x86/vdso: Enable static branches for the timens vdso Dmitry Safonov
2019-08-01 5:21 ` Andy Lutomirski
2019-08-01 6:48 ` Thomas Gleixner
2019-07-29 21:57 ` [PATCHv5 29/37] posix-clocks: Add align for timens_offsets Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 30/37] fs/proc: Introduce /proc/pid/timens_offsets Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 31/37] selftest/timens: Add Time Namespace test for supported clocks Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 32/37] selftest/timens: Add a test for timerfd Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 33/37] selftest/timens: Add a test for clock_nanosleep() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 34/37] selftest/timens: Add procfs selftest Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 35/37] selftest/timens: Add timer offsets test Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 36/37] selftests/timens: Add a simple perf test for clock_gettime() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 37/37] selftest/timens: Check that a right vdso is mapped after fork and exec Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 00/37] kernel: Introduce Time Namespace Dmitry Safonov
2019-07-29 22:07 ` Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 01/37] ns: " Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 02/37] timens: Add timens_offsets Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 03/37] posix-clocks: Rename the clock_get() into clock_get_timespec() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 04/37] posix-clocks: Rename *_clock_get() functions into *_clock_get_timespec() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 05/37] alarmtimer: Rename gettime() callback to get_ktime() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 06/37] alarmtimer: Provide get_timespec() callback Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 07/37] posix-clocks: Introduce clock_get_ktime() callback Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 08/37] posix-timers: Use clock_get_ktime() in common_timer_get() Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 09/37] posix-clocks: Introduce CLOCK_MONOTONIC time namespace offsets Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 10/37] posix-clocks: Introduce CLOCK_BOOTTIME time namespace offset Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 11/37] kernel: Add do_timens_ktime_to_host() helper Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 12/37] timerfd: Make timerfd_settime() time namespace aware Dmitry Safonov
2019-07-29 21:57 ` [PATCHv5 13/37] posix-timers: Make timer_settime() " Dmitry Safonov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190729215758.28405-25-dima@arista.com \
--to=dima@arista.com \
--cc=0x7f454c46@gmail.com \
--cc=adrian@lisas.de \
--cc=arnd@arndb.de \
--cc=avagin@openvz.org \
--cc=christian.brauner@ubuntu.com \
--cc=containers@lists.linux-foundation.org \
--cc=criu@openvz.org \
--cc=ebiederm@xmission.com \
--cc=gorcunov@openvz.org \
--cc=hpa@zytor.com \
--cc=jannh@google.com \
--cc=jdike@addtoit.com \
--cc=linux-api@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=mingo@redhat.com \
--cc=oleg@redhat.com \
--cc=shuah@kernel.org \
--cc=tglx@linutronix.de \
--cc=vincenzo.frascino@arm.com \
--cc=x86@kernel.org \
--cc=xemul@virtuozzo.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).