[patch 0/3] Text Section Edit Lock

[patch 0/3] Text Section Edit Lock

[Mathieu Desnoyers]

Hi,

This code adds locking around text section modifications. It can be used by
paravirt/alternatives, kprobes and eventually the immediate values to:

A - synchronize dynamic code patching
B - make sure the pages are writable

I only provided the i386 version, but it could be easily ported to other
architectures.

It applies at the end of the 2.6.22-rc4-mm2 series in the following order:

text-edit-lock-i386.patch
text-edit-lock-alternative-i386.patch
text-edit-lock-kprobes-i386.patch

Comments are welcome, (testing is welcome too)

Mathieu

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 1/3] Text Edit Lock - i386

[Mathieu Desnoyers]

[-- Attachment #1: text-edit-lock-i386.patch --]
[-- Type: text/plain, Size: 3636 bytes --]

Interface to use for code patching : uses a mutex to insure mutual edit
exclusion and makes sure the page is writable.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
---
 arch/i386/mm/init.c           |   52 ++++++++++++++++++++++++++++++++++--------
 include/asm-i386/cacheflush.h |    4 +++
 2 files changed, 47 insertions(+), 9 deletions(-)

Index: linux-2.6-lttng/arch/i386/mm/init.c
===================================================================
--- linux-2.6-lttng.orig/arch/i386/mm/init.c	2007-06-18 17:42:42.000000000 -0400
+++ linux-2.6-lttng/arch/i386/mm/init.c	2007-06-18 17:56:31.000000000 -0400
@@ -31,6 +31,7 @@
 #include <linux/memory_hotplug.h>
 #include <linux/initrd.h>
 #include <linux/cpumask.h>
+#include <linux/mutex.h>
 
 #include <asm/processor.h>
 #include <asm/system.h>
@@ -52,6 +53,9 @@
 
 static int noinline do_test_wp_bit(void);
 
+/* Mutex protecting text section modification (dynamic code patching) */
+static DEFINE_MUTEX(text_mutex);
+
 /*
  * Creates a middle page table and puts a pointer to it in the
  * given global directory entry. This only returns the gd entry
@@ -800,15 +804,9 @@
 	unsigned long start = PFN_ALIGN(_text);
 	unsigned long size = PFN_ALIGN(_etext) - start;
 
-#ifdef CONFIG_HOTPLUG_CPU
-	/* It must still be possible to apply SMP alternatives. */
-	if (num_possible_cpus() <= 1)
-#endif
-	{
-		change_page_attr(virt_to_page(start),
-		                 size >> PAGE_SHIFT, PAGE_KERNEL_RX);
-		printk("Write protecting the kernel text: %luk\n", size >> 10);
-	}
+	change_page_attr(virt_to_page(start),
+			 size >> PAGE_SHIFT, PAGE_KERNEL_RX);
+	printk("Write protecting the kernel text: %luk\n", size >> 10);
 
 	start += size;
 	size = (unsigned long)__end_rodata - start;
@@ -827,6 +825,42 @@
 }
 #endif
 
+/*
+ * Lock the kernel text for mutual write exclusion.
+ * Make sure the pages are writable.
+ */
+void kernel_text_lock(unsigned long address, size_t len)
+{
+	mutex_lock(&text_mutex);
+#if defined(CONFIG_DEBUG_RODATA)
+	if (is_kernel_text(address) && is_kernel_text(address + len)) {
+		unsigned long nr_pages;
+		nr_pages = ((address + len) >> PAGE_SHIFT)
+				- (address >> PAGE_SHIFT) + 1;
+		change_page_attr(virt_to_page(address), nr_pages,
+				PAGE_KERNEL_EXEC);
+		global_flush_tlb();
+	}
+#endif
+}
+EXPORT_SYMBOL_GPL(kernel_text_lock);
+
+void kernel_text_unlock(unsigned long address, size_t len)
+{
+#if defined(CONFIG_DEBUG_RODATA)
+	if (is_kernel_text(address) && is_kernel_text(address + len)) {
+		unsigned long nr_pages;
+		nr_pages = ((address + len) >> PAGE_SHIFT)
+				- (address >> PAGE_SHIFT) + 1;
+		change_page_attr(virt_to_page(address), nr_pages,
+				PAGE_KERNEL_RX);
+		global_flush_tlb();
+	}
+#endif
+	mutex_unlock(&text_mutex);
+}
+EXPORT_SYMBOL_GPL(kernel_text_unlock);
+
 void free_init_pages(char *what, unsigned long begin, unsigned long end)
 {
 	unsigned long addr;
Index: linux-2.6-lttng/include/asm-i386/cacheflush.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-i386/cacheflush.h	2007-06-18 17:42:42.000000000 -0400
+++ linux-2.6-lttng/include/asm-i386/cacheflush.h	2007-06-18 17:42:44.000000000 -0400
@@ -36,4 +36,8 @@
 void mark_rodata_ro(void);
 #endif
 
+/* lock kernel text and mark pages writable */
+extern void kernel_text_lock(unsigned long address, size_t len);
+extern void kernel_text_unlock(unsigned long address, size_t len);
+
 #endif /* _I386_CACHEFLUSH_H */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 2/3] Text Edit Lock - Alternative i386

[Mathieu Desnoyers]

[-- Attachment #1: text-edit-lock-alternative-i386.patch --]
[-- Type: text/plain, Size: 1664 bytes --]

alternative.c can use the text edit lock to remove CPU HOTPLUG special cases.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
---
 arch/i386/kernel/alternative.c |    5 +++++
 1 file changed, 5 insertions(+)

Index: linux-2.6-lttng/arch/i386/kernel/alternative.c
===================================================================
--- linux-2.6-lttng.orig/arch/i386/kernel/alternative.c	2007-06-18 17:38:41.000000000 -0400
+++ linux-2.6-lttng/arch/i386/kernel/alternative.c	2007-06-18 17:42:30.000000000 -0400
@@ -4,6 +4,7 @@
 #include <linux/list.h>
 #include <asm/alternative.h>
 #include <asm/sections.h>
+#include <asm/cacheflush.h>
 
 static int noreplace_smp     = 0;
 static int smp_alt_once      = 0;
@@ -179,9 +180,11 @@
 				__FUNCTION__, a->instr, instr);
 		}
 #endif
+		kernel_text_lock((unsigned long)instr, a->instrlen);
 		memcpy(instr, a->replacement, a->replacementlen);
 		diff = a->instrlen - a->replacementlen;
 		nop_out(instr + a->replacementlen, diff);
+		kernel_text_unlock((unsigned long)instr, a->instrlen);
 	}
 }
 
@@ -346,6 +349,7 @@
 	for (p = start; p < end; p++) {
 		unsigned int used;
 
+		kernel_text_lock((unsigned long)p->instr, p->len);
 		used = paravirt_ops.patch(p->instrtype, p->clobbers, p->instr,
 					  p->len);
 
@@ -353,6 +357,7 @@
 
 		/* Pad the rest with nops */
 		nop_out(p->instr + used, p->len - used);
+		kernel_text_unlock((unsigned long)p->instr, p->len);
 	}
 
 	/* Sync to be conservative, in case we patched following

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 3/3] Text Edit Lock - kprobes i386

[Mathieu Desnoyers]

[-- Attachment #1: text-edit-lock-kprobes-i386.patch --]
[-- Type: text/plain, Size: 1413 bytes --]

Kprobes can use the text edit lock to insure mutual exclusion when edition the
code and make sure the pages are writable.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
---
 arch/i386/kernel/kprobes.c |    4 ++++
 1 file changed, 4 insertions(+)

Index: linux-2.6-lttng/arch/i386/kernel/kprobes.c
===================================================================
--- linux-2.6-lttng.orig/arch/i386/kernel/kprobes.c	2007-06-18 17:41:27.000000000 -0400
+++ linux-2.6-lttng/arch/i386/kernel/kprobes.c	2007-06-18 17:41:31.000000000 -0400
@@ -169,14 +169,18 @@
 
 void __kprobes arch_arm_kprobe(struct kprobe *p)
 {
+	kernel_text_lock((unsigned long)p->addr, sizeof(kprobe_opcode_t));
 	*p->addr = BREAKPOINT_INSTRUCTION;
+	kernel_text_unlock((unsigned long)p->addr, sizeof(kprobe_opcode_t));
 	flush_icache_range((unsigned long) p->addr,
 			   (unsigned long) p->addr + sizeof(kprobe_opcode_t));
 }
 
 void __kprobes arch_disarm_kprobe(struct kprobe *p)
 {
+	kernel_text_lock((unsigned long)p->addr, sizeof(kprobe_opcode_t));
 	*p->addr = p->opcode;
+	kernel_text_unlock((unsigned long)p->addr, sizeof(kprobe_opcode_t));
 	flush_icache_range((unsigned long) p->addr,
 			   (unsigned long) p->addr + sizeof(kprobe_opcode_t));
 }

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: [patch 1/3] Text Edit Lock - i386

[Chuck Ebbert]

On 06/18/2007 05:58 PM, Mathieu Desnoyers wrote:
> Interface to use for code patching : uses a mutex to insure mutual edit
> exclusion and makes sure the page is writable.
> 
... 
> +/* Mutex protecting text section modification (dynamic code patching) */
> +static DEFINE_MUTEX(text_mutex);
> +

Probably should be a spinlock.

And it just occurred to me, how does smp_alternatives deal with this?
Is it broken now when the text section is read-only?

Re: [patch 1/3] Text Edit Lock - i386

[Andi Kleen]

On Tuesday 19 June 2007 00:52:25 Chuck Ebbert wrote:
> On 06/18/2007 05:58 PM, Mathieu Desnoyers wrote:
> > Interface to use for code patching : uses a mutex to insure mutual edit
> > exclusion and makes sure the page is writable.
> > 
> ... 
> > +/* Mutex protecting text section modification (dynamic code patching) */
> > +static DEFINE_MUTEX(text_mutex);
> > +
> 
> Probably should be a spinlock.
> 
> And it just occurred to me, how does smp_alternatives deal with this?
> Is it broken now when the text section is read-only?

The text section is only changed to ro very late,  alternative code
runs earlier.

But when you unplug all CPUs to go back to UP I suspect it may break.

-Andi

Re: [patch 1/3] Text Edit Lock - i386

[Mathieu Desnoyers]

* Chuck Ebbert (cebbert@redhat.com) wrote:
> On 06/18/2007 05:58 PM, Mathieu Desnoyers wrote:
> > Interface to use for code patching : uses a mutex to insure mutual edit
> > exclusion and makes sure the page is writable.
> > 
> ... 
> > +/* Mutex protecting text section modification (dynamic code patching) */
> > +static DEFINE_MUTEX(text_mutex);
> > +
> 
> Probably should be a spinlock.
> 
> And it just occurred to me, how does smp_alternatives deal with this?
> Is it broken now when the text section is read-only?

(note that the implementation I just posted is a proof of concept: I
just noticed that I need to keep track of wether or not I am called
before or after the mark_rodata is done, so the apply alternatives does
not crash at early boot because of the global_flush_tlb().)

A spinlock it will be then :)

SMP alternatives deals with this by simply disabling the whole
protection:

mark_rodata_ro():

#ifdef CONFIG_HOTPLUG_CPU
        /* It must still be possible to apply SMP alternatives. */
        if (num_possible_cpus() <= 1)
#endif
        {
                change_page_attr(virt_to_page(start),
                                 size >> PAGE_SHIFT, PAGE_KERNEL_RX);
                printk("Write protecting the kernel text: %luk\n", size >> 10);
        }

So it's ok if no CPU can be hotplugged, since the CPUs are brought up
before the mark_rodata_ro is done, but not if HOTPLUG is selected.

Mathieu

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

[patch 1/3] Text Edit Lock - i386

[Mathieu Desnoyers]

[-- Attachment #1: text-edit-lock-i386.patch --]
[-- Type: text/plain, Size: 4007 bytes --]

Interface to use for code patching : uses a mutex to insure mutual edit
exclusion and makes sure the page is writable.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
---
 arch/i386/mm/init.c           |   69 ++++++++++++++++++++++++++++++++++--------
 include/asm-i386/cacheflush.h |    4 ++
 2 files changed, 61 insertions(+), 12 deletions(-)

Index: linux-2.6-lttng/arch/i386/mm/init.c
===================================================================
--- linux-2.6-lttng.orig/arch/i386/mm/init.c	2007-06-29 14:15:39.000000000 -0400
+++ linux-2.6-lttng/arch/i386/mm/init.c	2007-06-29 14:22:02.000000000 -0400
@@ -31,6 +31,7 @@
 #include <linux/memory_hotplug.h>
 #include <linux/initrd.h>
 #include <linux/cpumask.h>
+#include <linux/mutex.h>
 
 #include <asm/processor.h>
 #include <asm/system.h>
@@ -53,6 +54,13 @@
 
 static int noinline do_test_wp_bit(void);
 
+/* spin lock protecting text section modification (dynamic code patching) */
+static DEFINE_SPINLOCK(text_lock);
+
+#ifdef CONFIG_DEBUG_RODATA
+static int rodata_marked;
+#endif
+
 /*
  * Creates a middle page table and puts a pointer to it in the
  * given global directory entry. This only returns the gd entry
@@ -802,18 +810,11 @@
 	unsigned long start = PFN_ALIGN(_text);
 	unsigned long size = PFN_ALIGN(_etext) - start;
 
-#ifndef CONFIG_KPROBES
-#ifdef CONFIG_HOTPLUG_CPU
-	/* It must still be possible to apply SMP alternatives. */
-	if (num_possible_cpus() <= 1)
-#endif
-	{
-		change_page_attr(virt_to_page(start),
-		                 size >> PAGE_SHIFT, PAGE_KERNEL_RX);
-		printk("Write protecting the kernel text: %luk\n", size >> 10);
-		kernel_text_is_ro = 1;
-	}
-#endif
+	change_page_attr(virt_to_page(start),
+	                 size >> PAGE_SHIFT, PAGE_KERNEL_RX);
+	printk("Write protecting the kernel text: %luk\n", size >> 10);
+	kernel_text_is_ro = 1;
+
 	start += size;
 	size = (unsigned long)__end_rodata - start;
 	change_page_attr(virt_to_page(start),
@@ -828,8 +829,52 @@
 	 * of who is the culprit.
 	 */
 	global_flush_tlb();
+	rodata_marked = 1;
+}
+#endif
+
+/*
+ * Lock the kernel text for mutual write exclusion.
+ * Make sure the pages are writable.
+ */
+void kernel_text_lock(unsigned long address, size_t len)
+{
+	spin_lock(&text_lock);
+#if defined(CONFIG_DEBUG_RODATA)
+	if (rodata_marked && address >= PFN_ALIGN(_text)
+			&& (address + len) <= PFN_ALIGN(_etext)) {
+		unsigned long nr_pages;
+		nr_pages = ((address + len) >> PAGE_SHIFT)
+				- (address >> PAGE_SHIFT) + 1;
+		change_page_attr(virt_to_page(address), nr_pages,
+				PAGE_KERNEL_EXEC);
+		mb();
+		global_flush_tlb();
+		mb();
+	}
+#endif
 }
+EXPORT_SYMBOL_GPL(kernel_text_lock);
+
+void kernel_text_unlock(unsigned long address, size_t len)
+{
+#if defined(CONFIG_DEBUG_RODATA)
+	if (rodata_marked && address >= PFN_ALIGN(_text)
+			&& (address + len) <= PFN_ALIGN(_etext)) {
+		unsigned long nr_pages;
+		wmb();
+		nr_pages = ((address + len) >> PAGE_SHIFT)
+				- (address >> PAGE_SHIFT) + 1;
+		mb();
+		change_page_attr(virt_to_page(address), nr_pages,
+				PAGE_KERNEL_RX);
+		mb();
+		global_flush_tlb();
+	}
 #endif
+	spin_unlock(&text_lock);
+}
+EXPORT_SYMBOL_GPL(kernel_text_unlock);
 
 void free_init_pages(char *what, unsigned long begin, unsigned long end)
 {
Index: linux-2.6-lttng/include/asm-i386/cacheflush.h
===================================================================
--- linux-2.6-lttng.orig/include/asm-i386/cacheflush.h	2007-06-29 10:21:47.000000000 -0400
+++ linux-2.6-lttng/include/asm-i386/cacheflush.h	2007-06-29 14:20:29.000000000 -0400
@@ -36,4 +36,8 @@
 void mark_rodata_ro(void);
 #endif
 
+/* lock kernel text and mark pages writable */
+extern void kernel_text_lock(unsigned long address, size_t len);
+extern void kernel_text_unlock(unsigned long address, size_t len);
+
 #endif /* _I386_CACHEFLUSH_H */

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68