[uml-devel] panic() call from non-kernel stack in uml_postsetup

[uml-devel] panic() call from non-kernel stack in uml_postsetup

[Thomas Meyer]

Hi,

When you pass an initrd to UML and the size of the initrd is too big for
the current kernel memory, a panic() call is done with the message("Out
of memory").

But this message never hits the stdout as the following stack trace
dumps because there is no current_thread_info() in the UML stack, which
is still active at this time:

(gdb) bt
#0  dump_stack_print_info (log_lvl=0x6053f583 "\001d") at kernel/printk/printk.c:2979
#1  0x000000006047126b in __dump_stack () at lib/dump_stack.c:14
#2  dump_stack () at lib/dump_stack.c:50
#3  0x000000006046bf87 in panic	(fmt=<optimized out>) at kernel/panic.c:110
#4  0x0000000060010bc7 in ___alloc_bootmem (limit=<optimized out>, goal=<optimized out>, align=<optimized out>, size=<optimized out>) a
t mm/bootmem.c:674
#5  __alloc_bootmem (size=55792228, align=<optimized out>, goal=<optimized out>) at mm/bootmem.c:696
#6  0x0000000060004eca in read_initrd () at arch/um/kernel/initrd.c:39
#7  0x0000000060004b71 in uml_postsetup	() at arch/um/kernel/um_arch.c:228
#8  linux_main (argc=<optimized out>, argv=0x7fffffffdf98) at arch/um/kernel/um_arch.c:377
#9  0x00000000600059db in main (argc=<optimized out>, argv=0x7fffffffdf98, envp=<optimized out>) at arch/um/os-Linux/main.c:156

So current_thread_info() will return the stack of the UML program
itself, which is an invalid struct thread_info:
thread info= 0x7fffffffc000

so the access to current->pid, will segfault as "current" is a macro for
current_thread_info()->task->pid and task is NULL.

2978│ void dump_stack_print_info(const char *log_lvl)
2979│ {
2980│         struct thread_info * ti = current_thread_info();
2981│         printf("thread info =%p\n", ti);
2982│
2983│         printk("%sCPU: %d PID: %d Comm: %.20s %s %s %.*s\n",
2984│                log_lvl, raw_smp_processor_id(), current->pid, current->comm,
2985│                print_tainted(), init_utsname()->release,
2986├>               (int)strcspn(init_utsname()->version, " "),
2987│                init_utsname()->version);


any ideas how to fix this?

I tried to add this, but this failed horribly:

diff --git a/arch/um/include/asm/thread_info.h b/arch/um/include/asm/thread_info.h
--- a/arch/um/include/asm/thread_info.h
+++ b/arch/um/include/asm/thread_info.h
@@ -12,6 +12,11 @@
 #include <asm/page.h>
 #include <asm/uaccess.h>
 
+#define DEBUG_CHECK_CURRENT_TI
+#ifdef DEBUG_CHECK_CURRENT_TI
+#include <kern_util.h>
+#endif
+
 struct thread_info {
 	struct task_struct	*task;		/* main task structure */
 	struct exec_domain	*exec_domain;	/* execution domain */
@@ -53,6 +59,12 @@ static inline struct thread_info *current_thread_info(void)
 
 	asm volatile ("" : "=r" (p) : "0" (&ti));
 	ti = (struct thread_info *) (((unsigned long)p) & ~mask);
+#ifdef DEBUG_CHECK_CURRENT_TI
+	if(!is_kernel_addr((void*)ti)) {
+		printk(KERN_CRIT "Access to current_thread_info() from non-kernel stack!\n");
+//		return &init_thread_info;
+	}
+#endif
 	return ti;
 }
 
diff --git a/arch/um/include/shared/kern_util.h b/arch/um/include/shared/kern_util.h
index 83a91f9..545503e 100644
--- a/arch/um/include/shared/kern_util.h
+++ b/arch/um/include/shared/kern_util.h
@@ -19,6 +19,7 @@ extern int kmalloc_ok;
 #define UML_ROUND_UP(addr) \
 	((((unsigned long) addr) + PAGE_SIZE - 1) & PAGE_MASK)
 
+extern int is_kernel_addr(void* addr);
 extern unsigned long alloc_stack(int order, int atomic);
 extern void free_stack(unsigned long stack, int order);
 
diff --git a/arch/um/os-Linux/main.c b/arch/um/os-Linux/main.c
index df9191a..a1caebb 100644
--- a/arch/um/os-Linux/main.c
+++ b/arch/um/os-Linux/main.c
@@ -254,7 +255,7 @@ void __wrap_free(void *ptr)
 	 * there is a possibility for memory leaks.
 	 */
 
-	if ((addr >= uml_physmem) && (addr < high_physmem)) {
+	if (is_kernel_addr(addr)) {
 		if (kmalloc_ok)
 			kfree(ptr);
 	}
@@ -264,3 +265,10 @@ void __wrap_free(void *ptr)
 	}
 	else __real_free(ptr);
 }
+
+int is_kernel_addr(void* addr)
+{
+	if((addr >= uml_physmem) && (addr < high_physmem))
+		return 1;
+	return 0;
+}
\ No newline at end of file



------------------------------------------------------------------------------
Meet PCI DSS 3.0 Compliance Requirements with EventLog Analyzer
Achieve PCI DSS 3.0 Compliant Status with Out-of-the-box PCI DSS Reports
Are you Audit-Ready for PCI DSS 3.0 Compliance? Download White paper
Comply to PCI DSS 3.0 Requirement 10 and 11.5 with EventLog Analyzer
http://pubads.g.doubleclick.net/gampad/clk?id=154622311&iu=/4140/ostg.clktrk
_______________________________________________
User-mode-linux-devel mailing list
User-mode-linux-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/user-mode-linux-devel

Re: [uml-devel] panic() call from non-kernel stack in uml_postsetup

[Richard Weinberger]

On Fri, Oct 3, 2014 at 1:21 AM, Thomas Meyer <thomas@m3y3r.de> wrote:
> Hi,
>
> When you pass an initrd to UML and the size of the initrd is too big for
> the current kernel memory, a panic() call is done with the message("Out
> of memory").
>
> But this message never hits the stdout as the following stack trace
> dumps because there is no current_thread_info() in the UML stack, which
> is still active at this time:

Do you see the message if you boot UML with the earlyprintk parameter?

> (gdb) bt
> #0  dump_stack_print_info (log_lvl=0x6053f583 "\001d") at kernel/printk/printk.c:2979
> #1  0x000000006047126b in __dump_stack () at lib/dump_stack.c:14
> #2  dump_stack () at lib/dump_stack.c:50
> #3  0x000000006046bf87 in panic (fmt=<optimized out>) at kernel/panic.c:110
> #4  0x0000000060010bc7 in ___alloc_bootmem (limit=<optimized out>, goal=<optimized out>, align=<optimized out>, size=<optimized out>) a
> t mm/bootmem.c:674
> #5  __alloc_bootmem (size=55792228, align=<optimized out>, goal=<optimized out>) at mm/bootmem.c:696
> #6  0x0000000060004eca in read_initrd () at arch/um/kernel/initrd.c:39
> #7  0x0000000060004b71 in uml_postsetup () at arch/um/kernel/um_arch.c:228
> #8  linux_main (argc=<optimized out>, argv=0x7fffffffdf98) at arch/um/kernel/um_arch.c:377
> #9  0x00000000600059db in main (argc=<optimized out>, argv=0x7fffffffdf98, envp=<optimized out>) at arch/um/os-Linux/main.c:156
>
> So current_thread_info() will return the stack of the UML program
> itself, which is an invalid struct thread_info:
> thread info= 0x7fffffffc000
>
> so the access to current->pid, will segfault as "current" is a macro for
> current_thread_info()->task->pid and task is NULL.
>
> 2978│ void dump_stack_print_info(const char *log_lvl)
> 2979│ {
> 2980│         struct thread_info * ti = current_thread_info();
> 2981│         printf("thread info =%p\n", ti);
> 2982│
> 2983│         printk("%sCPU: %d PID: %d Comm: %.20s %s %s %.*s\n",
> 2984│                log_lvl, raw_smp_processor_id(), current->pid, current->comm,
> 2985│                print_tainted(), init_utsname()->release,
> 2986├>               (int)strcspn(init_utsname()->version, " "),
> 2987│                init_utsname()->version);
>
>
> any ideas how to fix this?
>
> I tried to add this, but this failed horribly:
>
> diff --git a/arch/um/include/asm/thread_info.h b/arch/um/include/asm/thread_info.h
> --- a/arch/um/include/asm/thread_info.h
> +++ b/arch/um/include/asm/thread_info.h
> @@ -12,6 +12,11 @@
>  #include <asm/page.h>
>  #include <asm/uaccess.h>
>
> +#define DEBUG_CHECK_CURRENT_TI
> +#ifdef DEBUG_CHECK_CURRENT_TI
> +#include <kern_util.h>
> +#endif
> +
>  struct thread_info {
>         struct task_struct      *task;          /* main task structure */
>         struct exec_domain      *exec_domain;   /* execution domain */
> @@ -53,6 +59,12 @@ static inline struct thread_info *current_thread_info(void)
>
>         asm volatile ("" : "=r" (p) : "0" (&ti));
>         ti = (struct thread_info *) (((unsigned long)p) & ~mask);
> +#ifdef DEBUG_CHECK_CURRENT_TI
> +       if(!is_kernel_addr((void*)ti)) {
> +               printk(KERN_CRIT "Access to current_thread_info() from non-kernel stack!\n");
> +//             return &init_thread_info;
> +       }
> +#endif
>         return ti;
>  }
>
> diff --git a/arch/um/include/shared/kern_util.h b/arch/um/include/shared/kern_util.h
> index 83a91f9..545503e 100644
> --- a/arch/um/include/shared/kern_util.h
> +++ b/arch/um/include/shared/kern_util.h
> @@ -19,6 +19,7 @@ extern int kmalloc_ok;
>  #define UML_ROUND_UP(addr) \
>         ((((unsigned long) addr) + PAGE_SIZE - 1) & PAGE_MASK)
>
> +extern int is_kernel_addr(void* addr);
>  extern unsigned long alloc_stack(int order, int atomic);
>  extern void free_stack(unsigned long stack, int order);
>
> diff --git a/arch/um/os-Linux/main.c b/arch/um/os-Linux/main.c
> index df9191a..a1caebb 100644
> --- a/arch/um/os-Linux/main.c
> +++ b/arch/um/os-Linux/main.c
> @@ -254,7 +255,7 @@ void __wrap_free(void *ptr)
>          * there is a possibility for memory leaks.
>          */
>
> -       if ((addr >= uml_physmem) && (addr < high_physmem)) {
> +       if (is_kernel_addr(addr)) {
>                 if (kmalloc_ok)
>                         kfree(ptr);
>         }
> @@ -264,3 +265,10 @@ void __wrap_free(void *ptr)
>         }
>         else __real_free(ptr);
>  }
> +
> +int is_kernel_addr(void* addr)
> +{
> +       if((addr >= uml_physmem) && (addr < high_physmem))
> +               return 1;
> +       return 0;
> +}
> \ No newline at end of file
>
>
>
> ------------------------------------------------------------------------------
> Meet PCI DSS 3.0 Compliance Requirements with EventLog Analyzer
> Achieve PCI DSS 3.0 Compliant Status with Out-of-the-box PCI DSS Reports
> Are you Audit-Ready for PCI DSS 3.0 Compliance? Download White paper
> Comply to PCI DSS 3.0 Requirement 10 and 11.5 with EventLog Analyzer
> http://pubads.g.doubleclick.net/gampad/clk?id=154622311&iu=/4140/ostg.clktrk
> _______________________________________________
> User-mode-linux-devel mailing list
> User-mode-linux-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/user-mode-linux-devel



-- 
Thanks,
//richard

------------------------------------------------------------------------------
Meet PCI DSS 3.0 Compliance Requirements with EventLog Analyzer
Achieve PCI DSS 3.0 Compliant Status with Out-of-the-box PCI DSS Reports
Are you Audit-Ready for PCI DSS 3.0 Compliance? Download White paper
Comply to PCI DSS 3.0 Requirement 10 and 11.5 with EventLog Analyzer
http://pubads.g.doubleclick.net/gampad/clk?id=154622311&iu=/4140/ostg.clktrk
_______________________________________________
User-mode-linux-devel mailing list
User-mode-linux-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/user-mode-linux-devel

Re: [uml-devel] panic() call from non-kernel stack in uml_postsetup

[Thomas Meyer]

Am Freitag, den 03.10.2014, 10:17 +0200 schrieb Richard Weinberger:
> On Fri, Oct 3, 2014 at 1:21 AM, Thomas Meyer <thomas@m3y3r.de> wrote:
> > Hi,
> >
> > When you pass an initrd to UML and the size of the initrd is too big for
> > the current kernel memory, a panic() call is done with the message("Out
> > of memory").
> >
> > But this message never hits the stdout as the following stack trace
> > dumps because there is no current_thread_info() in the UML stack, which
> > is still active at this time:
> 
> Do you see the message if you boot UML with the earlyprintk parameter?

No it doesn't. The UML kernel crashes will trying to print the stack
trace.

I tried this drop in arch/um/kernel/dump_stack.c:

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/atomic.h>
#include <kern_util.h>

void dump_stack(void)
{
    struct thread_info* ti = current_thread_info();
    if(is_kernel_addr(ti)) {
	    dump_stack_print_info(KERN_DEFAULT);
	    show_stack(NULL, NULL);
	} else {
	    printk(KERN_CRIT "Dump_stack from non-kernel stack!\n");
	}
}

EXPORT_SYMBOL(dump_stack);


But this doesn't help as UML will crash then in kernel/panic.c while
trying to call atomic_notifier_call_chain(&panic_notifier_list, 0, buf);

I think the root cause is that current_thread_info() returns an invalid
thread_info. The proper fix would be to also have the UML stack an valid
thread_info at the begin of the stack, pointing to init_thread_info.

but don't know how to do this.

what do you think about this?

with kind regards
thomas


------------------------------------------------------------------------------
Meet PCI DSS 3.0 Compliance Requirements with EventLog Analyzer
Achieve PCI DSS 3.0 Compliant Status with Out-of-the-box PCI DSS Reports
Are you Audit-Ready for PCI DSS 3.0 Compliance? Download White paper
Comply to PCI DSS 3.0 Requirement 10 and 11.5 with EventLog Analyzer
http://pubads.g.doubleclick.net/gampad/clk?id=154622311&iu=/4140/ostg.clktrk
_______________________________________________
User-mode-linux-devel mailing list
User-mode-linux-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/user-mode-linux-devel

[uml-devel] [PATCH] Dump really early stack traces (was: panic() call from non-kernel stack in uml_postsetup)

[Thomas Meyer]

Am Freitag, den 03.10.2014, 10:17 +0200 schrieb Richard Weinberger:
> On Fri, Oct 3, 2014 at 1:21 AM, Thomas Meyer <thomas@m3y3r.de> wrote:
> > Hi,
> >
> > When you pass an initrd to UML and the size of the initrd is too big for
> > the current kernel memory, a panic() call is done with the message("Out
> > of memory").
> >
> > But this message never hits the stdout as the following stack trace
> > dumps because there is no current_thread_info() in the UML stack, which
> > is still active at this time:
> 
> Do you see the message if you boot UML with the earlyprintk parameter?
> 
Hi,

this panic happens really early, actually only the boot memory is setup
correctly at this point in time, no console, no early console.

The following patch enables UML to also print stack traces from really
early panics, like when the initrd is too big for the allocated memory.
The patch fixes the current_thread_info() access from non-kernel-stack,
i.e. the UML stack itself.
It also registers an kmsg_dumper that dumps the kernel messge buffer via
printf to stdout.
The panic_exit handler is changed to call the kmsg_dumper, as it would
have been the case if the panic_notifer would have return into the
panic() method.

Result looks like this:
[...]
UML running in SKAS0 mode
Adding 19550208 bytes to physical memory to account for exec-shield gap
<1>[    0.000000] bootmem alloc of 44301012 bytes failed!
<0>[    0.000000] Kernel panic - not syncing: Out of memory
<4>[    0.000000] CPU: 0 PID: 0 Comm: swapper Not tainted 3.16.2-dirty #144
<6>[    0.000000] Stack:
<4>[    0.000000]  605f2947 00000002 7fffd9728d68 00000000
<4>[    0.000000]  ffffffff 6002be30 7fffd9729078 60372980
<4>[    0.000000]  7fffd9728d40 604afa9c 60601d98 604ac0cb
<6>[    0.000000] Call Trace:
<6>[    0.000000]  [<6002be30>] ? current_thread_info+0x0/0x40
<6>[    0.000000]  [<60372980>] ? strlen+0x0/0x30
<6>[    0.000000]  [<604afa9c>] dump_stack+0x6b/0x87
<6>[    0.000000]  [<604ac0cb>] ? printk+0x0/0xa0
<6>[    0.000000]  [<60377b90>] ? bust_spinlocks+0x0/0x60
<6>[    0.000000]  [<604ab55b>] panic+0x15d/0x305
<6>[    0.000000]  [<604ab3fe>] ? panic+0x0/0x305
<6>[    0.000000]  [<60046990>] ? memcpy+0x0/0x140
<6>[    0.000000]  [<604ac0cb>] ? printk+0x0/0xa0
<6>[    0.000000]  [<60011862>] ? alloc_bootmem_core+0x0/0x114
<6>[    0.000000]  [<600119a9>] ? ___alloc_bootmem_nopanic+0x33/0x50
<6>[    0.000000]  [<60372980>] ? strlen+0x0/0x30
<6>[    0.000000]  [<60011f88>] __alloc_bootmem+0x4d/0x51
<6>[    0.000000]  [<600060fc>] read_initrd+0x83/0x156
<6>[    0.000000]  [<60005da1>] linux_main+0x466/0x4a9
<6>[    0.000000]  [<6003ff80>] ? change_sig+0x0/0x80
<6>[    0.000000]  [<60006c0b>] main+0x18b/0x390
<6>[    0.000000]  [<600261f0>] ? _start+0x0/0x30
<6>[    0.000000]  [<60006a80>] ? main+0x0/0x390
<6>[    0.000000]  [<600261f0>] ? _start+0x0/0x30
<6>[    0.000000]  [<604aa8c0>] ? __libc_csu_init+0x0/0x70
<6>[    0.000000]  [<600261f0>] ? _start+0x0/0x30
<6>[    0.000000]  [<60026219>] ? _start+0x29/0x30
<6>[    0.000000]  [<600261f0>] ? _start+0x0/0x30
<6>[    0.000000] 
<6>[    0.000000] Stopping all CPUs...done

Feedback/comments are most welcome.

diff --git a/arch/um/include/asm/thread_info.h b/arch/um/include/asm/thread_info.h
--- a/arch/um/include/asm/thread_info.h
+++ b/arch/um/include/asm/thread_info.h
@@ -44,17 +44,8 @@ struct thread_info {
 #define init_stack		(init_thread_union.stack)
 
 #define THREAD_SIZE ((1 << CONFIG_KERNEL_STACK_ORDER) * PAGE_SIZE)
-/* how to get the thread information struct from C */
-static inline struct thread_info *current_thread_info(void)
-{
-	struct thread_info *ti;
-	unsigned long mask = THREAD_SIZE - 1;
-	void *p;
 
-	asm volatile ("" : "=r" (p) : "0" (&ti));
-	ti = (struct thread_info *) (((unsigned long)p) & ~mask);
-	return ti;
-}
+extern struct thread_info *current_thread_info(void);
 
 #define THREAD_SIZE_ORDER CONFIG_KERNEL_STACK_ORDER
 
diff --git a/arch/um/include/shared/kern_util.h b/arch/um/include/shared/kern_util.h
--- a/arch/um/include/shared/kern_util.h
+++ b/arch/um/include/shared/kern_util.h
@@ -19,6 +19,7 @@ extern int kmalloc_ok;
 #define UML_ROUND_UP(addr) \
 	((((unsigned long) addr) + PAGE_SIZE - 1) & PAGE_MASK)
 
+extern int is_kernel_addr(void* addr);
 extern unsigned long alloc_stack(int order, int atomic);
 extern void free_stack(unsigned long stack, int order);
 
diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile
--- a/arch/um/kernel/Makefile
+++ b/arch/um/kernel/Makefile
@@ -13,7 +13,7 @@ clean-files :=
 obj-y = config.o exec.o exitcode.o irq.o ksyms.o mem.o \
 	physmem.o process.o ptrace.o reboot.o sigio.o \
 	signal.o smp.o syscall.o sysrq.o time.o tlb.o trap.o \
-	um_arch.o umid.o maccess.o skas/
+	um_arch.o umid.o maccess.o kmsg_dump.o skas/
 
 obj-$(CONFIG_BLK_DEV_INITRD) += initrd.o
 obj-$(CONFIG_GPROF)	+= gprof_syms.o
diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c
--- a/arch/um/kernel/um_arch.c
+++ b/arch/um/kernel/um_arch.c
@@ -11,6 +11,7 @@
 #include <linux/string.h>
 #include <linux/utsname.h>
 #include <linux/sched.h>
+#include <linux/kmsg_dump.h>
 #include <asm/pgtable.h>
 #include <asm/processor.h>
 #include <asm/sections.h>
 @@ -234,6 +235,7 @@ static void __init uml_postsetup(void)
 static int panic_exit(struct notifier_block *self, unsigned long unused1,
 		      void *unused2)
 {
+	kmsg_dump(KMSG_DUMP_PANIC);
 	bust_spinlocks(1);
 	bust_spinlocks(0);
 	uml_exitcode = 1;
@@ -407,3 +421,21 @@ void alternatives_smp_module_del(struct module *mod)
 {
 }
 #endif
+
+/* how to get the thread information struct from C */
+inline struct thread_info *current_thread_info(void)
+{
+	struct thread_info *ti;
+	unsigned long mask = THREAD_SIZE - 1;
+	void *p;
+
+	asm volatile ("" : "=r" (p) : "0" (&ti));
+	ti = (struct thread_info *) (((unsigned long)p) & ~mask);
+	if(!is_kernel_addr((void*)ti)) {
+		return &init_thread_info;
+	}
+	return ti;
+}
diff --git a/arch/um/os-Linux/main.c b/arch/um/os-Linux/main.c
--- a/arch/um/os-Linux/main.c
+++ b/arch/um/os-Linux/main.c
@@ -254,7 +255,7 @@ void __wrap_free(void *ptr)
 	 * there is a possibility for memory leaks.
 	 */
 
-	if ((addr >= uml_physmem) && (addr < high_physmem)) {
+	if (is_kernel_addr(addr)) {
 		if (kmalloc_ok)
 			kfree(ptr);
 	}
@@ -264,3 +265,10 @@ void __wrap_free(void *ptr)
 	}
 	else __real_free(ptr);
 }
+
+int is_kernel_addr(void* addr)
+{
+	if((addr >= uml_physmem) && (addr < high_physmem))
+		return 1;
+	return 0;
+}
\ No newline at end of file
--- /dev/null	2014-10-01 22:51:22.599698712 +0200
+++ arch/um/kernel/kmsg_dump.c	2014-10-04 01:33:45.792999523 +0200
@@ -0,0 +1,25 @@
+#include <linux/kmsg_dump.h>
+#include <shared/init.h>
+#include <shared/kern.h>
+
+static struct kmsg_dumper pf_dumper;
+
+static void pf_kmsg_dump(struct kmsg_dumper *dumper,
+			 enum kmsg_dump_reason reason)
+{
+	static char line[1024];
+	size_t len;
+
+	while (kmsg_dump_get_line(dumper, true, line, sizeof(line), &len)) {
+		line[len-1] = '\0';
+		printf("%s\n", line);
+	}
+}
+
+int kmsg_dump_printf(void)
+{
+	pf_dumper.dump = pf_kmsg_dump;
+	return kmsg_dump_register(&pf_dumper);
+}
+
+__uml_initcall(kmsg_dump_printf);



------------------------------------------------------------------------------
Meet PCI DSS 3.0 Compliance Requirements with EventLog Analyzer
Achieve PCI DSS 3.0 Compliant Status with Out-of-the-box PCI DSS Reports
Are you Audit-Ready for PCI DSS 3.0 Compliance? Download White paper
Comply to PCI DSS 3.0 Requirement 10 and 11.5 with EventLog Analyzer
http://pubads.g.doubleclick.net/gampad/clk?id=154622311&iu=/4140/ostg.clktrk
_______________________________________________
User-mode-linux-devel mailing list
User-mode-linux-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/user-mode-linux-devel