[PATCH] x86: Add a list for custom page fault handlers.

[PATCH] x86: Add a list for custom page fault handlers.

[Pekka Paalanen]

From: Pekka Paalanen <pq@iki.fi>

Provides kernel modules a way to register custom page fault handlers.
On every page fault, except those handled in vmalloc_fault(), this will
call a list of registered functions. The functions may handle the fault
and force do_page_fault() to return immediately.

This functionality is similar to the now removed page fault notifiers.
Custom page fault handlers are used by debugging and reverse engineering
tools. Mmio-trace is one such tool and a patch to add it into the tree
will follow.

The custom page fault handlers are called from the exact same points in
do_page_fault() as the page fault notifiers were.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---

This change is x86 specific purely because my intended in-tree user of
it is mmio-trace, which AFAIK is really useful only on x86 for reverse
engineering binary drivers. If someone later wishes to merge this with
notify_page_fault() from kprobes, they can do that.

I am not subscribed to LKML, so please keep me in CC.

I will reply to this email with the next patch adding mmio-trace into
kernel.

Thanks,
pq

 arch/x86/Kconfig.debug   |    9 ++++++
 arch/x86/mm/fault_32.c   |   64 ++++++++++++++++++++++++++++++++++++++++++++++
 arch/x86/mm/fault_64.c   |   64 ++++++++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/kdebug.h |    9 ++++++
 4 files changed, 146 insertions(+), 0 deletions(-)

diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 761ca7b..920a63d 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -112,4 +112,13 @@ config IOMMU_LEAK
 	  Add a simple leak tracer to the IOMMU code. This is useful when you
 	  are debugging a buggy device driver that leaks IOMMU mappings.
 
+config PAGE_FAULT_HANDLERS
+	bool "Custom page fault handlers"
+	depends on DEBUG_KERNEL
+	help
+	  Allow the use of custom page fault handlers. A kernel module may
+	  register a function that is called on every page fault not handled
+	  for vmalloc. Custom handlers are used by some debugging and reverse
+	  engineering tools.
+
 endmenu
diff --git a/arch/x86/mm/fault_32.c b/arch/x86/mm/fault_32.c
index a2273d4..19e16b0 100644
--- a/arch/x86/mm/fault_32.c
+++ b/arch/x86/mm/fault_32.c
@@ -33,6 +33,64 @@
 
 extern void die(const char *,struct pt_regs *,long);
 
+#ifdef CONFIG_PAGE_FAULT_HANDLERS
+static HLIST_HEAD(pf_handlers); /* protected by RCU */
+static DEFINE_SPINLOCK(pf_handlers_writer);
+
+void register_page_fault_handler(struct pf_handler *new_pfh)
+{
+	spin_lock(&pf_handlers_writer);
+	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
+	spin_unlock(&pf_handlers_writer);
+}
+EXPORT_SYMBOL_GPL(register_page_fault_handler);
+
+void unregister_page_fault_handler(struct pf_handler *old_pfh)
+{
+	might_sleep();
+	spin_lock(&pf_handlers_writer);
+	hlist_del_rcu(&old_pfh->hlist);
+	spin_unlock(&pf_handlers_writer);
+	synchronize_rcu();
+}
+EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
+
+/* returns non-zero if do_page_fault() should return */
+static int call_pf_handlers(struct pt_regs *regs, unsigned long error_code,
+							unsigned long address)
+{
+	int ret = 0;
+	struct pf_handler *cur;
+	struct hlist_node *ncur;
+
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
+		ret = cur->handler(regs, error_code, address);
+		if (ret)
+			break;
+	}
+	rcu_read_unlock();
+	return ret;
+}
+
+static inline int custom_pf_handlers(void)
+{
+	return !hlist_empty(&pf_handlers);
+}
+#else
+static inline int call_pf_handlers(struct pt_regs *regs,
+						unsigned long error_code,
+						unsigned long address)
+{
+	return 0;
+}
+
+static inline int custom_pf_handlers(void)
+{
+	return 0;
+}
+#endif /* CONFIG_PAGE_FAULT_HANDLERS */
+
 #ifdef CONFIG_KPROBES
 static inline int notify_page_fault(struct pt_regs *regs)
 {
@@ -333,6 +391,9 @@ fastcall void __kprobes do_page_fault(struct pt_regs *regs,
 			return;
 		if (notify_page_fault(regs))
 			return;
+		if (unlikely(custom_pf_handlers()))
+			if (call_pf_handlers(regs, error_code, address))
+				return;
 		/*
 		 * Don't take the mm semaphore here. If we fixup a prefetch
 		 * fault we could otherwise deadlock.
@@ -342,6 +403,9 @@ fastcall void __kprobes do_page_fault(struct pt_regs *regs,
 
 	if (notify_page_fault(regs))
 		return;
+	if (unlikely(custom_pf_handlers()))
+		if (call_pf_handlers(regs, error_code, address))
+			return;
 
 	/* It's safe to allow irq's after cr2 has been saved and the vmalloc
 	   fault has been handled. */
diff --git a/arch/x86/mm/fault_64.c b/arch/x86/mm/fault_64.c
index 0e26230..11f15cc 100644
--- a/arch/x86/mm/fault_64.c
+++ b/arch/x86/mm/fault_64.c
@@ -41,6 +41,64 @@
 #define PF_RSVD	(1<<3)
 #define PF_INSTR	(1<<4)
 
+#ifdef CONFIG_PAGE_FAULT_HANDLERS
+static HLIST_HEAD(pf_handlers); /* protected by RCU */
+static DEFINE_SPINLOCK(pf_handlers_writer);
+
+void register_page_fault_handler(struct pf_handler *new_pfh)
+{
+	spin_lock(&pf_handlers_writer);
+	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
+	spin_unlock(&pf_handlers_writer);
+}
+EXPORT_SYMBOL_GPL(register_page_fault_handler);
+
+void unregister_page_fault_handler(struct pf_handler *old_pfh)
+{
+	might_sleep();
+	spin_lock(&pf_handlers_writer);
+	hlist_del_rcu(&old_pfh->hlist);
+	spin_unlock(&pf_handlers_writer);
+	synchronize_rcu();
+}
+EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
+
+/* returns non-zero if do_page_fault() should return */
+static int call_pf_handlers(struct pt_regs *regs, unsigned long error_code,
+							unsigned long address)
+{
+	int ret = 0;
+	struct pf_handler *cur;
+	struct hlist_node *ncur;
+
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
+		ret = cur->handler(regs, error_code, address);
+		if (ret)
+			break;
+	}
+	rcu_read_unlock();
+	return ret;
+}
+
+static inline int custom_pf_handlers(void)
+{
+	return !hlist_empty(&pf_handlers);
+}
+#else
+static inline int call_pf_handlers(struct pt_regs *regs,
+						unsigned long error_code,
+						unsigned long address)
+{
+	return 0;
+}
+
+static inline int custom_pf_handlers(void)
+{
+	return 0;
+}
+#endif /* CONFIG_PAGE_FAULT_HANDLERS */
+
 #ifdef CONFIG_KPROBES
 static inline int notify_page_fault(struct pt_regs *regs)
 {
@@ -345,6 +403,9 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
 		}
 		if (notify_page_fault(regs))
 			return;
+		if (unlikely(custom_pf_handlers()))
+			if (call_pf_handlers(regs, error_code, address))
+				return;
 		/*
 		 * Don't take the mm semaphore here. If we fixup a prefetch
 		 * fault we could otherwise deadlock.
@@ -354,6 +415,9 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
 
 	if (notify_page_fault(regs))
 		return;
+	if (unlikely(custom_pf_handlers()))
+		if (call_pf_handlers(regs, error_code, address))
+			return;
 
 	if (likely(regs->eflags & X86_EFLAGS_IF))
 		local_irq_enable();
diff --git a/include/asm-x86/kdebug.h b/include/asm-x86/kdebug.h
index e2f9b62..7449ad2 100644
--- a/include/asm-x86/kdebug.h
+++ b/include/asm-x86/kdebug.h
@@ -30,4 +30,13 @@ extern void dump_pagetable(unsigned long);
 extern unsigned long oops_begin(void);
 extern void oops_end(unsigned long);
 
+struct pf_handler {
+	struct hlist_node hlist;
+	int (*handler)(struct pt_regs *regs, unsigned long error_code,
+						unsigned long address);
+};
+
+extern void register_page_fault_handler(struct pf_handler *new_pfh);
+extern void unregister_page_fault_handler(struct pf_handler *old_pfh);
+
 #endif
-- 
1.5.3.7

[RFC PATCH] x86: mmiotrace - trace memory mapped IO

[Pekka Paalanen]

Hello,

the patch itself is slightly bigger than recommended for LKML, so
it is available at:
http://jumi.lut.fi/~paalanen/scratch/mmio24/0002-x86-mmiotrace-trace-memory-mapped-IO.patch.txt

This seems to work fine on UP machine, I have not tested yet with
with an SMP machine. Originally mmio-trace was UP-only, so there
probably still are SMP issues. Mmio-trace has existed with its current
purpose for about a year now.

The following is a copy of the patch description:

Mmiotrace is a tool for trapping memory mapped IO (MMIO) accesses within
the kernel. It is used for debugging and especially for reverse
engineering evil binary drivers.

Mmiotrace works by wrapping the ioremap family of kernel functions and
marking the returned pages as not present. Access to the IO memory
triggers a page fault, which will be handled by mmiotrace's custom page
fault handler. This will single-step the faulted instruction with the
MMIO page marked as present. Access logs are directed to user space via
relay and debug_fs.

This page fault approach is necessary, because binary drivers have
readl/writel etc. calls inlined and therefore extremely difficult to
trap with with e.g. kprobes.

This patch depends on the custom page fault handlers patch.

---

More information on mmio-trace can be found at
http://nouveau.freedesktop.org/wiki/MmioTrace
and the out-of-three module with user space parts is at
http://gitweb.freedesktop.org/?p=users/pq/mmio-trace.git;a=summary

The out-of-tree git does not include all changes I made for the
in-tree version.

Mmiotrace does not appear to procude excruciating slowness when used.
Starting X with traced nvidia binary drivers takes a couple of seconds
longer than normally. Starting X, running glxgears for a moment and
shutting down X results in an mmio-trace log of roughly 1.5 million
events. Replace glxgears by running Quake3-demo's first demo play
twice and end up with 3.5 million events. Quake3-demo framerate with
mmio-traced driver goes down to 59 fps, compared to the normal 83 fps.

I know of one bad failure case with mmio-trace. It may trigger a
complete system freeze when tracing a certain nvidia driver version
on a machine with an 8000-series (nv50) graphics chip. Netconsole,
serial console and NMI watchdog have so far been useless, and I
am starting to believe that it is the nvidia driver that does
something bad to make mmio-trace fall over. I have heard that more
recent nvidia drivers do not make it crash anymore.

I'm hoping mmiotrace could be included in 2.6.25. It would help
FOSS driver projects (Nouveau, maybe madwifi, too) to gather the
information they need. Very often the information comes from
ordinary users, for whom going through kernel patching would be
a major struggle.


Thanks,
Pekka

Re: [PATCH] x86: Add a list for custom page fault handlers.

[Ingo Molnar]

* Pekka Paalanen <pq@iki.fi> wrote:

>  arch/x86/Kconfig.debug   |    9 ++++++
>  arch/x86/mm/fault_32.c   |   64 ++++++++++++++++++++++++++++++++++++++++++++++
>  arch/x86/mm/fault_64.c   |   64 ++++++++++++++++++++++++++++++++++++++++++++++
>  include/asm-x86/kdebug.h |    9 ++++++

hm, could you please base this against x86.git#mm, see:

    http://redhat.com/~mingo/x86.git/README

thanks,

	Ingo

Re: [PATCH] x86: Add a list for custom page fault handlers.

[Peter Zijlstra]

On Sun, 2008-01-27 at 18:52 +0200, Pekka Paalanen wrote:

> +void unregister_page_fault_handler(struct pf_handler *old_pfh)
> +{
> +	might_sleep();
> +	spin_lock(&pf_handlers_writer);
> +	hlist_del_rcu(&old_pfh->hlist);
> +	spin_unlock(&pf_handlers_writer);
> +	synchronize_rcu();
> +}
> +EXPORT_SYMBOL_GPL(unregister_page_fault_handler);

Could you remove that sync_rcu() call and put in the comments that the
calling function is to take care of proper freeing.

sync_rcu() is a _very_ expensive call, and oft time it is not needed in
favour of call_rcu().

Re: [PATCH] x86: Add a list for custom page fault handlers.

[Harvey Harrison]

From: Pekka Paalanen <pq@iki.fi>

Provides kernel modules a way to register custom page fault handlers.
On every page fault, except those handled in vmalloc_fault(), this will
call a list of registered functions. The functions may handle the fault
and force do_page_fault() to return immediately.

This functionality is similar to the now removed page fault notifiers.
Custom page fault handlers are used by debugging and reverse engineering
tools. Mmio-trace is one such tool and a patch to add it into the tree
will follow.

The custom page fault handlers are called from the exact same points in
do_page_fault() as the page fault notifiers were.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
Signed-off-by: Harvey Harrison <harvey.harrison@gmail.com>
---
Ingo, ported Pekka's patch to current x86.git for review.  I have not
incorporated the RCU comments from Peter Zjistra, but I have folded
some of the static functions into on handle_custom_pf function that
will match the handle_kprobe_fault helper that will be ready for
RC1.

Harvey

 arch/x86/Kconfig.debug |    9 ++++++++
 arch/x86/mm/fault.c    |   51
++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 0 deletions(-)

diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 2e1e3af..9b44bc5 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -225,4 +225,13 @@ config CPA_DEBUG
 	help
 	  Do change_page_attr self tests at boot.
 
+config PAGE_FAULT_HANDLERS
+	bool "Custom page fault handlers"
+	depends on DEBUG_KERNEL
+	help
+	  Allow the use of custom page fault handlers. A kernel module may
+	  register a function that is called on every page fault not handled
+	  for vmalloc. Custom handlers are used by some debugging and reverse
+	  engineering tools.
+
 endmenu
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index e28cc52..c6c8164 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -49,6 +49,54 @@
 #define PF_RSVD		(1<<3)
 #define PF_INSTR	(1<<4)
 
+#ifdef CONFIG_PAGE_FAULT_HANDLERS
+static HLIST_HEAD(pf_handlers); /* protected by RCU */
+static DEFINE_SPINLOCK(pf_handlers_writer);
+
+void register_page_fault_handler(struct pf_handler *new_pfh)
+{
+	spin_lock(&pf_handlers_writer);
+	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
+	spin_unlock(&pf_handlers_writer);
+}
+EXPORT_SYMBOL_GPL(register_page_fault_handler);
+
+void unregister_page_fault_handler(struct pf_handler *old_pfh)
+{
+	might_sleep();
+	spin_lock(&pf_handlers_writer);
+	hlist_del_rcu(&old_pfh->hlist);
+	spin_unlock(&pf_handlers_writer);
+	synchronize_rcu();
+}
+EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
+#endif
+
+/* returns non-zero if do_page_fault() should return */
+static int handle_custom_pf(struct pt_regs *regs, unsigned long
error_code,
+			    unsigned long address)
+{
+#ifdef CONFIG_PAGE_FAULT_HANDLERS
+	int ret = 0;
+	struct pf_handler *cur;
+	struct hlist_node *ncur;
+
+	if (hlist_empty(&pf_handlers))
+		return 0;
+
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
+		ret = cur->handler(regs, error_code, address);
+		if (ret)
+			break;
+	}
+	rcu_read_unlock();
+	return ret;
+#else
+	return 0;
+#endif
+}
+
 static inline int notify_page_fault(struct pt_regs *regs)
 {
 #ifdef CONFIG_KPROBES
@@ -588,6 +636,9 @@ void __kprobes do_page_fault(struct pt_regs *regs,
unsigned long error_code)
 	if (notify_page_fault(regs))
 		return;
 
+	if (handle_custom_pf(regs, error_code, address))
+		return;
+
 	/*
 	 * We fault-in kernel-space virtual memory on-demand. The
 	 * 'reference' page table is init_mm.pgd.
-- 
1.5.4.rc4.1142.gf5a97

Re: [PATCH] x86: Add a list for custom page fault handlers.

[Harvey Harrison]

From: Pekka Paalanen <pq@iki.fi>

Provides kernel modules a way to register custom page fault handlers.
On every page fault, except those handled in vmalloc_fault(), this will
call a list of registered functions. The functions may handle the fault
and force do_page_fault() to return immediately.

This functionality is similar to the now removed page fault notifiers.
Custom page fault handlers are used by debugging and reverse engineering
tools. Mmio-trace is one such tool and a patch to add it into the tree
will follow.

The custom page fault handlers are called from the exact same points in
do_page_fault() as the page fault notifiers were.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
Signed-off-by: Harvey Harrison <harvey.harrison@gmail.com>
---
Sorry, attached the wrong version to my last message missing the
kdebug.h hunk.  This is still just a straight port to current x86.git.

 arch/x86/Kconfig.debug   |    9 ++++++++
 arch/x86/mm/fault.c      |   51 ++++++++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/kdebug.h |    8 +++++++
 3 files changed, 68 insertions(+), 0 deletions(-)

diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 2e1e3af..9b44bc5 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -225,4 +225,13 @@ config CPA_DEBUG
 	help
 	  Do change_page_attr self tests at boot.
 
+config PAGE_FAULT_HANDLERS
+	bool "Custom page fault handlers"
+	depends on DEBUG_KERNEL
+	help
+	  Allow the use of custom page fault handlers. A kernel module may
+	  register a function that is called on every page fault not handled
+	  for vmalloc. Custom handlers are used by some debugging and reverse
+	  engineering tools.
+
 endmenu
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index e28cc52..c6c8164 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -49,6 +49,54 @@
 #define PF_RSVD		(1<<3)
 #define PF_INSTR	(1<<4)
 
+#ifdef CONFIG_PAGE_FAULT_HANDLERS
+static HLIST_HEAD(pf_handlers); /* protected by RCU */
+static DEFINE_SPINLOCK(pf_handlers_writer);
+
+void register_page_fault_handler(struct pf_handler *new_pfh)
+{
+	spin_lock(&pf_handlers_writer);
+	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
+	spin_unlock(&pf_handlers_writer);
+}
+EXPORT_SYMBOL_GPL(register_page_fault_handler);
+
+void unregister_page_fault_handler(struct pf_handler *old_pfh)
+{
+	might_sleep();
+	spin_lock(&pf_handlers_writer);
+	hlist_del_rcu(&old_pfh->hlist);
+	spin_unlock(&pf_handlers_writer);
+	synchronize_rcu();
+}
+EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
+#endif
+
+/* returns non-zero if do_page_fault() should return */
+static int handle_custom_pf(struct pt_regs *regs, unsigned long error_code,
+			    unsigned long address)
+{
+#ifdef CONFIG_PAGE_FAULT_HANDLERS
+	int ret = 0;
+	struct pf_handler *cur;
+	struct hlist_node *ncur;
+
+	if (hlist_empty(&pf_handlers))
+		return 0;
+
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
+		ret = cur->handler(regs, error_code, address);
+		if (ret)
+			break;
+	}
+	rcu_read_unlock();
+	return ret;
+#else
+	return 0;
+#endif
+}
+
 static inline int notify_page_fault(struct pt_regs *regs)
 {
 #ifdef CONFIG_KPROBES
@@ -588,6 +636,9 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 	if (notify_page_fault(regs))
 		return;
 
+	if (handle_custom_pf(regs, error_code, address))
+		return;
+
 	/*
 	 * We fault-in kernel-space virtual memory on-demand. The
 	 * 'reference' page table is init_mm.pgd.
diff --git a/include/asm-x86/kdebug.h b/include/asm-x86/kdebug.h
index dd442a1..ba03368 100644
--- a/include/asm-x86/kdebug.h
+++ b/include/asm-x86/kdebug.h
@@ -35,4 +35,12 @@ extern void dump_pagetable(unsigned long);
 extern unsigned long oops_begin(void);
 extern void oops_end(unsigned long, struct pt_regs *, int signr);
 
+struct pf_handler {
+	struct hlist_node hlist;
+	int (*handler)(struct pt_regs *regs, unsigned long error_code,
+		       unsigned long address);
+};
+
+extern void register_page_fault_handler(struct pf_handler *new_pfh);
+extern void unregister_page_fault_handler(struct pf_handler *old_pfh);
 #endif
-- 
1.5.4.rc4.1142.gf5a97

Re: [PATCH] x86: Add a list for custom page fault handlers.

[Pekka Paalanen]

On Tue, 29 Jan 2008 18:34:13 -0800
Harvey Harrison <harvey.harrison@gmail.com> wrote:

> Sorry, attached the wrong version to my last message missing the
> kdebug.h hunk.  This is still just a straight port to current x86.git.

Wow, thanks.

I have to say I already did this for myself, but I had to
change too many things to simply resubmit without testing. And it turned
out porting mmiotrace to x86/mm was a non-trivial task. I have not
been able to test yet even though I have been working on it every night.
Today I will see if things work for me, and hopefully also for someone else.
I might send new patches today or tomorrow, if all goes well.

btw. I have a bad feeling about global_flush_tlb() disappearing, it was used
in mmiotrace code, and I don't know what to use instead. The obvious
replacement for it is a static function elsewhere. Currently I am simply
hoping it was a redundant leftover from the time when we had trouble making
mmiotrace stable.

To be honest, I haven't yet had the time to educate myself about what is
TLB.

> diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
> index e28cc52..c6c8164 100644
> --- a/arch/x86/mm/fault.c
> +++ b/arch/x86/mm/fault.c
> @@ -49,6 +49,54 @@
>  #define PF_RSVD		(1<<3)
>  #define PF_INSTR	(1<<4)
>  
> +#ifdef CONFIG_PAGE_FAULT_HANDLERS
> +static HLIST_HEAD(pf_handlers); /* protected by RCU */
> +static DEFINE_SPINLOCK(pf_handlers_writer);
> +
> +void register_page_fault_handler(struct pf_handler *new_pfh)
> +{
> +	spin_lock(&pf_handlers_writer);
> +	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
> +	spin_unlock(&pf_handlers_writer);

Here I had to change to irqsave/irqrestore versions, as I found a possible
lock dependency issue. Lockdep debugging code pointed it out. I'm not sure
I fixed it completely, but using spin_lock_irqsave() is a safe bet.

> +}
> +EXPORT_SYMBOL_GPL(register_page_fault_handler);
> +
> +void unregister_page_fault_handler(struct pf_handler *old_pfh)
> +{
> +	might_sleep();
> +	spin_lock(&pf_handlers_writer);
> +	hlist_del_rcu(&old_pfh->hlist);
> +	spin_unlock(&pf_handlers_writer);
> +	synchronize_rcu();
> +}

And here. And I also removed sync RCU and might_sleep().

> +EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
> +#endif
> +
> +/* returns non-zero if do_page_fault() should return */
> +static int handle_custom_pf(struct pt_regs *regs, unsigned long error_code,
> +			    unsigned long address)
> +{
> +#ifdef CONFIG_PAGE_FAULT_HANDLERS
> +	int ret = 0;
> +	struct pf_handler *cur;
> +	struct hlist_node *ncur;
> +
> +	if (hlist_empty(&pf_handlers))
> +		return 0;

This looks prettier than my code. I might change my code to this.

> +	rcu_read_lock();
> +	hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
> +		ret = cur->handler(regs, error_code, address);
> +		if (ret)
> +			break;
> +	}
> +	rcu_read_unlock();
> +	return ret;
> +#else
> +	return 0;
> +#endif
> +}
> +
>  static inline int notify_page_fault(struct pt_regs *regs)
>  {
>  #ifdef CONFIG_KPROBES
> @@ -588,6 +636,9 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
>  	if (notify_page_fault(regs))
>  		return;
>  
> +	if (handle_custom_pf(regs, error_code, address))
> +		return;
> +

This is the non-trivial change that I really want to test before
submitting again. The call site of the handlers changes, and vmalloc
related faults reach the custom handlers. I don't think this would be a
problem for mmiotrace, but I don't know enough to be sure.
I guess using vmalloc'd memory from a page fault handler would be doomed,
anyway, so no point worrying about it, right?

if (unlikely(handle_custom_pf(regs, error_code, address)))
Maybe like this, even?

It is still supposedly making a function call every time, at least
when CONFIG_PAGE_FAULT_HANDLERS is defined. Would it have a non-negligible
impact on performance when no custom handlers are registered?

My original version had the separate inline function to check if the list
is empty. Which one is preferrable?

I'm hoping distros would ship some kernels with PAGE_FAULT_HANDLERS
enabled as it would make contributors' lives easier. IIRC some
already have enabled RELAY and DEBUG_FS for mmiotrace. This is why I
think keeping the performance impact to the minimum here is important,
for the case of no custom handlers registered.


Thanks,

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH] x86: Add a list for custom page fault handlers.

[Arjan van de Ven]

On Sun, 27 Jan 2008 18:52:38 +0200
Pekka Paalanen <pq@iki.fi> wrote:

> From: Pekka Paalanen <pq@iki.fi>
> 
> Provides kernel modules a way to register custom page fault handlers.
> On every page fault, except those handled in vmalloc_fault(), this
> will call a list of registered functions. The functions may handle
> the fault and force do_page_fault() to return immediately.
> 
> This functionality is similar to the now removed page fault notifiers.
> Custom page fault handlers are used by debugging and reverse
> engineering tools. Mmio-trace is one such tool and a patch to add it
> into the tree will follow.
> 
> The custom page fault handlers are called from the exact same points
> in do_page_fault() as the page fault notifiers were.

to be honest I don't like this approach.
I fully would like to see mmio tracing in the kernel, but please 
make the hook explicit and direct (eg call the mmio tracing function,
not some list) instead...


-- 
If you want to reach me at my work email, use arjan@linux.intel.com
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org

Re: [RFC PATCH] x86: mmiotrace - trace memory mapped IO

[Pekka Paalanen]

On Sun, 27 Jan 2008 19:55:36 +0200
Pekka Paalanen <pq@iki.fi> wrote:

> The following is a copy of the patch description:
...
> Mmiotrace works by wrapping the ioremap family of kernel functions and
> marking the returned pages as not present. Access to the IO memory
> triggers a page fault, which will be handled by mmiotrace's custom page
> fault handler. This will single-step the faulted instruction with the
> MMIO page marked as present. Access logs are directed to user space via
> relay and debug_fs.

Hi,

a test user found a problem with mmiotrace (ported to x86/mm git tree).
The symbol 'init_mm' is undefined, but required when building the 32-bit
mmiotrace kernel module.

commit 3abf024d2abb79614d8c4cb25a70d5596f77d0ad
Author: Thomas Gleixner <tglx@linutronix.de>
Date:   Wed Jan 30 13:30:28 2008 +0100

    x86: nuke a ton of unused exports

This commit removes EXPORT_SYMBOL(init_mm). Apparently mmiotrace requires
this symbol, but I do not know why. I do not use init_mm myself, so
apparently some function I call is inlined and requires it. Page table or
attribute handling stuff maybe?

Is it possible to simply put back this export or should I dig deeper into
why it seems to be required?
I can provide the simple patch to put it back in my next patch set, if
this is the right solution.

I will post the patch set after this issue is solved. In the mean time,
for the interested, the current patches are here:
http://jumi.lut.fi/~paalanen/scratch/mmio25/


Ps. I tested x86/mm version of mmiotrace on my 64-bit system and it seems
to work ok. I had to write a simple test module because I did not have any
drivers that a) called ioremap, b) were not broken on x86/mm, and c) I
had the hardware to actually use it. So this testing is very light.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH] x86: Add a list for custom page fault handlers.

[Ingo Molnar]

* Pekka Paalanen <pq@iki.fi> wrote:

> >  static inline int notify_page_fault(struct pt_regs *regs)
> >  {
> >  #ifdef CONFIG_KPROBES
> > @@ -588,6 +636,9 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
> >  	if (notify_page_fault(regs))
> >  		return;
> >  
> > +	if (handle_custom_pf(regs, error_code, address))
> > +		return;
> > +
> 
> This is the non-trivial change that I really want to test before 
> submitting again. The call site of the handlers changes, and vmalloc 
> related faults reach the custom handlers. I don't think this would be 
> a problem for mmiotrace, but I don't know enough to be sure. I guess 
> using vmalloc'd memory from a page fault handler would be doomed, 
> anyway, so no point worrying about it, right?

could you please send us a patch for the whole mmiotrace kernel-side 
feature, so that we can have a look at the general structure of this? 
(and the interaction with change_page_attr(), etc.) Even if it's not 
functional (and wont even build/boot) at the moment. Thanks,

	Ingo

[PATCH v2] x86: Add a list for custom page fault handlers.

[Pekka Paalanen]

> could you please send us a patch for the whole mmiotrace kernel-side 
> feature, so that we can have a look at the general structure of this? 
> (and the interaction with change_page_attr(), etc.) Even if it's not 
> functional (and wont even build/boot) at the moment. Thanks,

Very well, first the revised custom page fault handler patch.
Changes since the previous submit:
- use spin_lock_irqsave instead of spin_lock
- Harvey Harrison's clean-up with the #ifdefs
- handler call site moved earlier
- remove sync RCU call

I'm not aware of any functional problems with this one.

Arjan, you said you don't like this. May I ask why?
This is most convinient for mmiotrace as it is meant to be a module.
I'm also using this as an excuse to let other people to get into the
page fault handler with their out-of-tree-today modules.


>From adeb0f4f95f176d97c1157031b5dd01f49d4cb66 Mon Sep 17 00:00:00 2001
From: Pekka Paalanen <pq@iki.fi>
Date: Wed, 30 Jan 2008 21:49:52 +0200
Subject: [PATCH] x86: Add a list for custom page fault handlers.

Provides kernel modules a way to register custom page fault handlers.
On every page fault this will call a list of registered functions. The
functions may handle the fault and force do_page_fault() to return
immediately.

This functionality is similar to the now removed page fault notifiers.
Custom page fault handlers are used by debugging and reverse engineering
tools. Mmiotrace is one such tool and a patch to add it into the tree
will follow.

The custom page fault handlers are called earlier in do_page_fault()
than the page fault notifiers were.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---
 arch/x86/Kconfig.debug   |    8 ++++++
 arch/x86/mm/fault.c      |   56 ++++++++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/kdebug.h |    9 +++++++
 3 files changed, 73 insertions(+), 0 deletions(-)

diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 2e1e3af..1185dcc 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -138,6 +138,14 @@ config IOMMU_LEAK
 	  Add a simple leak tracer to the IOMMU code. This is useful when you
 	  are debugging a buggy device driver that leaks IOMMU mappings.
 
+config PAGE_FAULT_HANDLERS
+	bool "Custom page fault handlers"
+	depends on DEBUG_KERNEL
+	help
+	  Allow the use of custom page fault handlers. A kernel module may
+	  register a function that is called on every page fault. Custom
+	  handlers are used by some debugging and reverse engineering tools.
+
 #
 # IO delay types:
 #
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index e28cc52..11feebd 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -49,6 +49,60 @@
 #define PF_RSVD		(1<<3)
 #define PF_INSTR	(1<<4)
 
+#ifdef CONFIG_PAGE_FAULT_HANDLERS
+static HLIST_HEAD(pf_handlers); /* protected by RCU */
+static DEFINE_SPINLOCK(pf_handlers_writer);
+
+void register_page_fault_handler(struct pf_handler *new_pfh)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&pf_handlers_writer, flags);
+	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
+	spin_unlock_irqrestore(&pf_handlers_writer, flags);
+}
+EXPORT_SYMBOL_GPL(register_page_fault_handler);
+
+/**
+ * unregister_page_fault_handler:
+ * The caller must ensure @old_pfh is not in use anymore before freeing it.
+ * This function does not guarantee it. The list of handlers is protected by
+ * RCU, so you can do this by e.g. calling synchronize_rcu().
+ */
+void unregister_page_fault_handler(struct pf_handler *old_pfh)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&pf_handlers_writer, flags);
+	hlist_del_rcu(&old_pfh->hlist);
+	spin_unlock_irqrestore(&pf_handlers_writer, flags);
+}
+EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
+#endif
+
+/* returns non-zero if do_page_fault() should return */
+static int handle_custom_pf(struct pt_regs *regs, unsigned long error_code,
+							unsigned long address)
+{
+#ifdef CONFIG_PAGE_FAULT_HANDLERS
+	int ret = 0;
+	struct pf_handler *cur;
+	struct hlist_node *ncur;
+
+	if (hlist_empty(&pf_handlers))
+		return 0;
+
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
+		ret = cur->handler(regs, error_code, address);
+		if (ret)
+			break;
+	}
+	rcu_read_unlock();
+	return ret;
+#else
+	return 0;
+#endif
+}
+
 static inline int notify_page_fault(struct pt_regs *regs)
 {
 #ifdef CONFIG_KPROBES
@@ -587,6 +641,8 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 
 	if (notify_page_fault(regs))
 		return;
+	if (handle_custom_pf(regs, error_code, address))
+		return;
 
 	/*
 	 * We fault-in kernel-space virtual memory on-demand. The
diff --git a/include/asm-x86/kdebug.h b/include/asm-x86/kdebug.h
index dd442a1..7ae2118 100644
--- a/include/asm-x86/kdebug.h
+++ b/include/asm-x86/kdebug.h
@@ -35,4 +35,13 @@ extern void dump_pagetable(unsigned long);
 extern unsigned long oops_begin(void);
 extern void oops_end(unsigned long, struct pt_regs *, int signr);
 
+struct pf_handler {
+	struct hlist_node hlist;
+	int (*handler)(struct pt_regs *regs, unsigned long error_code,
+						unsigned long address);
+};
+
+extern void register_page_fault_handler(struct pf_handler *new_pfh);
+extern void unregister_page_fault_handler(struct pf_handler *old_pfh);
+
 #endif
-- 
1.5.3.7

Re: [PATCH v2] x86: Add a list for custom page fault handlers.

[Arjan van de Ven]

On Thu, 31 Jan 2008 18:02:53 +0200
Pekka Paalanen <pq@iki.fi> wrote:

> > could you please send us a patch for the whole mmiotrace
> > kernel-side feature, so that we can have a look at the general
> > structure of this? (and the interaction with change_page_attr(),
> > etc.) Even if it's not functional (and wont even build/boot) at the
> > moment. Thanks,
> 
> Very well, first the revised custom page fault handler patch.
> Changes since the previous submit:
> - use spin_lock_irqsave instead of spin_lock
> - Harvey Harrison's clean-up with the #ifdefs
> - handler call site moved earlier
> - remove sync RCU call
> 
> I'm not aware of any functional problems with this one.
> 
> Arjan, you said you don't like this. May I ask why?

it's overhead that's rarely used, and worse, it suffers from the LSM disease:
it's a hook without visibility into the callers. Esp if there's only 1 user, a direct call
is not only faster, it also shows you what is going on when you're reading the code.
THe page fault stuff is tricky enough as it is without any "call arbitrary thing" hooks into
it

> This is most convinient for mmiotrace as it is meant to be a module.

that's not per se mutually exclusive, there's other cases like this already
(but even then, I don't want to go through hoops to make mmiotracing a module;
for me I can totally see it becoming a full mature member of the kernel, and if
that means a few things have to be changed in the normal kernel, or infrastructure
added, I'm totally ok with that)

> I'm also using this as an excuse to let other people to get into the
> page fault handler with their out-of-tree-today modules.

this is exactly the point.. it becomes a totally opaque thing, a random hook.


-- 
If you want to reach me at my work email, use arjan@linux.intel.com
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org

[RFC PATCH v2] x86: mmiotrace - trace memory mapped IO

[Pekka Paalanen]

> > could you please send us a patch for the whole mmiotrace kernel-side 
> > feature, so that we can have a look at the general structure of this? 
> > (and the interaction with change_page_attr(), etc.) Even if it's not 
> > functional (and wont even build/boot) at the moment. Thanks,

And here's mmiotrace. This works for me (amd64), but not for a test user
who has 32-bit x86. We have not had the chance to debug it yet.
The symptom is "everything locked up" after insmodding testmmiotrace.ko,
targeting the mid-point of his VRAM. And there's the "undefined symbol
init_mm" problem on 32-bit, but we just put the export back in (not
included in these patches).

No-one has tried this on SMP. I have a Core 2 Duo laptop where I will
test this when I get the chance.


>From 10000bf67af4900b75a56cb47726d7b63d11d7e8 Mon Sep 17 00:00:00 2001
From: Pekka Paalanen <pq@iki.fi>
Date: Wed, 30 Jan 2008 21:51:51 +0200
Subject: [PATCH] x86: mmiotrace - trace memory mapped IO

Mmiotrace is a tool for trapping memory mapped IO (MMIO) accesses within
the kernel. It is used for debugging and especially for reverse
engineering evil binary drivers.

Mmiotrace works by wrapping the ioremap family of kernel functions and
marking the returned pages as not present. Access to the IO memory
triggers a page fault, which will be handled by mmiotrace's custom page
fault handler. This will single-step the faulted instruction with the
MMIO page marked as present. Access logs are directed to user space via
relay and debug_fs.

This page fault approach is necessary, because binary drivers have
readl/writel etc. calls inlined and therefore extremely difficult to
trap with with e.g. kprobes.

This patch depends on the custom page fault handlers patch.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---
 arch/x86/Kconfig.debug                    |   28 ++
 arch/x86/kernel/Makefile                  |    2 +
 arch/x86/kernel/mmiotrace/Makefile        |    4 +
 arch/x86/kernel/mmiotrace/kmmio.c         |  391 +++++++++++++++++++++
 arch/x86/kernel/mmiotrace/kmmio.h         |   58 ++++
 arch/x86/kernel/mmiotrace/mmio-mod.c      |  527 +++++++++++++++++++++++++++++
 arch/x86/kernel/mmiotrace/pf_in.c         |  489 ++++++++++++++++++++++++++
 arch/x86/kernel/mmiotrace/pf_in.h         |   39 +++
 arch/x86/kernel/mmiotrace/testmmiotrace.c |   77 +++++
 include/linux/mmiotrace.h                 |   62 ++++
 10 files changed, 1677 insertions(+), 0 deletions(-)
 create mode 100644 arch/x86/kernel/mmiotrace/Makefile
 create mode 100644 arch/x86/kernel/mmiotrace/kmmio.c
 create mode 100644 arch/x86/kernel/mmiotrace/kmmio.h
 create mode 100644 arch/x86/kernel/mmiotrace/mmio-mod.c
 create mode 100644 arch/x86/kernel/mmiotrace/pf_in.c
 create mode 100644 arch/x86/kernel/mmiotrace/pf_in.h
 create mode 100644 arch/x86/kernel/mmiotrace/testmmiotrace.c
 create mode 100644 include/linux/mmiotrace.h

diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 1185dcc..486dca5 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -146,6 +146,34 @@ config PAGE_FAULT_HANDLERS
 	  register a function that is called on every page fault. Custom
 	  handlers are used by some debugging and reverse engineering tools.
 
+config MMIOTRACE
+	tristate "Memory mapped IO tracing"
+	depends on DEBUG_KERNEL && m
+	depends on PAGE_FAULT_HANDLERS && RELAY && DEBUG_FS
+	default n
+	help
+	  This will build a kernel module called mmiotrace.
+
+	  Mmiotrace traces Memory Mapped I/O access and is meant for debugging
+	  and reverse engineering. The kernel module offers wrapped
+	  versions of the ioremap family of functions. The driver to be traced
+	  must be modified to call these wrappers. A user space program is
+	  required to collect the MMIO data.
+
+	  See http://nouveau.freedesktop.org/wiki/MmioTrace
+	  If you are not helping to develop drivers, say N.
+
+config MMIOTRACE_TEST
+	tristate "Test module for mmiotrace"
+	depends on MMIOTRACE && m
+	default n
+	help
+	  This is a dumb module for testing mmiotrace. It is very dangerous
+	  as it will write garbage to IO memory starting at a given address.
+	  However, it should be safe to use on e.g. unused portion of VRAM.
+
+	  Say N, unless you absolutely know what you are doing.
+
 #
 # IO delay types:
 #
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 7954e9d..469673f 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -61,6 +61,8 @@ obj-$(CONFIG_KGDB)		+= kgdb.o kgdb-jmp_$(BITS).o
 obj-$(CONFIG_VM86)		+= vm86_32.o
 obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
 
+obj-$(CONFIG_MMIOTRACE)		+= mmiotrace/
+
 obj-$(CONFIG_HPET_TIMER) 	+= hpet.o
 
 obj-$(CONFIG_K8_NB)		+= k8.o
diff --git a/arch/x86/kernel/mmiotrace/Makefile b/arch/x86/kernel/mmiotrace/Makefile
new file mode 100644
index 0000000..d6905f7
--- /dev/null
+++ b/arch/x86/kernel/mmiotrace/Makefile
@@ -0,0 +1,4 @@
+obj-$(CONFIG_MMIOTRACE) += mmiotrace.o
+mmiotrace-objs := pf_in.o kmmio.o mmio-mod.o
+
+obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o
diff --git a/arch/x86/kernel/mmiotrace/kmmio.c b/arch/x86/kernel/mmiotrace/kmmio.c
new file mode 100644
index 0000000..8ba48f9
--- /dev/null
+++ b/arch/x86/kernel/mmiotrace/kmmio.c
@@ -0,0 +1,391 @@
+/* Support for MMIO probes.
+ * Benfit many code from kprobes
+ * (C) 2002 Louis Zhuang <louis.zhuang@intel.com>.
+ *     2007 Alexander Eichner
+ *     2008 Pekka Paalanen <pq@iki.fi>
+ */
+
+#include <linux/version.h>
+#include <linux/spinlock.h>
+#include <linux/hash.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/uaccess.h>
+#include <linux/ptrace.h>
+#include <linux/preempt.h>
+#include <asm/io.h>
+#include <asm/cacheflush.h>
+#include <asm/errno.h>
+#include <asm/tlbflush.h>
+
+#include "kmmio.h"
+
+#define KMMIO_HASH_BITS 6
+#define KMMIO_TABLE_SIZE (1 << KMMIO_HASH_BITS)
+#define KMMIO_PAGE_HASH_BITS 4
+#define KMMIO_PAGE_TABLE_SIZE (1 << KMMIO_PAGE_HASH_BITS)
+
+struct kmmio_context {
+	struct kmmio_fault_page *fpage;
+	struct kmmio_probe *probe;
+	unsigned long saved_flags;
+	int active;
+};
+
+static int kmmio_page_fault(struct pt_regs *regs, unsigned long error_code,
+						unsigned long address);
+static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val,
+								void *args);
+
+static DEFINE_SPINLOCK(kmmio_lock);
+
+/* These are protected by kmmio_lock */
+unsigned int kmmio_count;
+static unsigned int handler_registered;
+static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE];
+static LIST_HEAD(kmmio_probes);
+
+static struct kmmio_context kmmio_ctx[NR_CPUS];
+
+static struct pf_handler kmmio_pf_hook = {
+	.handler = kmmio_page_fault
+};
+
+static struct notifier_block nb_die = {
+	.notifier_call = kmmio_die_notifier
+};
+
+int init_kmmio(void)
+{
+	int i;
+	for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++)
+		INIT_LIST_HEAD(&kmmio_page_table[i]);
+
+	register_die_notifier(&nb_die);
+	return 0;
+}
+
+void cleanup_kmmio(void)
+{
+	/*
+	 * Assume the following have been already cleaned by calling
+	 * unregister_kmmio_probe() appropriately:
+	 * kmmio_page_table, kmmio_probes
+	 */
+	if (handler_registered) {
+		unregister_page_fault_handler(&kmmio_pf_hook);
+		synchronize_rcu();
+	}
+	unregister_die_notifier(&nb_die);
+}
+
+/*
+ * this is basically a dynamic stabbing problem:
+ * Could use the existing prio tree code or
+ * Possible better implementations:
+ * The Interval Skip List: A Data Structure for Finding All Intervals That
+ * Overlap a Point (might be simple)
+ * Space Efficient Dynamic Stabbing with Fast Queries - Mikkel Thorup
+ */
+/* Get the kmmio at this addr (if any). You must be holding kmmio_lock. */
+static struct kmmio_probe *get_kmmio_probe(unsigned long addr)
+{
+	struct kmmio_probe *p;
+	list_for_each_entry(p, &kmmio_probes, list) {
+		if (addr >= p->addr && addr <= (p->addr + p->len))
+			return p;
+	}
+	return NULL;
+}
+
+static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page)
+{
+	struct list_head *head, *tmp;
+
+	page &= PAGE_MASK;
+	head = &kmmio_page_table[hash_long(page, KMMIO_PAGE_HASH_BITS)];
+	list_for_each(tmp, head) {
+		struct kmmio_fault_page *p
+			= list_entry(tmp, struct kmmio_fault_page, list);
+		if (p->page == page)
+			return p;
+	}
+
+	return NULL;
+}
+
+static void arm_kmmio_fault_page(unsigned long page, int *large)
+{
+	unsigned long address = page & PAGE_MASK;
+	pgd_t *pgd = pgd_offset_k(address);
+	pud_t *pud = pud_offset(pgd, address);
+	pmd_t *pmd = pmd_offset(pud, address);
+	pte_t *pte = pte_offset_kernel(pmd, address);
+
+	if (pmd_large(*pmd)) {
+		set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_PRESENT));
+		if (large)
+			*large = 1;
+	} else {
+		set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
+	}
+
+	__flush_tlb_one(page);
+}
+
+static void disarm_kmmio_fault_page(unsigned long page, int *large)
+{
+	unsigned long address = page & PAGE_MASK;
+	pgd_t *pgd = pgd_offset_k(address);
+	pud_t *pud = pud_offset(pgd, address);
+	pmd_t *pmd = pmd_offset(pud, address);
+	pte_t *pte = pte_offset_kernel(pmd, address);
+
+	if (large && *large) {
+		set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_PRESENT));
+		*large = 0;
+	} else {
+		set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
+	}
+
+	__flush_tlb_one(page);
+}
+
+/*
+ * Interrupts are disabled on entry as trap3 is an interrupt gate
+ * and they remain disabled thorough out this function.
+ */
+static int kmmio_handler(struct pt_regs *regs, unsigned long addr)
+{
+	struct kmmio_context *ctx;
+	int cpu;
+
+	/*
+	 * Preemption is now disabled to prevent process switch during
+	 * single stepping. We can only handle one active kmmio trace
+	 * per cpu, so ensure that we finish it before something else
+	 * gets to run.
+	 *
+	 * XXX what if an interrupt occurs between returning from
+	 * do_page_fault() and entering the single-step exception handler?
+	 * And that interrupt triggers a kmmio trap?
+	 */
+	preempt_disable();
+	cpu = smp_processor_id();
+	ctx = &kmmio_ctx[cpu];
+
+	/* interrupts disabled and CPU-local data => atomicity guaranteed. */
+	if (ctx->active) {
+		/*
+		 * This avoids a deadlock with kmmio_lock.
+		 * If this page fault really was due to kmmio trap,
+		 * all hell breaks loose.
+		 */
+		printk(KERN_EMERG "mmiotrace: recursive probe hit on CPU %d, "
+					"for address %lu. Ignoring.\n",
+					cpu, addr);
+		goto no_kmmio;
+	}
+	ctx->active++;
+
+	/*
+	 * Acquire the kmmio lock to prevent changes affecting
+	 * get_kmmio_fault_page() and get_kmmio_probe(), since we save their
+	 * returned pointers.
+	 * The lock is released in post_kmmio_handler().
+	 * XXX: could/should get_kmmio_*() be using RCU instead of spinlock?
+	 */
+	spin_lock(&kmmio_lock);
+
+	ctx->fpage = get_kmmio_fault_page(addr);
+	if (!ctx->fpage) {
+		/* this page fault is not caused by kmmio */
+		goto no_kmmio_locked;
+	}
+
+	ctx->probe = get_kmmio_probe(addr);
+	ctx->saved_flags = (regs->flags & (TF_MASK|IF_MASK));
+
+	if (ctx->probe && ctx->probe->pre_handler)
+		ctx->probe->pre_handler(ctx->probe, regs, addr);
+
+	regs->flags |= TF_MASK;
+	regs->flags &= ~IF_MASK;
+
+	/* We hold lock, now we set present bit in PTE and single step. */
+	disarm_kmmio_fault_page(ctx->fpage->page, NULL);
+
+	return 1;
+
+no_kmmio_locked:
+	spin_unlock(&kmmio_lock);
+	ctx->active--;
+no_kmmio:
+	preempt_enable_no_resched();
+	/* page fault not handled by kmmio */
+	return 0;
+}
+
+/*
+ * Interrupts are disabled on entry as trap1 is an interrupt gate
+ * and they remain disabled thorough out this function.
+ * And we hold kmmio lock.
+ */
+static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
+{
+	int cpu = smp_processor_id();
+	struct kmmio_context *ctx = &kmmio_ctx[cpu];
+
+	if (!ctx->active)
+		return 0;
+
+	if (ctx->probe && ctx->probe->post_handler)
+		ctx->probe->post_handler(ctx->probe, condition, regs);
+
+	arm_kmmio_fault_page(ctx->fpage->page, NULL);
+
+	regs->flags &= ~TF_MASK;
+	regs->flags |= ctx->saved_flags;
+
+	/* These were acquired in kmmio_handler(). */
+	ctx->active--;
+	spin_unlock(&kmmio_lock);
+	preempt_enable_no_resched();
+
+	/*
+	 * if somebody else is singlestepping across a probe point, flags
+	 * will have TF set, in which case, continue the remaining processing
+	 * of do_debug, as if this is not a probe hit.
+	 */
+	if (regs->flags & TF_MASK)
+		return 0;
+
+	return 1;
+}
+
+static int add_kmmio_fault_page(unsigned long page)
+{
+	struct kmmio_fault_page *f;
+
+	page &= PAGE_MASK;
+	f = get_kmmio_fault_page(page);
+	if (f) {
+		f->count++;
+		return 0;
+	}
+
+	f = kmalloc(sizeof(*f), GFP_ATOMIC);
+	if (!f)
+		return -1;
+
+	f->count = 1;
+	f->page = page;
+	list_add(&f->list,
+		 &kmmio_page_table[hash_long(f->page, KMMIO_PAGE_HASH_BITS)]);
+
+	arm_kmmio_fault_page(f->page, NULL);
+
+	return 0;
+}
+
+static void release_kmmio_fault_page(unsigned long page)
+{
+	struct kmmio_fault_page *f;
+
+	page &= PAGE_MASK;
+	f = get_kmmio_fault_page(page);
+	if (!f)
+		return;
+
+	f->count--;
+	if (!f->count) {
+		disarm_kmmio_fault_page(f->page, NULL);
+		list_del(&f->list);
+	}
+}
+
+int register_kmmio_probe(struct kmmio_probe *p)
+{
+	int ret = 0;
+	unsigned long size = 0;
+
+	spin_lock_irq(&kmmio_lock);
+	kmmio_count++;
+	if (get_kmmio_probe(p->addr)) {
+		ret = -EEXIST;
+		goto out;
+	}
+	list_add(&p->list, &kmmio_probes);
+	/*printk("adding fault pages...\n");*/
+	while (size < p->len) {
+		if (add_kmmio_fault_page(p->addr + size))
+			printk(KERN_ERR "mmio: Unable to set page fault.\n");
+		size += PAGE_SIZE;
+	}
+
+	if (!handler_registered) {
+		register_page_fault_handler(&kmmio_pf_hook);
+		handler_registered++;
+	}
+
+out:
+	spin_unlock_irq(&kmmio_lock);
+	/*
+	 * XXX: What should I do here?
+	 * Here was a call to global_flush_tlb(), but it does not exist
+	 * anymore.
+	 */
+	return ret;
+}
+
+void unregister_kmmio_probe(struct kmmio_probe *p)
+{
+	unsigned long size = 0;
+
+	spin_lock_irq(&kmmio_lock);
+	while (size < p->len) {
+		release_kmmio_fault_page(p->addr + size);
+		size += PAGE_SIZE;
+	}
+	list_del(&p->list);
+	kmmio_count--;
+	spin_unlock_irq(&kmmio_lock);
+}
+
+/*
+ * According to 2.6.20, mainly x86_64 arch:
+ * This is being called from do_page_fault(), via the page fault notifier
+ * chain. The chain is called for both user space faults and kernel space
+ * faults (address >= TASK_SIZE64), except not on faults serviced by
+ * vmalloc_fault().
+ *
+ * We may be in an interrupt or a critical section. Also prefecthing may
+ * trigger a page fault. We may be in the middle of process switch.
+ * The page fault hook functionality has put us inside RCU read lock.
+ *
+ * Local interrupts are disabled, so preemption cannot happen.
+ * Do not enable interrupts, do not sleep, and watch out for other CPUs.
+ */
+static int kmmio_page_fault(struct pt_regs *regs, unsigned long error_code,
+						unsigned long address)
+{
+	if (is_kmmio_active())
+		if (kmmio_handler(regs, address) == 1)
+			return -1;
+	return 0;
+}
+
+static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val,
+								void *args)
+{
+	struct die_args *arg = args;
+
+	if (val == DIE_DEBUG)
+		if (post_kmmio_handler(arg->err, arg->regs) == 1)
+			return NOTIFY_STOP;
+
+	return NOTIFY_DONE;
+}
diff --git a/arch/x86/kernel/mmiotrace/kmmio.h b/arch/x86/kernel/mmiotrace/kmmio.h
new file mode 100644
index 0000000..85b7f68
--- /dev/null
+++ b/arch/x86/kernel/mmiotrace/kmmio.h
@@ -0,0 +1,58 @@
+#ifndef _LINUX_KMMIO_H
+#define _LINUX_KMMIO_H
+
+#include <linux/list.h>
+#include <linux/notifier.h>
+#include <linux/smp.h>
+#include <linux/types.h>
+#include <linux/ptrace.h>
+#include <linux/version.h>
+#include <linux/kdebug.h>
+
+struct kmmio_probe;
+struct kmmio_fault_page;
+struct pt_regs;
+
+typedef void (*kmmio_pre_handler_t)(struct kmmio_probe *,
+				struct pt_regs *, unsigned long addr);
+typedef void (*kmmio_post_handler_t)(struct kmmio_probe *,
+				unsigned long condition, struct pt_regs *);
+
+struct kmmio_probe {
+	struct list_head list;
+
+	/* start location of the probe point */
+	unsigned long addr;
+
+	/* length of the probe region */
+	unsigned long len;
+
+	 /* Called before addr is executed. */
+	kmmio_pre_handler_t pre_handler;
+
+	/* Called after addr is executed, unless... */
+	kmmio_post_handler_t post_handler;
+};
+
+struct kmmio_fault_page {
+	struct list_head list;
+
+	/* location of the fault page */
+	unsigned long page;
+
+	int count;
+};
+
+/* kmmio is active by some kmmio_probes? */
+static inline int is_kmmio_active(void)
+{
+	extern unsigned int kmmio_count;
+	return kmmio_count;
+}
+
+int init_kmmio(void);
+void cleanup_kmmio(void);
+int register_kmmio_probe(struct kmmio_probe *p);
+void unregister_kmmio_probe(struct kmmio_probe *p);
+
+#endif /* _LINUX_KMMIO_H */
diff --git a/arch/x86/kernel/mmiotrace/mmio-mod.c b/arch/x86/kernel/mmiotrace/mmio-mod.c
new file mode 100644
index 0000000..4a8c859
--- /dev/null
+++ b/arch/x86/kernel/mmiotrace/mmio-mod.c
@@ -0,0 +1,527 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) IBM Corporation, 2005
+ *               Jeff Muizelaar, 2006, 2007
+ *               Pekka Paalanen, 2008 <pq@iki.fi>
+ *
+ * Derived from the read-mod example from relay-examples by Tom Zanussi.
+ */
+#include <linux/module.h>
+#include <linux/relay.h>
+#include <linux/debugfs.h>
+#include <linux/proc_fs.h>
+#include <asm/io.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <asm/pgtable.h>
+#include <linux/mmiotrace.h>
+#include <asm/e820.h> /* for ISA_START_ADDRESS */
+
+#include "kmmio.h"
+#include "pf_in.h"
+
+/* This app's relay channel files will appear in /debug/mmio-trace */
+#define APP_DIR		"mmio-trace"
+/* the marker injection file in /proc */
+#define MARKER_FILE     "mmio-marker"
+
+#define MODULE_NAME     "mmiotrace"
+
+struct trap_reason {
+	unsigned long addr;
+	unsigned long ip;
+	enum reason_type type;
+	int active_traces;
+};
+
+static struct trap_reason pf_reason[NR_CPUS];
+static struct mm_io_header_rw cpu_trace[NR_CPUS];
+
+static struct file_operations mmio_fops = {
+	.owner = THIS_MODULE,
+};
+
+static const size_t subbuf_size = 256*1024;
+static struct rchan *chan;
+static struct dentry *dir;
+static int suspended;      /* XXX should this be per cpu? */
+static struct proc_dir_entry *proc_marker_file;
+
+/* module parameters */
+static unsigned int      n_subbufs = 32*4;
+static unsigned long filter_offset;
+static int                 notrace;
+static int               ISA_trace;
+static int                trace_pc;
+
+module_param(n_subbufs, uint, 0);
+module_param(filter_offset, ulong, 0);
+module_param(notrace, bool, 0);
+module_param(ISA_trace, bool, 0);
+module_param(trace_pc, bool, 0);
+
+MODULE_PARM_DESC(n_subbufs, "Number of 256kB buffers, default 128.");
+MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
+MODULE_PARM_DESC(notrace, "Disable actual MMIO tracing.");
+MODULE_PARM_DESC(ISA_trace, "Do not exclude the low ISA range.");
+MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
+
+static void record_timestamp(struct mm_io_header *header)
+{
+	struct timespec now;
+
+	getnstimeofday(&now);
+	header->sec = now.tv_sec;
+	header->nsec = now.tv_nsec;
+}
+
+/*
+ * Write callback for the /proc entry:
+ * Read a marker and write it to the mmio trace log
+ */
+static int write_marker(struct file *file, const char __user *buffer,
+					unsigned long count, void *data)
+{
+	char *event = NULL;
+	struct mm_io_header *headp;
+	int len = (count > 65535) ? 65535 : count;
+
+	event = kzalloc(sizeof(*headp) + len, GFP_KERNEL);
+	if (!event)
+		return -ENOMEM;
+
+	headp = (struct mm_io_header *)event;
+	headp->type = MMIO_MAGIC | (MMIO_MARKER << MMIO_OPCODE_SHIFT);
+	headp->data_len = len;
+	record_timestamp(headp);
+
+	if (copy_from_user(event + sizeof(*headp), buffer, len)) {
+		kfree(event);
+		return -EFAULT;
+	}
+
+	relay_write(chan, event, sizeof(*headp) + len);
+	kfree(event);
+	return len;
+}
+
+static void print_pte(unsigned long address)
+{
+	pgd_t *pgd = pgd_offset_k(address);
+	pud_t *pud = pud_offset(pgd, address);
+	pmd_t *pmd = pmd_offset(pud, address);
+	if (pmd_large(*pmd)) {
+		printk(KERN_EMERG MODULE_NAME ": 4MB pages are not "
+						"currently supported: %lx\n",
+						address);
+		BUG();
+	}
+	printk(KERN_DEBUG MODULE_NAME ": pte for 0x%lx: 0x%lx 0x%lx\n",
+		address,
+		pte_val(*pte_offset_kernel(pmd, address)),
+		pte_val(*pte_offset_kernel(pmd, address)) & _PAGE_PRESENT);
+}
+
+/*
+ * For some reason the pre/post pairs have been called in an
+ * unmatched order. Report and die.
+ */
+static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
+{
+	const unsigned long cpu = smp_processor_id();
+	printk(KERN_EMERG MODULE_NAME ": unexpected fault for address: %lx, "
+					"last fault for address: %lx\n",
+					addr, pf_reason[cpu].addr);
+	print_pte(addr);
+#ifdef __i386__
+	print_symbol(KERN_EMERG "faulting EIP is at %s\n", regs->ip);
+	print_symbol(KERN_EMERG "last faulting EIP was at %s\n",
+							pf_reason[cpu].ip);
+	printk(KERN_EMERG
+			"eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
+			regs->ax, regs->bx, regs->cx, regs->dx);
+	printk(KERN_EMERG
+			"esi: %08lx   edi: %08lx   ebp: %08lx   esp: %08lx\n",
+			regs->si, regs->di, regs->bp, regs->sp);
+#else
+	print_symbol(KERN_EMERG "faulting RIP is at %s\n", regs->ip);
+	print_symbol(KERN_EMERG "last faulting RIP was at %s\n",
+							pf_reason[cpu].ip);
+	printk(KERN_EMERG "rax: %016lx   rcx: %016lx   rdx: %016lx\n",
+					regs->ax, regs->cx, regs->dx);
+	printk(KERN_EMERG "rsi: %016lx   rdi: %016lx   "
+				"rbp: %016lx   rsp: %016lx\n",
+				regs->si, regs->di, regs->bp, regs->sp);
+#endif
+	BUG();
+}
+
+static void pre(struct kmmio_probe *p, struct pt_regs *regs,
+						unsigned long addr)
+{
+	const unsigned long cpu = smp_processor_id();
+	const unsigned long instptr = instruction_pointer(regs);
+	const enum reason_type type = get_ins_type(instptr);
+
+	/* it doesn't make sense to have more than one active trace per cpu */
+	if (pf_reason[cpu].active_traces)
+		die_kmmio_nesting_error(regs, addr);
+	else
+		pf_reason[cpu].active_traces++;
+
+	pf_reason[cpu].type = type;
+	pf_reason[cpu].addr = addr;
+	pf_reason[cpu].ip = instptr;
+
+	cpu_trace[cpu].header.type = MMIO_MAGIC;
+	cpu_trace[cpu].header.pid = 0;
+	cpu_trace[cpu].header.data_len = sizeof(struct mm_io_rw);
+	cpu_trace[cpu].rw.address = addr;
+
+	/*
+	 * Only record the program counter when requested.
+	 * It may taint clean-room reverse engineering.
+	 */
+	if (trace_pc)
+		cpu_trace[cpu].rw.pc = instptr;
+	else
+		cpu_trace[cpu].rw.pc = 0;
+
+	record_timestamp(&cpu_trace[cpu].header);
+
+	switch (type) {
+	case REG_READ:
+		cpu_trace[cpu].header.type |=
+			(MMIO_READ << MMIO_OPCODE_SHIFT) |
+			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
+		break;
+	case REG_WRITE:
+		cpu_trace[cpu].header.type |=
+			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
+			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
+		cpu_trace[cpu].rw.value = get_ins_reg_val(instptr, regs);
+		break;
+	case IMM_WRITE:
+		cpu_trace[cpu].header.type |=
+			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
+			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
+		cpu_trace[cpu].rw.value = get_ins_imm_val(instptr);
+		break;
+	default:
+		{
+			unsigned char *ip = (unsigned char *)instptr;
+			cpu_trace[cpu].header.type |=
+					(MMIO_UNKNOWN_OP << MMIO_OPCODE_SHIFT);
+			cpu_trace[cpu].rw.value = (*ip) << 16 |
+							*(ip + 1) << 8 |
+							*(ip + 2);
+		}
+	}
+}
+
+static void post(struct kmmio_probe *p, unsigned long condition,
+							struct pt_regs *regs)
+{
+	const unsigned long cpu = smp_processor_id();
+
+	/* this should always return the active_trace count to 0 */
+	pf_reason[cpu].active_traces--;
+	if (pf_reason[cpu].active_traces) {
+		printk(KERN_EMERG MODULE_NAME ": unexpected post handler");
+		BUG();
+	}
+
+	switch (pf_reason[cpu].type) {
+	case REG_READ:
+		cpu_trace[cpu].rw.value = get_ins_reg_val(pf_reason[cpu].ip,
+									regs);
+		break;
+	default:
+		break;
+	}
+	relay_write(chan, &cpu_trace[cpu], sizeof(struct mm_io_header_rw));
+}
+
+/*
+ * subbuf_start() relay callback.
+ *
+ * Defined so that we know when events are dropped due to the buffer-full
+ * condition.
+ */
+static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf,
+					void *prev_subbuf, size_t prev_padding)
+{
+	if (relay_buf_full(buf)) {
+		if (!suspended) {
+			suspended = 1;
+			printk(KERN_ERR MODULE_NAME
+						": cpu %d buffer full!!!\n",
+						smp_processor_id());
+		}
+		return 0;
+	} else if (suspended) {
+		suspended = 0;
+		printk(KERN_ERR MODULE_NAME
+					": cpu %d buffer no longer full.\n",
+					smp_processor_id());
+	}
+
+	return 1;
+}
+
+/* file_create() callback.  Creates relay file in debugfs. */
+static struct dentry *create_buf_file_handler(const char *filename,
+						struct dentry *parent,
+						int mode,
+						struct rchan_buf *buf,
+						int *is_global)
+{
+	struct dentry *buf_file;
+
+	mmio_fops.read = relay_file_operations.read;
+	mmio_fops.open = relay_file_operations.open;
+	mmio_fops.poll = relay_file_operations.poll;
+	mmio_fops.mmap = relay_file_operations.mmap;
+	mmio_fops.release = relay_file_operations.release;
+	mmio_fops.splice_read = relay_file_operations.splice_read;
+
+	buf_file = debugfs_create_file(filename, mode, parent, buf,
+								&mmio_fops);
+
+	return buf_file;
+}
+
+/* file_remove() default callback.  Removes relay file in debugfs. */
+static int remove_buf_file_handler(struct dentry *dentry)
+{
+	debugfs_remove(dentry);
+	return 0;
+}
+
+static struct rchan_callbacks relay_callbacks = {
+	.subbuf_start = subbuf_start_handler,
+	.create_buf_file = create_buf_file_handler,
+	.remove_buf_file = remove_buf_file_handler,
+};
+
+/*
+ * create_channel - creates channel /debug/APP_DIR/cpuXXX
+ * Returns channel on success, NULL otherwise
+ */
+static struct rchan *create_channel(unsigned size, unsigned n)
+{
+	return relay_open("cpu", dir, size, n, &relay_callbacks, NULL);
+}
+
+/* destroy_channel - destroys channel /debug/APP_DIR/cpuXXX */
+static void destroy_channel(void)
+{
+	if (chan) {
+		relay_close(chan);
+		chan = NULL;
+	}
+}
+
+struct remap_trace {
+	struct list_head list;
+	struct kmmio_probe probe;
+};
+static LIST_HEAD(trace_list);
+static DEFINE_SPINLOCK(trace_list_lock);
+
+static void do_ioremap_trace_core(unsigned long offset, unsigned long size,
+							void __iomem *addr)
+{
+	struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
+	struct mm_io_header_map event = {
+		.header = {
+			.type = MMIO_MAGIC |
+					(MMIO_PROBE << MMIO_OPCODE_SHIFT),
+			.sec = 0,
+			.nsec = 0,
+			.pid = 0,
+			.data_len = sizeof(struct mm_io_map)
+		},
+		.map = {
+			.phys = offset,
+			.addr = (unsigned long)addr,
+			.len  = size,
+			.pc   = 0
+		}
+	};
+	record_timestamp(&event.header);
+
+	*trace = (struct remap_trace) {
+		.probe = {
+			.addr = (unsigned long)addr,
+			.len = size,
+			.pre_handler = pre,
+			.post_handler = post,
+		}
+	};
+
+	relay_write(chan, &event, sizeof(event));
+	spin_lock(&trace_list_lock);
+	list_add_tail(&trace->list, &trace_list);
+	spin_unlock(&trace_list_lock);
+	if (!notrace)
+		register_kmmio_probe(&trace->probe);
+}
+
+static void ioremap_trace_core(unsigned long offset, unsigned long size,
+							void __iomem *addr)
+{
+	if ((filter_offset) && (offset != filter_offset))
+		return;
+
+	/* Don't trace the low PCI/ISA area, it's always mapped.. */
+	if (!ISA_trace && (offset < ISA_END_ADDRESS) &&
+					(offset + size > ISA_START_ADDRESS)) {
+		printk(KERN_NOTICE MODULE_NAME ": Ignoring map of low "
+						"PCI/ISA area (0x%lx-0x%lx)\n",
+						offset, offset + size);
+		return;
+	}
+	do_ioremap_trace_core(offset, size, addr);
+}
+
+void __iomem *ioremap_cache_trace(unsigned long offset, unsigned long size)
+{
+	void __iomem *p = ioremap_cache(offset, size);
+	printk(KERN_DEBUG MODULE_NAME ": ioremap_cache(0x%lx, 0x%lx) = %p\n",
+							offset, size, p);
+	ioremap_trace_core(offset, size, p);
+	return p;
+}
+EXPORT_SYMBOL(ioremap_cache_trace);
+
+void __iomem *ioremap_nocache_trace(unsigned long offset, unsigned long size)
+{
+	void __iomem *p = ioremap_nocache(offset, size);
+	printk(KERN_DEBUG MODULE_NAME ": ioremap_nocache(0x%lx, 0x%lx) = %p\n",
+							offset, size, p);
+	ioremap_trace_core(offset, size, p);
+	return p;
+}
+EXPORT_SYMBOL(ioremap_nocache_trace);
+
+void iounmap_trace(volatile void __iomem *addr)
+{
+	struct mm_io_header_map event = {
+		.header = {
+			.type = MMIO_MAGIC |
+				(MMIO_UNPROBE << MMIO_OPCODE_SHIFT),
+			.sec = 0,
+			.nsec = 0,
+			.pid = 0,
+			.data_len = sizeof(struct mm_io_map)
+		},
+		.map = {
+			.phys = 0,
+			.addr = (unsigned long)addr,
+			.len  = 0,
+			.pc   = 0
+		}
+	};
+	struct remap_trace *trace;
+	struct remap_trace *tmp;
+	printk(KERN_DEBUG MODULE_NAME ": Unmapping %p.\n", addr);
+	record_timestamp(&event.header);
+
+	spin_lock(&trace_list_lock);
+	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
+		if ((unsigned long)addr == trace->probe.addr) {
+			if (!notrace)
+				unregister_kmmio_probe(&trace->probe);
+			list_del(&trace->list);
+			kfree(trace);
+			break;
+		}
+	}
+	spin_unlock(&trace_list_lock);
+	relay_write(chan, &event, sizeof(event));
+	iounmap(addr);
+}
+EXPORT_SYMBOL(iounmap_trace);
+
+static void clear_trace_list(void)
+{
+	struct remap_trace *trace;
+	struct remap_trace *tmp;
+
+	spin_lock(&trace_list_lock);
+	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
+		printk(KERN_WARNING MODULE_NAME ": purging non-iounmapped "
+					"trace @0x%08lx, size 0x%lx.\n",
+					trace->probe.addr, trace->probe.len);
+		if (!notrace)
+			unregister_kmmio_probe(&trace->probe);
+		list_del(&trace->list);
+		kfree(trace);
+		break;
+	}
+	spin_unlock(&trace_list_lock);
+}
+
+static int __init init(void)
+{
+	if (n_subbufs < 2)
+		return -EINVAL;
+
+	dir = debugfs_create_dir(APP_DIR, NULL);
+	if (!dir) {
+		printk(KERN_ERR MODULE_NAME
+				": Couldn't create relay app directory.\n");
+		return -ENOMEM;
+	}
+
+	chan = create_channel(subbuf_size, n_subbufs);
+	if (!chan) {
+		debugfs_remove(dir);
+		printk(KERN_ERR MODULE_NAME
+				": relay app channel creation failed\n");
+		return -ENOMEM;
+	}
+
+	init_kmmio();
+
+	proc_marker_file = create_proc_entry(MARKER_FILE, 0, NULL);
+	if (proc_marker_file)
+		proc_marker_file->write_proc = write_marker;
+
+	printk(KERN_DEBUG MODULE_NAME ": loaded.\n");
+	if (notrace)
+		printk(KERN_DEBUG MODULE_NAME ": MMIO tracing disabled.\n");
+	if (ISA_trace)
+		printk(KERN_WARNING MODULE_NAME
+				": Warning! low ISA range will be traced.\n");
+	return 0;
+}
+
+static void __exit cleanup(void)
+{
+	printk(KERN_DEBUG MODULE_NAME ": unload...\n");
+	clear_trace_list();
+	cleanup_kmmio();
+	remove_proc_entry(MARKER_FILE, NULL);
+	destroy_channel();
+	if (dir)
+		debugfs_remove(dir);
+}
+
+module_init(init);
+module_exit(cleanup);
+MODULE_LICENSE("GPL");
diff --git a/arch/x86/kernel/mmiotrace/pf_in.c b/arch/x86/kernel/mmiotrace/pf_in.c
new file mode 100644
index 0000000..67ea520
--- /dev/null
+++ b/arch/x86/kernel/mmiotrace/pf_in.c
@@ -0,0 +1,489 @@
+/*
+ *  Fault Injection Test harness (FI)
+ *  Copyright (C) Intel Crop.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ *
+ */
+
+/*  $Id: pf_in.c,v 1.1.1.1 2002/11/12 05:56:32 brlock Exp $
+ *  Copyright by Intel Crop., 2002
+ *  Louis Zhuang (louis.zhuang@intel.com)
+ *
+ *  Bjorn Steinbrink (B.Steinbrink@gmx.de), 2007
+ */
+
+#include <linux/module.h>
+#include <linux/ptrace.h> /* struct pt_regs */
+#include "pf_in.h"
+
+#ifdef __i386__
+/* IA32 Manual 3, 2-1 */
+static unsigned char prefix_codes[] = {
+	0xF0, 0xF2, 0xF3, 0x2E, 0x36, 0x3E, 0x26, 0x64,
+	0x65, 0x2E, 0x3E, 0x66, 0x67
+};
+/* IA32 Manual 3, 3-432*/
+static unsigned int reg_rop[] = {
+	0x8A, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
+};
+static unsigned int reg_wop[] = { 0x88, 0x89 };
+static unsigned int imm_wop[] = { 0xC6, 0xC7 };
+/* IA32 Manual 3, 3-432*/
+static unsigned int rw8[] = { 0x88, 0x8A, 0xC6 };
+static unsigned int rw32[] = {
+	0x89, 0x8B, 0xC7, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
+};
+static unsigned int mw8[] = { 0x88, 0x8A, 0xC6, 0xB60F, 0xBE0F };
+static unsigned int mw16[] = { 0xB70F, 0xBF0F };
+static unsigned int mw32[] = { 0x89, 0x8B, 0xC7 };
+static unsigned int mw64[] = {};
+#else /* not __i386__ */
+static unsigned char prefix_codes[] = {
+	0x66, 0x67, 0x2E, 0x3E, 0x26, 0x64, 0x65, 0x36,
+	0xF0, 0xF3, 0xF2,
+	/* REX Prefixes */
+	0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+	0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f
+};
+/* AMD64 Manual 3, Appendix A*/
+static unsigned int reg_rop[] = {
+	0x8A, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
+};
+static unsigned int reg_wop[] = { 0x88, 0x89 };
+static unsigned int imm_wop[] = { 0xC6, 0xC7 };
+static unsigned int rw8[] = { 0xC6, 0x88, 0x8A };
+static unsigned int rw32[] = {
+	0xC7, 0x89, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
+};
+/* 8 bit only */
+static unsigned int mw8[] = { 0xC6, 0x88, 0x8A, 0xB60F, 0xBE0F };
+/* 16 bit only */
+static unsigned int mw16[] = { 0xB70F, 0xBF0F };
+/* 16 or 32 bit */
+static unsigned int mw32[] = { 0xC7 };
+/* 16, 32 or 64 bit */
+static unsigned int mw64[] = { 0x89, 0x8B };
+#endif /* not __i386__ */
+
+static int skip_prefix(unsigned char *addr, int *shorted, int *enlarged,
+								int *rexr)
+{
+	int i;
+	unsigned char *p = addr;
+	*shorted = 0;
+	*enlarged = 0;
+	*rexr = 0;
+
+restart:
+	for (i = 0; i < ARRAY_SIZE(prefix_codes); i++) {
+		if (*p == prefix_codes[i]) {
+			if (*p == 0x66)
+				*shorted = 1;
+#ifdef __amd64__
+			if ((*p & 0xf8) == 0x48)
+				*enlarged = 1;
+			if ((*p & 0xf4) == 0x44)
+				*rexr = 1;
+#endif
+			p++;
+			goto restart;
+		}
+	}
+
+	return (p - addr);
+}
+
+static int get_opcode(unsigned char *addr, unsigned int *opcode)
+{
+	int len;
+
+	if (*addr == 0x0F) {
+		/* 0x0F is extension instruction */
+		*opcode = *(unsigned short *)addr;
+		len = 2;
+	} else {
+		*opcode = *addr;
+		len = 1;
+	}
+
+	return len;
+}
+
+#define CHECK_OP_TYPE(opcode, array, type) \
+	for (i = 0; i < ARRAY_SIZE(array); i++) { \
+		if (array[i] == opcode) { \
+			rv = type; \
+			goto exit; \
+		} \
+	}
+
+enum reason_type get_ins_type(unsigned long ins_addr)
+{
+	unsigned int opcode;
+	unsigned char *p;
+	int shorted, enlarged, rexr;
+	int i;
+	enum reason_type rv = OTHERS;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+
+	CHECK_OP_TYPE(opcode, reg_rop, REG_READ);
+	CHECK_OP_TYPE(opcode, reg_wop, REG_WRITE);
+	CHECK_OP_TYPE(opcode, imm_wop, IMM_WRITE);
+
+exit:
+	return rv;
+}
+#undef CHECK_OP_TYPE
+
+static unsigned int get_ins_reg_width(unsigned long ins_addr)
+{
+	unsigned int opcode;
+	unsigned char *p;
+	int i, shorted, enlarged, rexr;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+
+	for (i = 0; i < ARRAY_SIZE(rw8); i++)
+		if (rw8[i] == opcode)
+			return 1;
+
+	for (i = 0; i < ARRAY_SIZE(rw32); i++)
+		if (rw32[i] == opcode)
+			return (shorted ? 2 : (enlarged ? 8 : 4));
+
+	printk(KERN_ERR "mmiotrace: Unknown opcode 0x%02x\n", opcode);
+	return 0;
+}
+
+unsigned int get_ins_mem_width(unsigned long ins_addr)
+{
+	unsigned int opcode;
+	unsigned char *p;
+	int i, shorted, enlarged, rexr;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+
+	for (i = 0; i < ARRAY_SIZE(mw8); i++)
+		if (mw8[i] == opcode)
+			return 1;
+
+	for (i = 0; i < ARRAY_SIZE(mw16); i++)
+		if (mw16[i] == opcode)
+			return 2;
+
+	for (i = 0; i < ARRAY_SIZE(mw32); i++)
+		if (mw32[i] == opcode)
+			return shorted ? 2 : 4;
+
+	for (i = 0; i < ARRAY_SIZE(mw64); i++)
+		if (mw64[i] == opcode)
+			return shorted ? 2 : (enlarged ? 8 : 4);
+
+	printk(KERN_ERR "mmiotrace: Unknown opcode 0x%02x\n", opcode);
+	return 0;
+}
+
+/*
+ * Define register ident in mod/rm byte.
+ * Note: these are NOT the same as in ptrace-abi.h.
+ */
+enum {
+	arg_AL = 0,
+	arg_CL = 1,
+	arg_DL = 2,
+	arg_BL = 3,
+	arg_AH = 4,
+	arg_CH = 5,
+	arg_DH = 6,
+	arg_BH = 7,
+
+	arg_AX = 0,
+	arg_CX = 1,
+	arg_DX = 2,
+	arg_BX = 3,
+	arg_SP = 4,
+	arg_BP = 5,
+	arg_SI = 6,
+	arg_DI = 7,
+#ifdef __amd64__
+	arg_R8  = 8,
+	arg_R9  = 9,
+	arg_R10 = 10,
+	arg_R11 = 11,
+	arg_R12 = 12,
+	arg_R13 = 13,
+	arg_R14 = 14,
+	arg_R15 = 15
+#endif
+};
+
+static unsigned char *get_reg_w8(int no, struct pt_regs *regs)
+{
+	unsigned char *rv = NULL;
+
+	switch (no) {
+	case arg_AL:
+		rv = (unsigned char *)&regs->ax;
+		break;
+	case arg_BL:
+		rv = (unsigned char *)&regs->bx;
+		break;
+	case arg_CL:
+		rv = (unsigned char *)&regs->cx;
+		break;
+	case arg_DL:
+		rv = (unsigned char *)&regs->dx;
+		break;
+	case arg_AH:
+		rv = 1 + (unsigned char *)&regs->ax;
+		break;
+	case arg_BH:
+		rv = 1 + (unsigned char *)&regs->bx;
+		break;
+	case arg_CH:
+		rv = 1 + (unsigned char *)&regs->cx;
+		break;
+	case arg_DH:
+		rv = 1 + (unsigned char *)&regs->dx;
+		break;
+#ifdef __amd64__
+	case arg_R8:
+		rv = (unsigned char *)&regs->r8;
+		break;
+	case arg_R9:
+		rv = (unsigned char *)&regs->r9;
+		break;
+	case arg_R10:
+		rv = (unsigned char *)&regs->r10;
+		break;
+	case arg_R11:
+		rv = (unsigned char *)&regs->r11;
+		break;
+	case arg_R12:
+		rv = (unsigned char *)&regs->r12;
+		break;
+	case arg_R13:
+		rv = (unsigned char *)&regs->r13;
+		break;
+	case arg_R14:
+		rv = (unsigned char *)&regs->r14;
+		break;
+	case arg_R15:
+		rv = (unsigned char *)&regs->r15;
+		break;
+#endif
+	default:
+		printk(KERN_ERR "mmiotrace: Error reg no# %d\n", no);
+		break;
+	}
+	return rv;
+}
+
+static unsigned long *get_reg_w32(int no, struct pt_regs *regs)
+{
+	unsigned long *rv = NULL;
+
+	switch (no) {
+	case arg_AX:
+		rv = &regs->ax;
+		break;
+	case arg_BX:
+		rv = &regs->bx;
+		break;
+	case arg_CX:
+		rv = &regs->cx;
+		break;
+	case arg_DX:
+		rv = &regs->dx;
+		break;
+	case arg_SP:
+		rv = &regs->sp;
+		break;
+	case arg_BP:
+		rv = &regs->bp;
+		break;
+	case arg_SI:
+		rv = &regs->si;
+		break;
+	case arg_DI:
+		rv = &regs->di;
+		break;
+#ifdef __amd64__
+	case arg_R8:
+		rv = &regs->r8;
+		break;
+	case arg_R9:
+		rv = &regs->r9;
+		break;
+	case arg_R10:
+		rv = &regs->r10;
+		break;
+	case arg_R11:
+		rv = &regs->r11;
+		break;
+	case arg_R12:
+		rv = &regs->r12;
+		break;
+	case arg_R13:
+		rv = &regs->r13;
+		break;
+	case arg_R14:
+		rv = &regs->r14;
+		break;
+	case arg_R15:
+		rv = &regs->r15;
+		break;
+#endif
+	default:
+		printk(KERN_ERR "mmiotrace: Error reg no# %d\n", no);
+	}
+
+	return rv;
+}
+
+unsigned long get_ins_reg_val(unsigned long ins_addr, struct pt_regs *regs)
+{
+	unsigned int opcode;
+	unsigned char mod_rm;
+	int reg;
+	unsigned char *p;
+	int i, shorted, enlarged, rexr;
+	unsigned long rv;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+	for (i = 0; i < ARRAY_SIZE(reg_rop); i++)
+		if (reg_rop[i] == opcode) {
+			rv = REG_READ;
+			goto do_work;
+		}
+
+	for (i = 0; i < ARRAY_SIZE(reg_wop); i++)
+		if (reg_wop[i] == opcode) {
+			rv = REG_WRITE;
+			goto do_work;
+		}
+
+	printk(KERN_ERR "mmiotrace: Not a register instruction, opcode "
+							"0x%02x\n", opcode);
+	goto err;
+
+do_work:
+	mod_rm = *p;
+	reg = ((mod_rm >> 3) & 0x7) | (rexr << 3);
+	switch (get_ins_reg_width(ins_addr)) {
+	case 1:
+		return *get_reg_w8(reg, regs);
+
+	case 2:
+		return *(unsigned short *)get_reg_w32(reg, regs);
+
+	case 4:
+		return *(unsigned int *)get_reg_w32(reg, regs);
+
+#ifdef __amd64__
+	case 8:
+		return *(unsigned long *)get_reg_w32(reg, regs);
+#endif
+
+	default:
+		printk(KERN_ERR "mmiotrace: Error width# %d\n", reg);
+	}
+
+err:
+	return 0;
+}
+
+unsigned long get_ins_imm_val(unsigned long ins_addr)
+{
+	unsigned int opcode;
+	unsigned char mod_rm;
+	unsigned char mod;
+	unsigned char *p;
+	int i, shorted, enlarged, rexr;
+	unsigned long rv;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+	for (i = 0; i < ARRAY_SIZE(imm_wop); i++)
+		if (imm_wop[i] == opcode) {
+			rv = IMM_WRITE;
+			goto do_work;
+		}
+
+	printk(KERN_ERR "mmiotrace: Not an immediate instruction, opcode "
+							"0x%02x\n", opcode);
+	goto err;
+
+do_work:
+	mod_rm = *p;
+	mod = mod_rm >> 6;
+	p++;
+	switch (mod) {
+	case 0:
+		/* if r/m is 5 we have a 32 disp (IA32 Manual 3, Table 2-2)  */
+		/* AMD64: XXX Check for address size prefix? */
+		if ((mod_rm & 0x7) == 0x5)
+			p += 4;
+		break;
+
+	case 1:
+		p += 1;
+		break;
+
+	case 2:
+		p += 4;
+		break;
+
+	case 3:
+	default:
+		printk(KERN_ERR "mmiotrace: not a memory access instruction "
+						"at 0x%lx, rm_mod=0x%02x\n",
+						ins_addr, mod_rm);
+	}
+
+	switch (get_ins_reg_width(ins_addr)) {
+	case 1:
+		return *(unsigned char *)p;
+
+	case 2:
+		return *(unsigned short *)p;
+
+	case 4:
+		return *(unsigned int *)p;
+
+#ifdef __amd64__
+	case 8:
+		return *(unsigned long *)p;
+#endif
+
+	default:
+		printk(KERN_ERR "mmiotrace: Error: width.\n");
+	}
+
+err:
+	return 0;
+}
diff --git a/arch/x86/kernel/mmiotrace/pf_in.h b/arch/x86/kernel/mmiotrace/pf_in.h
new file mode 100644
index 0000000..e05341a
--- /dev/null
+++ b/arch/x86/kernel/mmiotrace/pf_in.h
@@ -0,0 +1,39 @@
+/*
+ *  Fault Injection Test harness (FI)
+ *  Copyright (C) Intel Crop.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ *
+ */
+
+#ifndef __PF_H_
+#define __PF_H_
+
+enum reason_type {
+	NOT_ME,	/* page fault is not in regions */
+	NOTHING,	/* access others point in regions */
+	REG_READ,	/* read from addr to reg */
+	REG_WRITE,	/* write from reg to addr */
+	IMM_WRITE,	/* write from imm to addr */
+	OTHERS	/* Other instructions can not intercept */
+};
+
+enum reason_type get_ins_type(unsigned long ins_addr);
+unsigned int get_ins_mem_width(unsigned long ins_addr);
+unsigned long get_ins_reg_val(unsigned long ins_addr, struct pt_regs *regs);
+unsigned long get_ins_imm_val(unsigned long ins_addr);
+
+#endif /* __PF_H_ */
diff --git a/arch/x86/kernel/mmiotrace/testmmiotrace.c b/arch/x86/kernel/mmiotrace/testmmiotrace.c
new file mode 100644
index 0000000..40e66b0
--- /dev/null
+++ b/arch/x86/kernel/mmiotrace/testmmiotrace.c
@@ -0,0 +1,77 @@
+/*
+ * Written by Pekka Paalanen, 2008 <pq@iki.fi>
+ */
+#include <linux/module.h>
+#include <asm/io.h>
+
+extern void __iomem *ioremap_nocache_trace(unsigned long offset,
+						unsigned long size);
+extern void iounmap_trace(volatile void __iomem *addr);
+
+#define MODULE_NAME "testmmiotrace"
+
+static unsigned long mmio_address;
+module_param(mmio_address, ulong, 0);
+MODULE_PARM_DESC(mmio_address, "Start address of the mapping of 16 kB.");
+
+static void do_write_test(void __iomem *p)
+{
+	unsigned int i;
+	for (i = 0; i < 256; i++)
+		iowrite8(i, p + i);
+	for (i = 1024; i < (5 * 1024); i += 2)
+		iowrite16(i * 12 + 7, p + i);
+	for (i = (5 * 1024); i < (16 * 1024); i += 4)
+		iowrite32(i * 212371 + 13, p + i);
+}
+
+static void do_read_test(void __iomem *p)
+{
+	unsigned int i;
+	volatile unsigned int v;
+	for (i = 0; i < 256; i++)
+		v = ioread8(p + i);
+	for (i = 1024; i < (5 * 1024); i += 2)
+		v = ioread16(p + i);
+	for (i = (5 * 1024); i < (16 * 1024); i += 4)
+		v = ioread32(p + i);
+}
+
+static void do_test(void)
+{
+	void __iomem *p = ioremap_nocache_trace(mmio_address, 0x4000);
+	if (!p) {
+		printk(KERN_ERR MODULE_NAME ": could not ioremap IO memory, "
+							"aborting.\n");
+		return;
+	}
+	do_write_test(p);
+	do_read_test(p);
+	iounmap_trace(p);
+}
+
+static int __init init(void)
+{
+	if (mmio_address == 0) {
+		printk(KERN_ERR MODULE_NAME ": you have to use the module "
+						"argument mmio_address.\n");
+		printk(KERN_ERR MODULE_NAME ": DO NOT LOAD THIS MODULE UNLESS"
+				" YOU REALLY KNOW WHAT YOU ARE DOING!\n");
+		return -ENXIO;
+	}
+
+	printk(KERN_WARNING MODULE_NAME ": WARNING: mapping 16 kB @ 0x%08lx "
+					"in PCI address space, and writing "
+					"rubbish in there.\n", mmio_address);
+	do_test();
+	return 0;
+}
+
+static void __exit cleanup(void)
+{
+	printk(KERN_DEBUG MODULE_NAME ": unloaded.\n");
+}
+
+module_init(init);
+module_exit(cleanup);
+MODULE_LICENSE("GPL");
diff --git a/include/linux/mmiotrace.h b/include/linux/mmiotrace.h
new file mode 100644
index 0000000..cb24782
--- /dev/null
+++ b/include/linux/mmiotrace.h
@@ -0,0 +1,62 @@
+#ifndef MMIOTRACE_H
+#define MMIOTRACE_H
+
+#include <asm/types.h>
+
+#define MMIO_VERSION 0x04
+
+/* mm_io_header.type */
+#define MMIO_OPCODE_MASK 0xff
+#define MMIO_OPCODE_SHIFT 0
+#define MMIO_WIDTH_MASK 0xff00
+#define MMIO_WIDTH_SHIFT 8
+#define MMIO_MAGIC (0x6f000000 | (MMIO_VERSION<<16))
+#define MMIO_MAGIC_MASK 0xffff0000
+
+enum mm_io_opcode {          /* payload type: */
+	MMIO_READ = 0x1,     /* struct mm_io_rw */
+	MMIO_WRITE = 0x2,    /* struct mm_io_rw */
+	MMIO_PROBE = 0x3,    /* struct mm_io_map */
+	MMIO_UNPROBE = 0x4,  /* struct mm_io_map */
+	MMIO_MARKER = 0x5,   /* raw char data */
+	MMIO_UNKNOWN_OP = 0x6, /* struct mm_io_rw */
+};
+
+struct mm_io_header {
+	__u32 type;
+	__u32 sec;      /* timestamp */
+	__u32 nsec;
+	__u32 pid;      /* PID of the process, or 0 for kernel core */
+	__u16 data_len; /* length of the following payload */
+};
+
+struct mm_io_rw {
+	__u64 address; /* virtual address of register */
+	__u64 value;
+	__u64 pc;      /* optional program counter */
+};
+
+struct mm_io_map {
+	__u64 phys;  /* base address in PCI space */
+	__u64 addr;  /* base virtual address */
+	__u64 len;   /* mapping size */
+	__u64 pc;    /* optional program counter */
+};
+
+
+/*
+ * These structures are used to allow a single relay_write()
+ * call to write a full packet.
+ */
+
+struct mm_io_header_rw {
+	struct mm_io_header header;
+	struct mm_io_rw rw;
+} __attribute__((packed));
+
+struct mm_io_header_map {
+	struct mm_io_header header;
+	struct mm_io_map map;
+} __attribute__((packed));
+
+#endif /* MMIOTRACE_H */
-- 
1.5.3.7

Re: [RFC PATCH v2] x86: mmiotrace - trace memory mapped IO

[Arjan van de Ven]

On Thu, 31 Jan 2008 18:16:30 +0200
Pekka Paalanen <pq@iki.fi> wrote:

> > > could you please send us a patch for the whole mmiotrace
> > > kernel-side feature, so that we can have a look at the general
> > > structure of this? (and the interaction with change_page_attr(),
> > > etc.) Even if it's not functional (and wont even build/boot) at
> > > the moment. Thanks,
> 
> And here's mmiotrace. This works for me (amd64), but not for a test
> user who has 32-bit x86. We have not had the chance to debug it yet.
> The symptom is "everything locked up" after insmodding
> testmmiotrace.ko, targeting the mid-point of his VRAM. And there's
> the "undefined symbol init_mm" problem on 32-bit, but we just put the
> export back in (not included in these patches).
> 
> No-one has tried this on SMP. I have a Core 2 Duo laptop where I will
> test this when I get the chance.

 +static void print_pte(unsigned long address)
> +{
> +	pgd_t *pgd = pgd_offset_k(address);
> +	pud_t *pud = pud_offset(pgd, address);
> +	pmd_t *pmd = pmd_offset(pud, address);
> +	if (pmd_large(*pmd)) {
> +		printk(KERN_EMERG MODULE_NAME ": 4MB pages are not "
> +						"currently
> supported: %lx\n",
> +						address);
> +		BUG();
> +	}
> +	printk(KERN_DEBUG MODULE_NAME ": pte for 0x%lx: 0x%lx
> 0x%lx\n",
> +		address,
> +		pte_val(*pte_offset_kernel(pmd, address)),
> +		pte_val(*pte_offset_kernel(pmd, address)) &
> _PAGE_PRESENT); +}

ok this wants to use some of the page_address() and other helpers from pageattr.c;
that should be easy to solve during integration
(I can even see a more generic version of this function moving there)


> +/* file_create() callback.  Creates relay file in debugfs. */
> +static struct dentry *create_buf_file_handler(const char *filename,
> +						struct dentry
> *parent,
> +						int mode,
> +						struct rchan_buf
> *buf,
> +						int *is_global)
> +{
> +	struct dentry *buf_file;
> +
> +	mmio_fops.read = relay_file_operations.read;
> +	mmio_fops.open = relay_file_operations.open;
> +	mmio_fops.poll = relay_file_operations.poll;
> +	mmio_fops.mmap = relay_file_operations.mmap;
> +	mmio_fops.release = relay_file_operations.release;
> +	mmio_fops.splice_read = relay_file_operations.splice_read;

can't we do this at compile time?


> +
> +void __iomem *ioremap_cache_trace(unsigned long offset, unsigned
> long size) +{
> +	void __iomem *p = ioremap_cache(offset, size);
> +	printk(KERN_DEBUG MODULE_NAME ": ioremap_cache(0x%lx, 0x%lx)
> = %p\n",
> +							offset,
> size, p);
> +	ioremap_trace_core(offset, size, p);
> +	return p;
> +}
> +EXPORT_SYMBOL(ioremap_cache_trace);
> +
> +void __iomem *ioremap_nocache_trace(unsigned long offset, unsigned
> long size) +{
> +	void __iomem *p = ioremap_nocache(offset, size);
> +	printk(KERN_DEBUG MODULE_NAME ": ioremap_nocache(0x%lx,
> 0x%lx) = %p\n",
> +							offset,
> size, p);
> +	ioremap_trace_core(offset, size, p);
> +	return p;
> +}
> +EXPORT_SYMBOL(ioremap_nocache_trace);

I would rather keep the split I think and make the trace call explicit

or.. even nicer, we could go fancy and allow the mmio tracer to "subscribe"
to, say, a PCI bar, and auto-enable tracing when the bar gets ioremap'd



about the isntruction decoding.. would be nice if we can share that with kvm
so that we can debug that stuff once ;-)


other than that I think the code is quite nice already, seems getting this merge
ready isn't going to be an enormous deal in terms of complexity

Re: [PATCH v2] x86: Add a list for custom page fault handlers.

[Pekka Paalanen]

On Thu, 31 Jan 2008 08:15:07 -0800
Arjan van de Ven <arjan@infradead.org> wrote:

> On Thu, 31 Jan 2008 18:02:53 +0200
> Pekka Paalanen <pq@iki.fi> wrote:
> > 
> > Very well, first the revised custom page fault handler patch.
> > Changes since the previous submit:
> > - use spin_lock_irqsave instead of spin_lock
> > - Harvey Harrison's clean-up with the #ifdefs
> > - handler call site moved earlier
> > - remove sync RCU call

> it's overhead that's rarely used, and worse, it suffers from the LSM disease:
> it's a hook without visibility into the callers. Esp if there's only 1 user, a direct call
> is not only faster, it also shows you what is going on when you're reading the code.
> THe page fault stuff is tricky enough as it is without any "call arbitrary thing" hooks into
> it

I understand your point.

> > This is most convinient for mmiotrace as it is meant to be a module.
> 
> that's not per se mutually exclusive, there's other cases like this already
> (but even then, I don't want to go through hoops to make mmiotracing a module;
> for me I can totally see it becoming a full mature member of the kernel, and if
> that means a few things have to be changed in the normal kernel, or infrastructure
> added, I'm totally ok with that)

I have two reasons why I'd like to let it be a module:
- it's "broken", the relay buffers are cleared on module unload
- it's a lot easier to push updated version for testing to people

Ok, the first one is just a silly excuse, but the second one is to
avoid forcing non-developer users to patch their kernels. This is a
much needed tool for the Nouveau project even in it's current form.

> > I'm also using this as an excuse to let other people to get into the
> > page fault handler with their out-of-tree-today modules.
> 
> this is exactly the point.. it becomes a totally opaque thing, a random hook.

And this was the third reason I wanted a module, but I understand this
is an unacceptable justification.

What are these "othes cases" I could look at as an example of a hook,
that is meant for a single specific module? I do not understand what
could prevent out-of-tree modules from using the hook if my module can
use it (and my module is not using it at the same time).

What is the verdict?
Will the custom page fault handler patch v2 get merged as it is,
do I need to modify it (how?), do I need to rewrite it based on
some example (which?), or do I have to drop the whole idea of
mmiotrace being kernel module and make it work as built-in?

In the case of making it a built-in, I do not think I have any
chance to get it done for 2.6.25.

There's one more option. I could make kmmio.c things as builtin,
and keep the mmio-mod.c as the module. This is how things were in
the very beginning of mmiotrace, when the page fault notifiers did
not exist. This could be an acceptable compromise, no?
But I don't think I will get that ready for 2.6.25, either.

What about merging the custom page fault handler patch now, and
removing it for 2.6.26?

Looks like Beulich and Roskin will be disappointed anyway. They
were hoping for a usable page fault hook.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH v2] x86: Add a list for custom page fault handlers.

[Ingo Molnar]

* Pekka Paalanen <pq@iki.fi> wrote:

> I have two reasons why I'd like to let it be a module:
> - it's "broken", the relay buffers are cleared on module unload
> - it's a lot easier to push updated version for testing to people
> 
> Ok, the first one is just a silly excuse, but the second one is to 
> avoid forcing non-developer users to patch their kernels. This is a 
> much needed tool for the Nouveau project even in it's current form.

i dont think this second problem is a practical one: i've test-merged 
mmiotrace two days ago into x86.git and it's holding up very well in my 
testing (it has not caused a single hickup so far). It's nice optional 
debug functionality and i dont see any reason why not to merge it.

a few cleanup requests:

- please make the pagefault callbacks explicit as Arjan has asked

- any reason why it lives in arch/x86/kernel/mmiotrace/? It should be a
  prime-time member of arch/x86/mm/ i think.

- please make it build-in-able, not modules-only.

- [ longer-term: think about integrating pf_in.c with the x86
    emulator/disassembler of KVM. ]

	Ingo

Re: [RFC PATCH v2] x86: mmiotrace - trace memory mapped IO

[Pekka Paalanen]

On Thu, 31 Jan 2008 08:29:06 -0800
Arjan van de Ven <arjan@infradead.org> wrote:

> On Thu, 31 Jan 2008 18:16:30 +0200
> Pekka Paalanen <pq@iki.fi> wrote:
> 
> > And here's mmiotrace. This works for me (amd64), but not for a test
> > user who has 32-bit x86. We have not had the chance to debug it yet.
> > The symptom is "everything locked up" after insmodding
> > testmmiotrace.ko, targeting the mid-point of his VRAM. And there's
> > the "undefined symbol init_mm" problem on 32-bit, but we just put the
> > export back in (not included in these patches).

This problem seems to have been due to kgdb being enabled. Quote from
Jaap Stolk, the test user:
"(test)mmiotrace will trigger wait for serial debugger if kgdb and
CONFIG_KGDB_ATTACH_WAIT is enabled."
After disabling it, mmiotrace ran fine.

> > No-one has tried this on SMP. I have a Core 2 Duo laptop where I will
> > test this when I get the chance.

I tested, worked just fine. Well, the test module does not really test
simultaneous iomem access. And I have an issue with following relay
buffers' full/not-full status, I'm looking into that now.

> ok this wants to use some of the page_address() and other helpers from pageattr.c;
> that should be easy to solve during integration
> (I can even see a more generic version of this function moving there)

page_address() seems to be dealing with struct page, someone said we
usually don't have that (ioremap).

Is this what you had in mind? I had to add an export for lookup_address()
to make this work.

 arch/x86/kernel/mmiotrace/kmmio.c    |   46 +++++++++++++++++++++++-----------
 arch/x86/kernel/mmiotrace/mmio-mod.c |   19 +++++++++-----
 arch/x86/mm/pageattr.c               |    1 +
 3 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/arch/x86/kernel/mmiotrace/kmmio.c b/arch/x86/kernel/mmiotrace/kmmio.c
index 8ba48f9..28411da 100644
--- a/arch/x86/kernel/mmiotrace/kmmio.c
+++ b/arch/x86/kernel/mmiotrace/kmmio.c
@@ -20,6 +20,7 @@
 #include <asm/cacheflush.h>
 #include <asm/errno.h>
 #include <asm/tlbflush.h>
+#include <asm/pgtable.h>
 
 #include "kmmio.h"
 
@@ -117,40 +118,55 @@ static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page)
 	return NULL;
 }
 
-static void arm_kmmio_fault_page(unsigned long page, int *large)
+static void arm_kmmio_fault_page(unsigned long page, int *page_level)
 {
 	unsigned long address = page & PAGE_MASK;
-	pgd_t *pgd = pgd_offset_k(address);
-	pud_t *pud = pud_offset(pgd, address);
-	pmd_t *pmd = pmd_offset(pud, address);
-	pte_t *pte = pte_offset_kernel(pmd, address);
+	int level;
+	pte_t *pte = lookup_address(address, &level);
 
-	if (pmd_large(*pmd)) {
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, page);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
+		pmd_t *pmd = (pmd_t *)pte;
 		set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_PRESENT));
-		if (large)
-			*large = 1;
 	} else {
+		/* PG_LEVEL_4K */
 		set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
 	}
 
+	if (page_level)
+		*page_level = level;
+
 	__flush_tlb_one(page);
 }
 
-static void disarm_kmmio_fault_page(unsigned long page, int *large)
+static void disarm_kmmio_fault_page(unsigned long page, int *page_level)
 {
 	unsigned long address = page & PAGE_MASK;
-	pgd_t *pgd = pgd_offset_k(address);
-	pud_t *pud = pud_offset(pgd, address);
-	pmd_t *pmd = pmd_offset(pud, address);
-	pte_t *pte = pte_offset_kernel(pmd, address);
+	int level;
+	pte_t *pte = lookup_address(address, &level);
 
-	if (large && *large) {
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, page);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
+		pmd_t *pmd = (pmd_t *)pte;
 		set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_PRESENT));
-		*large = 0;
 	} else {
+		/* PG_LEVEL_4K */
 		set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
 	}
 
+	if (page_level)
+		*page_level = level;
+
 	__flush_tlb_one(page);
 }
 
diff --git a/arch/x86/kernel/mmiotrace/mmio-mod.c b/arch/x86/kernel/mmiotrace/mmio-mod.c
index 4a8c859..82ae920 100644
--- a/arch/x86/kernel/mmiotrace/mmio-mod.c
+++ b/arch/x86/kernel/mmiotrace/mmio-mod.c
@@ -120,19 +120,24 @@ static int write_marker(struct file *file, const char __user *buffer,
 
 static void print_pte(unsigned long address)
 {
-	pgd_t *pgd = pgd_offset_k(address);
-	pud_t *pud = pud_offset(pgd, address);
-	pmd_t *pmd = pmd_offset(pud, address);
-	if (pmd_large(*pmd)) {
+	int level;
+	pte_t *pte = lookup_address(address, &level);
+
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, address);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
 		printk(KERN_EMERG MODULE_NAME ": 4MB pages are not "
 						"currently supported: %lx\n",
 						address);
 		BUG();
 	}
 	printk(KERN_DEBUG MODULE_NAME ": pte for 0x%lx: 0x%lx 0x%lx\n",
-		address,
-		pte_val(*pte_offset_kernel(pmd, address)),
-		pte_val(*pte_offset_kernel(pmd, address)) & _PAGE_PRESENT);
+					address, pte_val(*pte),
+					pte_val(*pte) & _PAGE_PRESENT);
 }
 
 /*
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index b8a8408..7f0a84e 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -194,6 +194,7 @@ pte_t *lookup_address(unsigned long address, int *level)
 	*level = PG_LEVEL_4K;
 	return pte_offset_kernel(pmd, address);
 }
+EXPORT_SYMBOL(lookup_address);
 
 static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
 {


> > +	mmio_fops.read = relay_file_operations.read;
> > +	mmio_fops.open = relay_file_operations.open;
> > +	mmio_fops.poll = relay_file_operations.poll;
> > +	mmio_fops.mmap = relay_file_operations.mmap;
> > +	mmio_fops.release = relay_file_operations.release;
> > +	mmio_fops.splice_read = relay_file_operations.splice_read;
> 
> can't we do this at compile time?

Nope, the compiler says the initializer is not constant.

> > +
> > +void __iomem *ioremap_cache_trace(unsigned long offset, unsigned
> > long size) +{
> > +	void __iomem *p = ioremap_cache(offset, size);
> > +	printk(KERN_DEBUG MODULE_NAME ": ioremap_cache(0x%lx, 0x%lx)
> > = %p\n",
> > +							offset,
> > size, p);
> > +	ioremap_trace_core(offset, size, p);
> > +	return p;
> > +}
> > +EXPORT_SYMBOL(ioremap_cache_trace);
> > +
> > +void __iomem *ioremap_nocache_trace(unsigned long offset, unsigned
> > long size) +{
> > +	void __iomem *p = ioremap_nocache(offset, size);
> > +	printk(KERN_DEBUG MODULE_NAME ": ioremap_nocache(0x%lx,
> > 0x%lx) = %p\n",
> > +							offset,
> > size, p);
> > +	ioremap_trace_core(offset, size, p);
> > +	return p;
> > +}
> > +EXPORT_SYMBOL(ioremap_nocache_trace);
> 
> I would rather keep the split I think and make the trace call explicit

I don't think I understand what you mean. Get rid of
(do_)ioremap_trace_core?

> or.. even nicer, we could go fancy and allow the mmio tracer to "subscribe"
> to, say, a PCI bar, and auto-enable tracing when the bar gets ioremap'd

Might be cool, but not until there's no hope of getting into 2.6.25.

> about the isntruction decoding.. would be nice if we can share that with kvm
> so that we can debug that stuff once ;-)

Uhh yeah, if you mean the stuff in drivers/kvm/x86_emulate.c, it indeed looks
like it's decoding instructions. Other than that, I understood almost nothing
in there. There's one gigantic function that seems to decode and emulate an
instruction. I would have to refactor all that code and I'm not up for it.

> other than that I think the code is quite nice already, seems getting this merge
> ready isn't going to be an enormous deal in terms of complexity

Thank you for your kind and constructive comments.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH v2] x86: Add a list for custom page fault handlers.

[Pekka Paalanen]

On Sun, 3 Feb 2008 08:03:21 +0100
Ingo Molnar <mingo@elte.hu> wrote:

> i dont think this second problem is a practical one: i've test-merged 
> mmiotrace two days ago into x86.git and it's holding up very well in my 
> testing (it has not caused a single hickup so far). It's nice optional 
> debug functionality and i dont see any reason why not to merge it.

Good to hear!

I've made some fixes since my last mmiotrace patch (v2). Rewritten things
to use lookup_address(), added EXPORT_SYMBOL(lookup_address) into
pageattr.c to make it work (this hopefully fixed the undefined symbol
'init_mm' on x86-32), and fixed relay-buffer-full flags on SMP.

> a few cleanup requests:
> 
> - please make the pagefault callbacks explicit as Arjan has asked

What does this mean in practice? It should be able to call into a module,
so the only thing I can come up with is a single settable function
pointer. Just like I proposed in patch v2, but a function pointer instead
of a list. Is this what you and Arjan suggest?
We are talking about the hook in do_page_fault(), right?

> - any reason why it lives in arch/x86/kernel/mmiotrace/? It should be a
>   prime-time member of arch/x86/mm/ i think.

I was waiting for someone to comment on that, I didn't know what was the
proper place. I'll move it into there.

> - please make it build-in-able, not modules-only.

Then I need a way to clear the relay buffers at some point. Currently
they get cleared only on module unload. Maybe preferably also allocate
and deallocate them dynamically. I've no idea how to do this yet, any
pointers to examples?

Hmm, I guess relay buffers could be allocated on the first intercepted
call to ioremap, but then the userspace could not be started before
hand, and at that point it is too late.

Making it build-in-able will take so much time that I'd prefer to aim
it for 2.6.26.

> - [ longer-term: think about integrating pf_in.c with the x86
>     emulator/disassembler of KVM. ]

Yikes. Like I replied to Arjan, it looks daunting.


Thanks, I'll do what I can. When do you need the final (as for 2.6.25)
mmiotrace patch, or is it going in as v2 and then we fix it in RCs?

-- 
Pekka Paalanen
http://www.iki.fi/pq/

[PATCH 1/4] x86 mmiotrace: use lookup_address()

[Pekka Paalanen]

Use lookup_address() from pageattr.c instead of doing the same
manually. Also had to EXPORT_SYMBOL(lookup_address) to make this
work for modules. This also fixes "undefined symbol 'init_mm'"
compile error for x86_32.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---
 arch/x86/kernel/mmiotrace/kmmio.c    |   46 +++++++++++++++++++++++-----------
 arch/x86/kernel/mmiotrace/mmio-mod.c |   19 +++++++++-----
 arch/x86/mm/pageattr.c               |    1 +
 3 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/arch/x86/kernel/mmiotrace/kmmio.c b/arch/x86/kernel/mmiotrace/kmmio.c
index 8ba48f9..28411da 100644
--- a/arch/x86/kernel/mmiotrace/kmmio.c
+++ b/arch/x86/kernel/mmiotrace/kmmio.c
@@ -20,6 +20,7 @@
 #include <asm/cacheflush.h>
 #include <asm/errno.h>
 #include <asm/tlbflush.h>
+#include <asm/pgtable.h>
 
 #include "kmmio.h"
 
@@ -117,40 +118,55 @@ static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page)
 	return NULL;
 }
 
-static void arm_kmmio_fault_page(unsigned long page, int *large)
+static void arm_kmmio_fault_page(unsigned long page, int *page_level)
 {
 	unsigned long address = page & PAGE_MASK;
-	pgd_t *pgd = pgd_offset_k(address);
-	pud_t *pud = pud_offset(pgd, address);
-	pmd_t *pmd = pmd_offset(pud, address);
-	pte_t *pte = pte_offset_kernel(pmd, address);
+	int level;
+	pte_t *pte = lookup_address(address, &level);
 
-	if (pmd_large(*pmd)) {
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, page);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
+		pmd_t *pmd = (pmd_t *)pte;
 		set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_PRESENT));
-		if (large)
-			*large = 1;
 	} else {
+		/* PG_LEVEL_4K */
 		set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
 	}
 
+	if (page_level)
+		*page_level = level;
+
 	__flush_tlb_one(page);
 }
 
-static void disarm_kmmio_fault_page(unsigned long page, int *large)
+static void disarm_kmmio_fault_page(unsigned long page, int *page_level)
 {
 	unsigned long address = page & PAGE_MASK;
-	pgd_t *pgd = pgd_offset_k(address);
-	pud_t *pud = pud_offset(pgd, address);
-	pmd_t *pmd = pmd_offset(pud, address);
-	pte_t *pte = pte_offset_kernel(pmd, address);
+	int level;
+	pte_t *pte = lookup_address(address, &level);
 
-	if (large && *large) {
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, page);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
+		pmd_t *pmd = (pmd_t *)pte;
 		set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_PRESENT));
-		*large = 0;
 	} else {
+		/* PG_LEVEL_4K */
 		set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
 	}
 
+	if (page_level)
+		*page_level = level;
+
 	__flush_tlb_one(page);
 }
 
diff --git a/arch/x86/kernel/mmiotrace/mmio-mod.c b/arch/x86/kernel/mmiotrace/mmio-mod.c
index 4a8c859..82ae920 100644
--- a/arch/x86/kernel/mmiotrace/mmio-mod.c
+++ b/arch/x86/kernel/mmiotrace/mmio-mod.c
@@ -120,19 +120,24 @@ static int write_marker(struct file *file, const char __user *buffer,
 
 static void print_pte(unsigned long address)
 {
-	pgd_t *pgd = pgd_offset_k(address);
-	pud_t *pud = pud_offset(pgd, address);
-	pmd_t *pmd = pmd_offset(pud, address);
-	if (pmd_large(*pmd)) {
+	int level;
+	pte_t *pte = lookup_address(address, &level);
+
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, address);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
 		printk(KERN_EMERG MODULE_NAME ": 4MB pages are not "
 						"currently supported: %lx\n",
 						address);
 		BUG();
 	}
 	printk(KERN_DEBUG MODULE_NAME ": pte for 0x%lx: 0x%lx 0x%lx\n",
-		address,
-		pte_val(*pte_offset_kernel(pmd, address)),
-		pte_val(*pte_offset_kernel(pmd, address)) & _PAGE_PRESENT);
+					address, pte_val(*pte),
+					pte_val(*pte) & _PAGE_PRESENT);
 }
 
 /*
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index bb55a78..fc7e9ea 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -225,6 +225,7 @@ pte_t *lookup_address(unsigned long address, int *level)
 
 	return pte_offset_kernel(pmd, address);
 }
+EXPORT_SYMBOL(lookup_address);
 
 /*
  * Set the new pmd in all the pgds we know about:
-- 
1.5.3.7

[PATCH 2/4] x86 mmiotrace: fix relay-buffer-full flag for SMP

[Pekka Paalanen]

Relay has per-cpu buffers, but mmiotrace was using only a single flag
for detecting buffer full/not-full transitions. The new code makes
this per-cpu and actually counts missed events.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---
 arch/x86/kernel/mmiotrace/mmio-mod.c |   26 ++++++++++++++++----------
 1 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/arch/x86/kernel/mmiotrace/mmio-mod.c b/arch/x86/kernel/mmiotrace/mmio-mod.c
index 82ae920..f492b65 100644
--- a/arch/x86/kernel/mmiotrace/mmio-mod.c
+++ b/arch/x86/kernel/mmiotrace/mmio-mod.c
@@ -29,6 +29,7 @@
 #include <asm/pgtable.h>
 #include <linux/mmiotrace.h>
 #include <asm/e820.h> /* for ISA_START_ADDRESS */
+#include <asm/atomic.h>
 
 #include "kmmio.h"
 #include "pf_in.h"
@@ -47,9 +48,13 @@ struct trap_reason {
 	int active_traces;
 };
 
+/* Accessed per-cpu. */
 static struct trap_reason pf_reason[NR_CPUS];
 static struct mm_io_header_rw cpu_trace[NR_CPUS];
 
+/* Access to this is not per-cpu. */
+static atomic_t dropped[NR_CPUS];
+
 static struct file_operations mmio_fops = {
 	.owner = THIS_MODULE,
 };
@@ -57,7 +62,6 @@ static struct file_operations mmio_fops = {
 static const size_t subbuf_size = 256*1024;
 static struct rchan *chan;
 static struct dentry *dir;
-static int suspended;      /* XXX should this be per cpu? */
 static struct proc_dir_entry *proc_marker_file;
 
 /* module parameters */
@@ -269,19 +273,21 @@ static void post(struct kmmio_probe *p, unsigned long condition,
 static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf,
 					void *prev_subbuf, size_t prev_padding)
 {
+	unsigned int cpu = buf->cpu;
+	atomic_t *drop = &dropped[cpu];
+	int count;
 	if (relay_buf_full(buf)) {
-		if (!suspended) {
-			suspended = 1;
-			printk(KERN_ERR MODULE_NAME
-						": cpu %d buffer full!!!\n",
-						smp_processor_id());
+		if (atomic_inc_return(drop) == 1) {
+			printk(KERN_ERR MODULE_NAME ": cpu %d buffer full!\n",
+									cpu);
 		}
 		return 0;
-	} else if (suspended) {
-		suspended = 0;
+	} else if ((count = atomic_read(drop))) {
 		printk(KERN_ERR MODULE_NAME
-					": cpu %d buffer no longer full.\n",
-					smp_processor_id());
+					": cpu %d buffer no longer full, "
+					"missed %d events.\n",
+					cpu, count);
+		atomic_sub(count, drop);
 	}
 
 	return 1;
-- 
1.5.3.7

[PATCH 3/4] x86 mmiotrace: comment about user space ABI

[Pekka Paalanen]

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---
 include/linux/mmiotrace.h |    6 +++++-
 1 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/include/linux/mmiotrace.h b/include/linux/mmiotrace.h
index cb24782..6ec288f 100644
--- a/include/linux/mmiotrace.h
+++ b/include/linux/mmiotrace.h
@@ -3,6 +3,10 @@
 
 #include <asm/types.h>
 
+/*
+ * If you change anything here, you must bump MMIO_VERSION.
+ * This is the relay data format for user space.
+ */
 #define MMIO_VERSION 0x04
 
 /* mm_io_header.type */
@@ -23,7 +27,7 @@ enum mm_io_opcode {          /* payload type: */
 };
 
 struct mm_io_header {
-	__u32 type;
+	__u32 type;     /* see MMIO_* macros above */
 	__u32 sec;      /* timestamp */
 	__u32 nsec;
 	__u32 pid;      /* PID of the process, or 0 for kernel core */
-- 
1.5.3.7

[PATCH 4/4] x86 mmiotrace: move files into arch/x86/mm/

[Pekka Paalanen]

As this patch is too big for the list, it can be found at:
http://jumi.lut.fi/~paalanen/scratch/mmio25-b/0004-x86-mmiotrace-move-files-into-arch-x86-mm.patch

The patch is 85 kB and Documentation/SubmittingPatches says I should not
post it to the list if it exceeds 40 kB.

Ingo, I can also send this directly to your email, if you wish.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH 2/4] x86 mmiotrace: fix relay-buffer-full flag for SMP

[Eric Dumazet]

Pekka Paalanen a écrit :
> Relay has per-cpu buffers, but mmiotrace was using only a single flag
> for detecting buffer full/not-full transitions. The new code makes
> this per-cpu and actually counts missed events.
> 
> Signed-off-by: Pekka Paalanen <pq@iki.fi>
> ---
>  arch/x86/kernel/mmiotrace/mmio-mod.c |   26 ++++++++++++++++----------
>  1 files changed, 16 insertions(+), 10 deletions(-)
> 
> diff --git a/arch/x86/kernel/mmiotrace/mmio-mod.c b/arch/x86/kernel/mmiotrace/mmio-mod.c
> index 82ae920..f492b65 100644
> --- a/arch/x86/kernel/mmiotrace/mmio-mod.c
> +++ b/arch/x86/kernel/mmiotrace/mmio-mod.c
> @@ -29,6 +29,7 @@
>  #include <asm/pgtable.h>
>  #include <linux/mmiotrace.h>
>  #include <asm/e820.h> /* for ISA_START_ADDRESS */
> +#include <asm/atomic.h>
>  
>  #include "kmmio.h"
>  #include "pf_in.h"
> @@ -47,9 +48,13 @@ struct trap_reason {
>  	int active_traces;
>  };
>  
> +/* Accessed per-cpu. */
>  static struct trap_reason pf_reason[NR_CPUS];
>  static struct mm_io_header_rw cpu_trace[NR_CPUS];
>  
> +/* Access to this is not per-cpu. */
> +static atomic_t dropped[NR_CPUS];
> +

Please dont introduce NR_CPUS new arrays, since people are working hard to zap 
them from kernel.

You probably can use a per_cpu variable ?

Thank you

Re: [PATCH 2/4] x86 mmiotrace: fix relay-buffer-full flag for SMP

[Pekka Paalanen]

On Tue, 05 Feb 2008 21:44:07 +0100
Eric Dumazet <dada1@cosmosbay.com> wrote:

> Pekka Paalanen a écrit :
> > diff --git a/arch/x86/kernel/mmiotrace/mmio-mod.c b/arch/x86/kernel/mmiotrace/mmio-mod.c
> > index 82ae920..f492b65 100644
> > --- a/arch/x86/kernel/mmiotrace/mmio-mod.c
> > +++ b/arch/x86/kernel/mmiotrace/mmio-mod.c
> > @@ -47,9 +48,13 @@ struct trap_reason {
> >  	int active_traces;
> >  };
> >  
> > +/* Accessed per-cpu. */
> >  static struct trap_reason pf_reason[NR_CPUS];
> >  static struct mm_io_header_rw cpu_trace[NR_CPUS];
> >  
> > +/* Access to this is not per-cpu. */
> > +static atomic_t dropped[NR_CPUS];
> > +
> 
> Please dont introduce NR_CPUS new arrays, since people are working hard to zap 
> them from kernel.
> 
> You probably can use a per_cpu variable ?

Yes, it would probably be more appropriate to use DEFINE_PER_CPU()
for 'pf_reason' and 'cpu_trace', but I wasn't sure since the examples
of DEFINE_PER_CPU I saw always had integers or pointers, not
whole structs. Is it okay for whole structs?

'dropped' on the other hand is not accessed in per-cpu style, any cpu
may access any element. DEFINE_PER_CPU is not valid here, is it?

Thanks for the note, I knew I should fix that at some point.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH 2/4] x86 mmiotrace: fix relay-buffer-full flag for SMP

[Eric Dumazet]

Pekka Paalanen a écrit :
> On Tue, 05 Feb 2008 21:44:07 +0100
> Eric Dumazet <dada1@cosmosbay.com> wrote:
> 
>> Pekka Paalanen a écrit :
>>> diff --git a/arch/x86/kernel/mmiotrace/mmio-mod.c b/arch/x86/kernel/mmiotrace/mmio-mod.c
>>> index 82ae920..f492b65 100644
>>> --- a/arch/x86/kernel/mmiotrace/mmio-mod.c
>>> +++ b/arch/x86/kernel/mmiotrace/mmio-mod.c
>>> @@ -47,9 +48,13 @@ struct trap_reason {
>>>  	int active_traces;
>>>  };
>>>  
>>> +/* Accessed per-cpu. */
>>>  static struct trap_reason pf_reason[NR_CPUS];
>>>  static struct mm_io_header_rw cpu_trace[NR_CPUS];
>>>  
>>> +/* Access to this is not per-cpu. */
>>> +static atomic_t dropped[NR_CPUS];
>>> +
>> Please dont introduce NR_CPUS new arrays, since people are working hard to zap 
>> them from kernel.
>>
>> You probably can use a per_cpu variable ?
> 
> Yes, it would probably be more appropriate to use DEFINE_PER_CPU()
> for 'pf_reason' and 'cpu_trace', but I wasn't sure since the examples
> of DEFINE_PER_CPU I saw always had integers or pointers, not
> whole structs. Is it okay for whole structs?

yes you can use a structure, you can check for example :

net/ipv4/route.c:static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);

> 
> 'dropped' on the other hand is not accessed in per-cpu style, any cpu
> may access any element. DEFINE_PER_CPU is not valid here, is it?

It is valid, you can use per_cpu() accessor to get a pointer to a particular 
cpu data.

check net/ipv4/route.c for an example :

return &per_cpu(rt_cache_stat, cpu);

Re: [PATCH 4/4] x86 mmiotrace: move files into arch/x86/mm/

[Randy Dunlap]

On Tue, 5 Feb 2008 22:39:58 +0200 Pekka Paalanen wrote:

> As this patch is too big for the list, it can be found at:
> http://jumi.lut.fi/~paalanen/scratch/mmio25-b/0004-x86-mmiotrace-move-files-into-arch-x86-mm.patch
> 
> The patch is 85 kB and Documentation/SubmittingPatches says I should not
> post it to the list if it exceeds 40 kB.

Hm, we need to change that comment. The mailing list limit
(for lkml) is now 400 KB AFAIK.  Certainly we see plenty of
patches that are larger than 40 KB.

> Ingo, I can also send this directly to your email, if you wish.

Please email it to the mailing list.

---
~Randy

Re: [PATCH 1/4] x86 mmiotrace: use lookup_address()

[Christoph Hellwig]

On Tue, Feb 05, 2008 at 10:28:07PM +0200, Pekka Paalanen wrote:
> Use lookup_address() from pageattr.c instead of doing the same
> manually. Also had to EXPORT_SYMBOL(lookup_address) to make this
> work for modules. This also fixes "undefined symbol 'init_mm'"
> compile error for x86_32.

This should be a _GPL export for sure.

Re: [PATCH 1/4] x86 mmiotrace: use lookup_address()

[Ingo Molnar]

* Christoph Hellwig <hch@infradead.org> wrote:

> On Tue, Feb 05, 2008 at 10:28:07PM +0200, Pekka Paalanen wrote:
> > Use lookup_address() from pageattr.c instead of doing the same
> > manually. Also had to EXPORT_SYMBOL(lookup_address) to make this
> > work for modules. This also fixes "undefined symbol 'init_mm'"
> > compile error for x86_32.
> 
> This should be a _GPL export for sure.

i've changed it to that.

	Ingo

Re: [PATCH 4/4] x86 mmiotrace: move files into arch/x86/mm/

[Ingo Molnar]

Pekka,

i've picked up your latest cleanups, but a fundamental worry still 
remains: all the ugly 'custom pagefault handlers' stuff. Lets just have 
one clean direct callback to mmiotrace, ok?

	Ingo

Re: [PATCH 4/4] x86 mmiotrace: move files into arch/x86/mm/

[Christoph Hellwig]

On Thu, Feb 07, 2008 at 01:53:27PM +0100, Ingo Molnar wrote:
> 
> Pekka,
> 
> i've picked up your latest cleanups, but a fundamental worry still 
> remains: all the ugly 'custom pagefault handlers' stuff. Lets just have 
> one clean direct callback to mmiotrace, ok?

Yes, please.  And any chance we you could resend the current version of
the patch to lkml?  I've lost a little track of how it looks currently.

Re: [PATCH 4/4] x86 mmiotrace: move files into arch/x86/mm/

[Pekka Paalanen]

>From 788b7aed85c8ef0256dc8e71ae981c4973a6a304 Mon Sep 17 00:00:00 2001
From: Pekka Paalanen <pq@iki.fi>
Date: Mon, 4 Feb 2008 21:26:41 +0200
Subject: [PATCH] x86 mmiotrace: move files into arch/x86/mm/.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---

Hi,

Sorry for being silent, but I am having trouble with my Internet service,
I have been practically cut off since Wednesday.

Christoph, this patch shows my last version of mmiotrace, since it is
moving all mmiotrace specific files, except include/linux/mmiotrace.h
which contains just the relay data representation for user space.
This does not include changes Ingo or someone else may have done.

When I catch up with x86/mm git, I can repost everything in one patch
series. Given that my ISP's hardware permits...

About the page fault hook, I've been trying to ask what to do with it
exactly, wishing for a pointer to an existing example or somesuch.
So in lack of better ideas, I'll redo my very first idea of a single
function pointer, and see what you think of it. Peter Zijlstra
said on IRC it would never get accepted and suggested the list
approach, which was the first I posted to LKML.

 arch/x86/kernel/Makefile                  |    2 -
 arch/x86/kernel/mmiotrace/Makefile        |    4 -
 arch/x86/kernel/mmiotrace/kmmio.c         |  407 ----------------------
 arch/x86/kernel/mmiotrace/kmmio.h         |   58 ---
 arch/x86/kernel/mmiotrace/mmio-mod.c      |  538 -----------------------------
 arch/x86/kernel/mmiotrace/pf_in.c         |  489 --------------------------
 arch/x86/kernel/mmiotrace/pf_in.h         |   39 --
 arch/x86/kernel/mmiotrace/testmmiotrace.c |   77 ----
 arch/x86/mm/Makefile                      |    5 +
 arch/x86/mm/kmmio.c                       |  407 ++++++++++++++++++++++
 arch/x86/mm/kmmio.h                       |   58 +++
 arch/x86/mm/mmio-mod.c                    |  538 +++++++++++++++++++++++++++++
 arch/x86/mm/pf_in.c                       |  489 ++++++++++++++++++++++++++
 arch/x86/mm/pf_in.h                       |   39 ++
 arch/x86/mm/testmmiotrace.c               |   77 ++++
 15 files changed, 1613 insertions(+), 1614 deletions(-)
 delete mode 100644 arch/x86/kernel/mmiotrace/Makefile
 delete mode 100644 arch/x86/kernel/mmiotrace/kmmio.c
 delete mode 100644 arch/x86/kernel/mmiotrace/kmmio.h
 delete mode 100644 arch/x86/kernel/mmiotrace/mmio-mod.c
 delete mode 100644 arch/x86/kernel/mmiotrace/pf_in.c
 delete mode 100644 arch/x86/kernel/mmiotrace/pf_in.h
 delete mode 100644 arch/x86/kernel/mmiotrace/testmmiotrace.c
 create mode 100644 arch/x86/mm/kmmio.c
 create mode 100644 arch/x86/mm/kmmio.h
 create mode 100644 arch/x86/mm/mmio-mod.c
 create mode 100644 arch/x86/mm/pf_in.c
 create mode 100644 arch/x86/mm/pf_in.h
 create mode 100644 arch/x86/mm/testmmiotrace.c

diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 61f5c73..b046f50 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -62,8 +62,6 @@ obj-$(CONFIG_KGDB)		+= kgdb.o kgdb-jmp_$(BITS).o
 obj-$(CONFIG_VM86)		+= vm86_32.o
 obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
 
-obj-$(CONFIG_MMIOTRACE)		+= mmiotrace/
-
 obj-$(CONFIG_HPET_TIMER) 	+= hpet.o
 
 obj-$(CONFIG_K8_NB)		+= k8.o
diff --git a/arch/x86/kernel/mmiotrace/Makefile b/arch/x86/kernel/mmiotrace/Makefile
deleted file mode 100644
index d6905f7..0000000
--- a/arch/x86/kernel/mmiotrace/Makefile
+++ /dev/null
@@ -1,4 +0,0 @@
-obj-$(CONFIG_MMIOTRACE) += mmiotrace.o
-mmiotrace-objs := pf_in.o kmmio.o mmio-mod.o
-
-obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o
diff --git a/arch/x86/kernel/mmiotrace/kmmio.c b/arch/x86/kernel/mmiotrace/kmmio.c
deleted file mode 100644
index 28411da..0000000
--- a/arch/x86/kernel/mmiotrace/kmmio.c
+++ /dev/null
@@ -1,407 +0,0 @@
-/* Support for MMIO probes.
- * Benfit many code from kprobes
- * (C) 2002 Louis Zhuang <louis.zhuang@intel.com>.
- *     2007 Alexander Eichner
- *     2008 Pekka Paalanen <pq@iki.fi>
- */
-
-#include <linux/version.h>
-#include <linux/spinlock.h>
-#include <linux/hash.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/uaccess.h>
-#include <linux/ptrace.h>
-#include <linux/preempt.h>
-#include <asm/io.h>
-#include <asm/cacheflush.h>
-#include <asm/errno.h>
-#include <asm/tlbflush.h>
-#include <asm/pgtable.h>
-
-#include "kmmio.h"
-
-#define KMMIO_HASH_BITS 6
-#define KMMIO_TABLE_SIZE (1 << KMMIO_HASH_BITS)
-#define KMMIO_PAGE_HASH_BITS 4
-#define KMMIO_PAGE_TABLE_SIZE (1 << KMMIO_PAGE_HASH_BITS)
-
-struct kmmio_context {
-	struct kmmio_fault_page *fpage;
-	struct kmmio_probe *probe;
-	unsigned long saved_flags;
-	int active;
-};
-
-static int kmmio_page_fault(struct pt_regs *regs, unsigned long error_code,
-						unsigned long address);
-static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val,
-								void *args);
-
-static DEFINE_SPINLOCK(kmmio_lock);
-
-/* These are protected by kmmio_lock */
-unsigned int kmmio_count;
-static unsigned int handler_registered;
-static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE];
-static LIST_HEAD(kmmio_probes);
-
-static struct kmmio_context kmmio_ctx[NR_CPUS];
-
-static struct pf_handler kmmio_pf_hook = {
-	.handler = kmmio_page_fault
-};
-
-static struct notifier_block nb_die = {
-	.notifier_call = kmmio_die_notifier
-};
-
-int init_kmmio(void)
-{
-	int i;
-	for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++)
-		INIT_LIST_HEAD(&kmmio_page_table[i]);
-
-	register_die_notifier(&nb_die);
-	return 0;
-}
-
-void cleanup_kmmio(void)
-{
-	/*
-	 * Assume the following have been already cleaned by calling
-	 * unregister_kmmio_probe() appropriately:
-	 * kmmio_page_table, kmmio_probes
-	 */
-	if (handler_registered) {
-		unregister_page_fault_handler(&kmmio_pf_hook);
-		synchronize_rcu();
-	}
-	unregister_die_notifier(&nb_die);
-}
-
-/*
- * this is basically a dynamic stabbing problem:
- * Could use the existing prio tree code or
- * Possible better implementations:
- * The Interval Skip List: A Data Structure for Finding All Intervals That
- * Overlap a Point (might be simple)
- * Space Efficient Dynamic Stabbing with Fast Queries - Mikkel Thorup
- */
-/* Get the kmmio at this addr (if any). You must be holding kmmio_lock. */
-static struct kmmio_probe *get_kmmio_probe(unsigned long addr)
-{
-	struct kmmio_probe *p;
-	list_for_each_entry(p, &kmmio_probes, list) {
-		if (addr >= p->addr && addr <= (p->addr + p->len))
-			return p;
-	}
-	return NULL;
-}
-
-static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page)
-{
-	struct list_head *head, *tmp;
-
-	page &= PAGE_MASK;
-	head = &kmmio_page_table[hash_long(page, KMMIO_PAGE_HASH_BITS)];
-	list_for_each(tmp, head) {
-		struct kmmio_fault_page *p
-			= list_entry(tmp, struct kmmio_fault_page, list);
-		if (p->page == page)
-			return p;
-	}
-
-	return NULL;
-}
-
-static void arm_kmmio_fault_page(unsigned long page, int *page_level)
-{
-	unsigned long address = page & PAGE_MASK;
-	int level;
-	pte_t *pte = lookup_address(address, &level);
-
-	if (!pte) {
-		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
-						__FUNCTION__, page);
-		return;
-	}
-
-	if (level == PG_LEVEL_2M) {
-		pmd_t *pmd = (pmd_t *)pte;
-		set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_PRESENT));
-	} else {
-		/* PG_LEVEL_4K */
-		set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
-	}
-
-	if (page_level)
-		*page_level = level;
-
-	__flush_tlb_one(page);
-}
-
-static void disarm_kmmio_fault_page(unsigned long page, int *page_level)
-{
-	unsigned long address = page & PAGE_MASK;
-	int level;
-	pte_t *pte = lookup_address(address, &level);
-
-	if (!pte) {
-		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
-						__FUNCTION__, page);
-		return;
-	}
-
-	if (level == PG_LEVEL_2M) {
-		pmd_t *pmd = (pmd_t *)pte;
-		set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_PRESENT));
-	} else {
-		/* PG_LEVEL_4K */
-		set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
-	}
-
-	if (page_level)
-		*page_level = level;
-
-	__flush_tlb_one(page);
-}
-
-/*
- * Interrupts are disabled on entry as trap3 is an interrupt gate
- * and they remain disabled thorough out this function.
- */
-static int kmmio_handler(struct pt_regs *regs, unsigned long addr)
-{
-	struct kmmio_context *ctx;
-	int cpu;
-
-	/*
-	 * Preemption is now disabled to prevent process switch during
-	 * single stepping. We can only handle one active kmmio trace
-	 * per cpu, so ensure that we finish it before something else
-	 * gets to run.
-	 *
-	 * XXX what if an interrupt occurs between returning from
-	 * do_page_fault() and entering the single-step exception handler?
-	 * And that interrupt triggers a kmmio trap?
-	 */
-	preempt_disable();
-	cpu = smp_processor_id();
-	ctx = &kmmio_ctx[cpu];
-
-	/* interrupts disabled and CPU-local data => atomicity guaranteed. */
-	if (ctx->active) {
-		/*
-		 * This avoids a deadlock with kmmio_lock.
-		 * If this page fault really was due to kmmio trap,
-		 * all hell breaks loose.
-		 */
-		printk(KERN_EMERG "mmiotrace: recursive probe hit on CPU %d, "
-					"for address %lu. Ignoring.\n",
-					cpu, addr);
-		goto no_kmmio;
-	}
-	ctx->active++;
-
-	/*
-	 * Acquire the kmmio lock to prevent changes affecting
-	 * get_kmmio_fault_page() and get_kmmio_probe(), since we save their
-	 * returned pointers.
-	 * The lock is released in post_kmmio_handler().
-	 * XXX: could/should get_kmmio_*() be using RCU instead of spinlock?
-	 */
-	spin_lock(&kmmio_lock);
-
-	ctx->fpage = get_kmmio_fault_page(addr);
-	if (!ctx->fpage) {
-		/* this page fault is not caused by kmmio */
-		goto no_kmmio_locked;
-	}
-
-	ctx->probe = get_kmmio_probe(addr);
-	ctx->saved_flags = (regs->flags & (TF_MASK|IF_MASK));
-
-	if (ctx->probe && ctx->probe->pre_handler)
-		ctx->probe->pre_handler(ctx->probe, regs, addr);
-
-	regs->flags |= TF_MASK;
-	regs->flags &= ~IF_MASK;
-
-	/* We hold lock, now we set present bit in PTE and single step. */
-	disarm_kmmio_fault_page(ctx->fpage->page, NULL);
-
-	return 1;
-
-no_kmmio_locked:
-	spin_unlock(&kmmio_lock);
-	ctx->active--;
-no_kmmio:
-	preempt_enable_no_resched();
-	/* page fault not handled by kmmio */
-	return 0;
-}
-
-/*
- * Interrupts are disabled on entry as trap1 is an interrupt gate
- * and they remain disabled thorough out this function.
- * And we hold kmmio lock.
- */
-static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
-{
-	int cpu = smp_processor_id();
-	struct kmmio_context *ctx = &kmmio_ctx[cpu];
-
-	if (!ctx->active)
-		return 0;
-
-	if (ctx->probe && ctx->probe->post_handler)
-		ctx->probe->post_handler(ctx->probe, condition, regs);
-
-	arm_kmmio_fault_page(ctx->fpage->page, NULL);
-
-	regs->flags &= ~TF_MASK;
-	regs->flags |= ctx->saved_flags;
-
-	/* These were acquired in kmmio_handler(). */
-	ctx->active--;
-	spin_unlock(&kmmio_lock);
-	preempt_enable_no_resched();
-
-	/*
-	 * if somebody else is singlestepping across a probe point, flags
-	 * will have TF set, in which case, continue the remaining processing
-	 * of do_debug, as if this is not a probe hit.
-	 */
-	if (regs->flags & TF_MASK)
-		return 0;
-
-	return 1;
-}
-
-static int add_kmmio_fault_page(unsigned long page)
-{
-	struct kmmio_fault_page *f;
-
-	page &= PAGE_MASK;
-	f = get_kmmio_fault_page(page);
-	if (f) {
-		f->count++;
-		return 0;
-	}
-
-	f = kmalloc(sizeof(*f), GFP_ATOMIC);
-	if (!f)
-		return -1;
-
-	f->count = 1;
-	f->page = page;
-	list_add(&f->list,
-		 &kmmio_page_table[hash_long(f->page, KMMIO_PAGE_HASH_BITS)]);
-
-	arm_kmmio_fault_page(f->page, NULL);
-
-	return 0;
-}
-
-static void release_kmmio_fault_page(unsigned long page)
-{
-	struct kmmio_fault_page *f;
-
-	page &= PAGE_MASK;
-	f = get_kmmio_fault_page(page);
-	if (!f)
-		return;
-
-	f->count--;
-	if (!f->count) {
-		disarm_kmmio_fault_page(f->page, NULL);
-		list_del(&f->list);
-	}
-}
-
-int register_kmmio_probe(struct kmmio_probe *p)
-{
-	int ret = 0;
-	unsigned long size = 0;
-
-	spin_lock_irq(&kmmio_lock);
-	kmmio_count++;
-	if (get_kmmio_probe(p->addr)) {
-		ret = -EEXIST;
-		goto out;
-	}
-	list_add(&p->list, &kmmio_probes);
-	/*printk("adding fault pages...\n");*/
-	while (size < p->len) {
-		if (add_kmmio_fault_page(p->addr + size))
-			printk(KERN_ERR "mmio: Unable to set page fault.\n");
-		size += PAGE_SIZE;
-	}
-
-	if (!handler_registered) {
-		register_page_fault_handler(&kmmio_pf_hook);
-		handler_registered++;
-	}
-
-out:
-	spin_unlock_irq(&kmmio_lock);
-	/*
-	 * XXX: What should I do here?
-	 * Here was a call to global_flush_tlb(), but it does not exist
-	 * anymore.
-	 */
-	return ret;
-}
-
-void unregister_kmmio_probe(struct kmmio_probe *p)
-{
-	unsigned long size = 0;
-
-	spin_lock_irq(&kmmio_lock);
-	while (size < p->len) {
-		release_kmmio_fault_page(p->addr + size);
-		size += PAGE_SIZE;
-	}
-	list_del(&p->list);
-	kmmio_count--;
-	spin_unlock_irq(&kmmio_lock);
-}
-
-/*
- * According to 2.6.20, mainly x86_64 arch:
- * This is being called from do_page_fault(), via the page fault notifier
- * chain. The chain is called for both user space faults and kernel space
- * faults (address >= TASK_SIZE64), except not on faults serviced by
- * vmalloc_fault().
- *
- * We may be in an interrupt or a critical section. Also prefecthing may
- * trigger a page fault. We may be in the middle of process switch.
- * The page fault hook functionality has put us inside RCU read lock.
- *
- * Local interrupts are disabled, so preemption cannot happen.
- * Do not enable interrupts, do not sleep, and watch out for other CPUs.
- */
-static int kmmio_page_fault(struct pt_regs *regs, unsigned long error_code,
-						unsigned long address)
-{
-	if (is_kmmio_active())
-		if (kmmio_handler(regs, address) == 1)
-			return -1;
-	return 0;
-}
-
-static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val,
-								void *args)
-{
-	struct die_args *arg = args;
-
-	if (val == DIE_DEBUG)
-		if (post_kmmio_handler(arg->err, arg->regs) == 1)
-			return NOTIFY_STOP;
-
-	return NOTIFY_DONE;
-}
diff --git a/arch/x86/kernel/mmiotrace/kmmio.h b/arch/x86/kernel/mmiotrace/kmmio.h
deleted file mode 100644
index 85b7f68..0000000
--- a/arch/x86/kernel/mmiotrace/kmmio.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#ifndef _LINUX_KMMIO_H
-#define _LINUX_KMMIO_H
-
-#include <linux/list.h>
-#include <linux/notifier.h>
-#include <linux/smp.h>
-#include <linux/types.h>
-#include <linux/ptrace.h>
-#include <linux/version.h>
-#include <linux/kdebug.h>
-
-struct kmmio_probe;
-struct kmmio_fault_page;
-struct pt_regs;
-
-typedef void (*kmmio_pre_handler_t)(struct kmmio_probe *,
-				struct pt_regs *, unsigned long addr);
-typedef void (*kmmio_post_handler_t)(struct kmmio_probe *,
-				unsigned long condition, struct pt_regs *);
-
-struct kmmio_probe {
-	struct list_head list;
-
-	/* start location of the probe point */
-	unsigned long addr;
-
-	/* length of the probe region */
-	unsigned long len;
-
-	 /* Called before addr is executed. */
-	kmmio_pre_handler_t pre_handler;
-
-	/* Called after addr is executed, unless... */
-	kmmio_post_handler_t post_handler;
-};
-
-struct kmmio_fault_page {
-	struct list_head list;
-
-	/* location of the fault page */
-	unsigned long page;
-
-	int count;
-};
-
-/* kmmio is active by some kmmio_probes? */
-static inline int is_kmmio_active(void)
-{
-	extern unsigned int kmmio_count;
-	return kmmio_count;
-}
-
-int init_kmmio(void);
-void cleanup_kmmio(void);
-int register_kmmio_probe(struct kmmio_probe *p);
-void unregister_kmmio_probe(struct kmmio_probe *p);
-
-#endif /* _LINUX_KMMIO_H */
diff --git a/arch/x86/kernel/mmiotrace/mmio-mod.c b/arch/x86/kernel/mmiotrace/mmio-mod.c
deleted file mode 100644
index f492b65..0000000
--- a/arch/x86/kernel/mmiotrace/mmio-mod.c
+++ /dev/null
@@ -1,538 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright (C) IBM Corporation, 2005
- *               Jeff Muizelaar, 2006, 2007
- *               Pekka Paalanen, 2008 <pq@iki.fi>
- *
- * Derived from the read-mod example from relay-examples by Tom Zanussi.
- */
-#include <linux/module.h>
-#include <linux/relay.h>
-#include <linux/debugfs.h>
-#include <linux/proc_fs.h>
-#include <asm/io.h>
-#include <linux/version.h>
-#include <linux/kallsyms.h>
-#include <asm/pgtable.h>
-#include <linux/mmiotrace.h>
-#include <asm/e820.h> /* for ISA_START_ADDRESS */
-#include <asm/atomic.h>
-
-#include "kmmio.h"
-#include "pf_in.h"
-
-/* This app's relay channel files will appear in /debug/mmio-trace */
-#define APP_DIR		"mmio-trace"
-/* the marker injection file in /proc */
-#define MARKER_FILE     "mmio-marker"
-
-#define MODULE_NAME     "mmiotrace"
-
-struct trap_reason {
-	unsigned long addr;
-	unsigned long ip;
-	enum reason_type type;
-	int active_traces;
-};
-
-/* Accessed per-cpu. */
-static struct trap_reason pf_reason[NR_CPUS];
-static struct mm_io_header_rw cpu_trace[NR_CPUS];
-
-/* Access to this is not per-cpu. */
-static atomic_t dropped[NR_CPUS];
-
-static struct file_operations mmio_fops = {
-	.owner = THIS_MODULE,
-};
-
-static const size_t subbuf_size = 256*1024;
-static struct rchan *chan;
-static struct dentry *dir;
-static struct proc_dir_entry *proc_marker_file;
-
-/* module parameters */
-static unsigned int      n_subbufs = 32*4;
-static unsigned long filter_offset;
-static int                 notrace;
-static int               ISA_trace;
-static int                trace_pc;
-
-module_param(n_subbufs, uint, 0);
-module_param(filter_offset, ulong, 0);
-module_param(notrace, bool, 0);
-module_param(ISA_trace, bool, 0);
-module_param(trace_pc, bool, 0);
-
-MODULE_PARM_DESC(n_subbufs, "Number of 256kB buffers, default 128.");
-MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
-MODULE_PARM_DESC(notrace, "Disable actual MMIO tracing.");
-MODULE_PARM_DESC(ISA_trace, "Do not exclude the low ISA range.");
-MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
-
-static void record_timestamp(struct mm_io_header *header)
-{
-	struct timespec now;
-
-	getnstimeofday(&now);
-	header->sec = now.tv_sec;
-	header->nsec = now.tv_nsec;
-}
-
-/*
- * Write callback for the /proc entry:
- * Read a marker and write it to the mmio trace log
- */
-static int write_marker(struct file *file, const char __user *buffer,
-					unsigned long count, void *data)
-{
-	char *event = NULL;
-	struct mm_io_header *headp;
-	int len = (count > 65535) ? 65535 : count;
-
-	event = kzalloc(sizeof(*headp) + len, GFP_KERNEL);
-	if (!event)
-		return -ENOMEM;
-
-	headp = (struct mm_io_header *)event;
-	headp->type = MMIO_MAGIC | (MMIO_MARKER << MMIO_OPCODE_SHIFT);
-	headp->data_len = len;
-	record_timestamp(headp);
-
-	if (copy_from_user(event + sizeof(*headp), buffer, len)) {
-		kfree(event);
-		return -EFAULT;
-	}
-
-	relay_write(chan, event, sizeof(*headp) + len);
-	kfree(event);
-	return len;
-}
-
-static void print_pte(unsigned long address)
-{
-	int level;
-	pte_t *pte = lookup_address(address, &level);
-
-	if (!pte) {
-		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
-						__FUNCTION__, address);
-		return;
-	}
-
-	if (level == PG_LEVEL_2M) {
-		printk(KERN_EMERG MODULE_NAME ": 4MB pages are not "
-						"currently supported: %lx\n",
-						address);
-		BUG();
-	}
-	printk(KERN_DEBUG MODULE_NAME ": pte for 0x%lx: 0x%lx 0x%lx\n",
-					address, pte_val(*pte),
-					pte_val(*pte) & _PAGE_PRESENT);
-}
-
-/*
- * For some reason the pre/post pairs have been called in an
- * unmatched order. Report and die.
- */
-static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
-{
-	const unsigned long cpu = smp_processor_id();
-	printk(KERN_EMERG MODULE_NAME ": unexpected fault for address: %lx, "
-					"last fault for address: %lx\n",
-					addr, pf_reason[cpu].addr);
-	print_pte(addr);
-#ifdef __i386__
-	print_symbol(KERN_EMERG "faulting EIP is at %s\n", regs->ip);
-	print_symbol(KERN_EMERG "last faulting EIP was at %s\n",
-							pf_reason[cpu].ip);
-	printk(KERN_EMERG
-			"eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
-			regs->ax, regs->bx, regs->cx, regs->dx);
-	printk(KERN_EMERG
-			"esi: %08lx   edi: %08lx   ebp: %08lx   esp: %08lx\n",
-			regs->si, regs->di, regs->bp, regs->sp);
-#else
-	print_symbol(KERN_EMERG "faulting RIP is at %s\n", regs->ip);
-	print_symbol(KERN_EMERG "last faulting RIP was at %s\n",
-							pf_reason[cpu].ip);
-	printk(KERN_EMERG "rax: %016lx   rcx: %016lx   rdx: %016lx\n",
-					regs->ax, regs->cx, regs->dx);
-	printk(KERN_EMERG "rsi: %016lx   rdi: %016lx   "
-				"rbp: %016lx   rsp: %016lx\n",
-				regs->si, regs->di, regs->bp, regs->sp);
-#endif
-	BUG();
-}
-
-static void pre(struct kmmio_probe *p, struct pt_regs *regs,
-						unsigned long addr)
-{
-	const unsigned long cpu = smp_processor_id();
-	const unsigned long instptr = instruction_pointer(regs);
-	const enum reason_type type = get_ins_type(instptr);
-
-	/* it doesn't make sense to have more than one active trace per cpu */
-	if (pf_reason[cpu].active_traces)
-		die_kmmio_nesting_error(regs, addr);
-	else
-		pf_reason[cpu].active_traces++;
-
-	pf_reason[cpu].type = type;
-	pf_reason[cpu].addr = addr;
-	pf_reason[cpu].ip = instptr;
-
-	cpu_trace[cpu].header.type = MMIO_MAGIC;
-	cpu_trace[cpu].header.pid = 0;
-	cpu_trace[cpu].header.data_len = sizeof(struct mm_io_rw);
-	cpu_trace[cpu].rw.address = addr;
-
-	/*
-	 * Only record the program counter when requested.
-	 * It may taint clean-room reverse engineering.
-	 */
-	if (trace_pc)
-		cpu_trace[cpu].rw.pc = instptr;
-	else
-		cpu_trace[cpu].rw.pc = 0;
-
-	record_timestamp(&cpu_trace[cpu].header);
-
-	switch (type) {
-	case REG_READ:
-		cpu_trace[cpu].header.type |=
-			(MMIO_READ << MMIO_OPCODE_SHIFT) |
-			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
-		break;
-	case REG_WRITE:
-		cpu_trace[cpu].header.type |=
-			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
-			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
-		cpu_trace[cpu].rw.value = get_ins_reg_val(instptr, regs);
-		break;
-	case IMM_WRITE:
-		cpu_trace[cpu].header.type |=
-			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
-			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
-		cpu_trace[cpu].rw.value = get_ins_imm_val(instptr);
-		break;
-	default:
-		{
-			unsigned char *ip = (unsigned char *)instptr;
-			cpu_trace[cpu].header.type |=
-					(MMIO_UNKNOWN_OP << MMIO_OPCODE_SHIFT);
-			cpu_trace[cpu].rw.value = (*ip) << 16 |
-							*(ip + 1) << 8 |
-							*(ip + 2);
-		}
-	}
-}
-
-static void post(struct kmmio_probe *p, unsigned long condition,
-							struct pt_regs *regs)
-{
-	const unsigned long cpu = smp_processor_id();
-
-	/* this should always return the active_trace count to 0 */
-	pf_reason[cpu].active_traces--;
-	if (pf_reason[cpu].active_traces) {
-		printk(KERN_EMERG MODULE_NAME ": unexpected post handler");
-		BUG();
-	}
-
-	switch (pf_reason[cpu].type) {
-	case REG_READ:
-		cpu_trace[cpu].rw.value = get_ins_reg_val(pf_reason[cpu].ip,
-									regs);
-		break;
-	default:
-		break;
-	}
-	relay_write(chan, &cpu_trace[cpu], sizeof(struct mm_io_header_rw));
-}
-
-/*
- * subbuf_start() relay callback.
- *
- * Defined so that we know when events are dropped due to the buffer-full
- * condition.
- */
-static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf,
-					void *prev_subbuf, size_t prev_padding)
-{
-	unsigned int cpu = buf->cpu;
-	atomic_t *drop = &dropped[cpu];
-	int count;
-	if (relay_buf_full(buf)) {
-		if (atomic_inc_return(drop) == 1) {
-			printk(KERN_ERR MODULE_NAME ": cpu %d buffer full!\n",
-									cpu);
-		}
-		return 0;
-	} else if ((count = atomic_read(drop))) {
-		printk(KERN_ERR MODULE_NAME
-					": cpu %d buffer no longer full, "
-					"missed %d events.\n",
-					cpu, count);
-		atomic_sub(count, drop);
-	}
-
-	return 1;
-}
-
-/* file_create() callback.  Creates relay file in debugfs. */
-static struct dentry *create_buf_file_handler(const char *filename,
-						struct dentry *parent,
-						int mode,
-						struct rchan_buf *buf,
-						int *is_global)
-{
-	struct dentry *buf_file;
-
-	mmio_fops.read = relay_file_operations.read;
-	mmio_fops.open = relay_file_operations.open;
-	mmio_fops.poll = relay_file_operations.poll;
-	mmio_fops.mmap = relay_file_operations.mmap;
-	mmio_fops.release = relay_file_operations.release;
-	mmio_fops.splice_read = relay_file_operations.splice_read;
-
-	buf_file = debugfs_create_file(filename, mode, parent, buf,
-								&mmio_fops);
-
-	return buf_file;
-}
-
-/* file_remove() default callback.  Removes relay file in debugfs. */
-static int remove_buf_file_handler(struct dentry *dentry)
-{
-	debugfs_remove(dentry);
-	return 0;
-}
-
-static struct rchan_callbacks relay_callbacks = {
-	.subbuf_start = subbuf_start_handler,
-	.create_buf_file = create_buf_file_handler,
-	.remove_buf_file = remove_buf_file_handler,
-};
-
-/*
- * create_channel - creates channel /debug/APP_DIR/cpuXXX
- * Returns channel on success, NULL otherwise
- */
-static struct rchan *create_channel(unsigned size, unsigned n)
-{
-	return relay_open("cpu", dir, size, n, &relay_callbacks, NULL);
-}
-
-/* destroy_channel - destroys channel /debug/APP_DIR/cpuXXX */
-static void destroy_channel(void)
-{
-	if (chan) {
-		relay_close(chan);
-		chan = NULL;
-	}
-}
-
-struct remap_trace {
-	struct list_head list;
-	struct kmmio_probe probe;
-};
-static LIST_HEAD(trace_list);
-static DEFINE_SPINLOCK(trace_list_lock);
-
-static void do_ioremap_trace_core(unsigned long offset, unsigned long size,
-							void __iomem *addr)
-{
-	struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
-	struct mm_io_header_map event = {
-		.header = {
-			.type = MMIO_MAGIC |
-					(MMIO_PROBE << MMIO_OPCODE_SHIFT),
-			.sec = 0,
-			.nsec = 0,
-			.pid = 0,
-			.data_len = sizeof(struct mm_io_map)
-		},
-		.map = {
-			.phys = offset,
-			.addr = (unsigned long)addr,
-			.len  = size,
-			.pc   = 0
-		}
-	};
-	record_timestamp(&event.header);
-
-	*trace = (struct remap_trace) {
-		.probe = {
-			.addr = (unsigned long)addr,
-			.len = size,
-			.pre_handler = pre,
-			.post_handler = post,
-		}
-	};
-
-	relay_write(chan, &event, sizeof(event));
-	spin_lock(&trace_list_lock);
-	list_add_tail(&trace->list, &trace_list);
-	spin_unlock(&trace_list_lock);
-	if (!notrace)
-		register_kmmio_probe(&trace->probe);
-}
-
-static void ioremap_trace_core(unsigned long offset, unsigned long size,
-							void __iomem *addr)
-{
-	if ((filter_offset) && (offset != filter_offset))
-		return;
-
-	/* Don't trace the low PCI/ISA area, it's always mapped.. */
-	if (!ISA_trace && (offset < ISA_END_ADDRESS) &&
-					(offset + size > ISA_START_ADDRESS)) {
-		printk(KERN_NOTICE MODULE_NAME ": Ignoring map of low "
-						"PCI/ISA area (0x%lx-0x%lx)\n",
-						offset, offset + size);
-		return;
-	}
-	do_ioremap_trace_core(offset, size, addr);
-}
-
-void __iomem *ioremap_cache_trace(unsigned long offset, unsigned long size)
-{
-	void __iomem *p = ioremap_cache(offset, size);
-	printk(KERN_DEBUG MODULE_NAME ": ioremap_cache(0x%lx, 0x%lx) = %p\n",
-							offset, size, p);
-	ioremap_trace_core(offset, size, p);
-	return p;
-}
-EXPORT_SYMBOL(ioremap_cache_trace);
-
-void __iomem *ioremap_nocache_trace(unsigned long offset, unsigned long size)
-{
-	void __iomem *p = ioremap_nocache(offset, size);
-	printk(KERN_DEBUG MODULE_NAME ": ioremap_nocache(0x%lx, 0x%lx) = %p\n",
-							offset, size, p);
-	ioremap_trace_core(offset, size, p);
-	return p;
-}
-EXPORT_SYMBOL(ioremap_nocache_trace);
-
-void iounmap_trace(volatile void __iomem *addr)
-{
-	struct mm_io_header_map event = {
-		.header = {
-			.type = MMIO_MAGIC |
-				(MMIO_UNPROBE << MMIO_OPCODE_SHIFT),
-			.sec = 0,
-			.nsec = 0,
-			.pid = 0,
-			.data_len = sizeof(struct mm_io_map)
-		},
-		.map = {
-			.phys = 0,
-			.addr = (unsigned long)addr,
-			.len  = 0,
-			.pc   = 0
-		}
-	};
-	struct remap_trace *trace;
-	struct remap_trace *tmp;
-	printk(KERN_DEBUG MODULE_NAME ": Unmapping %p.\n", addr);
-	record_timestamp(&event.header);
-
-	spin_lock(&trace_list_lock);
-	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
-		if ((unsigned long)addr == trace->probe.addr) {
-			if (!notrace)
-				unregister_kmmio_probe(&trace->probe);
-			list_del(&trace->list);
-			kfree(trace);
-			break;
-		}
-	}
-	spin_unlock(&trace_list_lock);
-	relay_write(chan, &event, sizeof(event));
-	iounmap(addr);
-}
-EXPORT_SYMBOL(iounmap_trace);
-
-static void clear_trace_list(void)
-{
-	struct remap_trace *trace;
-	struct remap_trace *tmp;
-
-	spin_lock(&trace_list_lock);
-	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
-		printk(KERN_WARNING MODULE_NAME ": purging non-iounmapped "
-					"trace @0x%08lx, size 0x%lx.\n",
-					trace->probe.addr, trace->probe.len);
-		if (!notrace)
-			unregister_kmmio_probe(&trace->probe);
-		list_del(&trace->list);
-		kfree(trace);
-		break;
-	}
-	spin_unlock(&trace_list_lock);
-}
-
-static int __init init(void)
-{
-	if (n_subbufs < 2)
-		return -EINVAL;
-
-	dir = debugfs_create_dir(APP_DIR, NULL);
-	if (!dir) {
-		printk(KERN_ERR MODULE_NAME
-				": Couldn't create relay app directory.\n");
-		return -ENOMEM;
-	}
-
-	chan = create_channel(subbuf_size, n_subbufs);
-	if (!chan) {
-		debugfs_remove(dir);
-		printk(KERN_ERR MODULE_NAME
-				": relay app channel creation failed\n");
-		return -ENOMEM;
-	}
-
-	init_kmmio();
-
-	proc_marker_file = create_proc_entry(MARKER_FILE, 0, NULL);
-	if (proc_marker_file)
-		proc_marker_file->write_proc = write_marker;
-
-	printk(KERN_DEBUG MODULE_NAME ": loaded.\n");
-	if (notrace)
-		printk(KERN_DEBUG MODULE_NAME ": MMIO tracing disabled.\n");
-	if (ISA_trace)
-		printk(KERN_WARNING MODULE_NAME
-				": Warning! low ISA range will be traced.\n");
-	return 0;
-}
-
-static void __exit cleanup(void)
-{
-	printk(KERN_DEBUG MODULE_NAME ": unload...\n");
-	clear_trace_list();
-	cleanup_kmmio();
-	remove_proc_entry(MARKER_FILE, NULL);
-	destroy_channel();
-	if (dir)
-		debugfs_remove(dir);
-}
-
-module_init(init);
-module_exit(cleanup);
-MODULE_LICENSE("GPL");
diff --git a/arch/x86/kernel/mmiotrace/pf_in.c b/arch/x86/kernel/mmiotrace/pf_in.c
deleted file mode 100644
index 67ea520..0000000
--- a/arch/x86/kernel/mmiotrace/pf_in.c
+++ /dev/null
@@ -1,489 +0,0 @@
-/*
- *  Fault Injection Test harness (FI)
- *  Copyright (C) Intel Crop.
- *
- *  This program is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU General Public License
- *  as published by the Free Software Foundation; either version 2
- *  of the License, or (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
- *  USA.
- *
- */
-
-/*  $Id: pf_in.c,v 1.1.1.1 2002/11/12 05:56:32 brlock Exp $
- *  Copyright by Intel Crop., 2002
- *  Louis Zhuang (louis.zhuang@intel.com)
- *
- *  Bjorn Steinbrink (B.Steinbrink@gmx.de), 2007
- */
-
-#include <linux/module.h>
-#include <linux/ptrace.h> /* struct pt_regs */
-#include "pf_in.h"
-
-#ifdef __i386__
-/* IA32 Manual 3, 2-1 */
-static unsigned char prefix_codes[] = {
-	0xF0, 0xF2, 0xF3, 0x2E, 0x36, 0x3E, 0x26, 0x64,
-	0x65, 0x2E, 0x3E, 0x66, 0x67
-};
-/* IA32 Manual 3, 3-432*/
-static unsigned int reg_rop[] = {
-	0x8A, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
-};
-static unsigned int reg_wop[] = { 0x88, 0x89 };
-static unsigned int imm_wop[] = { 0xC6, 0xC7 };
-/* IA32 Manual 3, 3-432*/
-static unsigned int rw8[] = { 0x88, 0x8A, 0xC6 };
-static unsigned int rw32[] = {
-	0x89, 0x8B, 0xC7, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
-};
-static unsigned int mw8[] = { 0x88, 0x8A, 0xC6, 0xB60F, 0xBE0F };
-static unsigned int mw16[] = { 0xB70F, 0xBF0F };
-static unsigned int mw32[] = { 0x89, 0x8B, 0xC7 };
-static unsigned int mw64[] = {};
-#else /* not __i386__ */
-static unsigned char prefix_codes[] = {
-	0x66, 0x67, 0x2E, 0x3E, 0x26, 0x64, 0x65, 0x36,
-	0xF0, 0xF3, 0xF2,
-	/* REX Prefixes */
-	0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
-	0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f
-};
-/* AMD64 Manual 3, Appendix A*/
-static unsigned int reg_rop[] = {
-	0x8A, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
-};
-static unsigned int reg_wop[] = { 0x88, 0x89 };
-static unsigned int imm_wop[] = { 0xC6, 0xC7 };
-static unsigned int rw8[] = { 0xC6, 0x88, 0x8A };
-static unsigned int rw32[] = {
-	0xC7, 0x89, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
-};
-/* 8 bit only */
-static unsigned int mw8[] = { 0xC6, 0x88, 0x8A, 0xB60F, 0xBE0F };
-/* 16 bit only */
-static unsigned int mw16[] = { 0xB70F, 0xBF0F };
-/* 16 or 32 bit */
-static unsigned int mw32[] = { 0xC7 };
-/* 16, 32 or 64 bit */
-static unsigned int mw64[] = { 0x89, 0x8B };
-#endif /* not __i386__ */
-
-static int skip_prefix(unsigned char *addr, int *shorted, int *enlarged,
-								int *rexr)
-{
-	int i;
-	unsigned char *p = addr;
-	*shorted = 0;
-	*enlarged = 0;
-	*rexr = 0;
-
-restart:
-	for (i = 0; i < ARRAY_SIZE(prefix_codes); i++) {
-		if (*p == prefix_codes[i]) {
-			if (*p == 0x66)
-				*shorted = 1;
-#ifdef __amd64__
-			if ((*p & 0xf8) == 0x48)
-				*enlarged = 1;
-			if ((*p & 0xf4) == 0x44)
-				*rexr = 1;
-#endif
-			p++;
-			goto restart;
-		}
-	}
-
-	return (p - addr);
-}
-
-static int get_opcode(unsigned char *addr, unsigned int *opcode)
-{
-	int len;
-
-	if (*addr == 0x0F) {
-		/* 0x0F is extension instruction */
-		*opcode = *(unsigned short *)addr;
-		len = 2;
-	} else {
-		*opcode = *addr;
-		len = 1;
-	}
-
-	return len;
-}
-
-#define CHECK_OP_TYPE(opcode, array, type) \
-	for (i = 0; i < ARRAY_SIZE(array); i++) { \
-		if (array[i] == opcode) { \
-			rv = type; \
-			goto exit; \
-		} \
-	}
-
-enum reason_type get_ins_type(unsigned long ins_addr)
-{
-	unsigned int opcode;
-	unsigned char *p;
-	int shorted, enlarged, rexr;
-	int i;
-	enum reason_type rv = OTHERS;
-
-	p = (unsigned char *)ins_addr;
-	p += skip_prefix(p, &shorted, &enlarged, &rexr);
-	p += get_opcode(p, &opcode);
-
-	CHECK_OP_TYPE(opcode, reg_rop, REG_READ);
-	CHECK_OP_TYPE(opcode, reg_wop, REG_WRITE);
-	CHECK_OP_TYPE(opcode, imm_wop, IMM_WRITE);
-
-exit:
-	return rv;
-}
-#undef CHECK_OP_TYPE
-
-static unsigned int get_ins_reg_width(unsigned long ins_addr)
-{
-	unsigned int opcode;
-	unsigned char *p;
-	int i, shorted, enlarged, rexr;
-
-	p = (unsigned char *)ins_addr;
-	p += skip_prefix(p, &shorted, &enlarged, &rexr);
-	p += get_opcode(p, &opcode);
-
-	for (i = 0; i < ARRAY_SIZE(rw8); i++)
-		if (rw8[i] == opcode)
-			return 1;
-
-	for (i = 0; i < ARRAY_SIZE(rw32); i++)
-		if (rw32[i] == opcode)
-			return (shorted ? 2 : (enlarged ? 8 : 4));
-
-	printk(KERN_ERR "mmiotrace: Unknown opcode 0x%02x\n", opcode);
-	return 0;
-}
-
-unsigned int get_ins_mem_width(unsigned long ins_addr)
-{
-	unsigned int opcode;
-	unsigned char *p;
-	int i, shorted, enlarged, rexr;
-
-	p = (unsigned char *)ins_addr;
-	p += skip_prefix(p, &shorted, &enlarged, &rexr);
-	p += get_opcode(p, &opcode);
-
-	for (i = 0; i < ARRAY_SIZE(mw8); i++)
-		if (mw8[i] == opcode)
-			return 1;
-
-	for (i = 0; i < ARRAY_SIZE(mw16); i++)
-		if (mw16[i] == opcode)
-			return 2;
-
-	for (i = 0; i < ARRAY_SIZE(mw32); i++)
-		if (mw32[i] == opcode)
-			return shorted ? 2 : 4;
-
-	for (i = 0; i < ARRAY_SIZE(mw64); i++)
-		if (mw64[i] == opcode)
-			return shorted ? 2 : (enlarged ? 8 : 4);
-
-	printk(KERN_ERR "mmiotrace: Unknown opcode 0x%02x\n", opcode);
-	return 0;
-}
-
-/*
- * Define register ident in mod/rm byte.
- * Note: these are NOT the same as in ptrace-abi.h.
- */
-enum {
-	arg_AL = 0,
-	arg_CL = 1,
-	arg_DL = 2,
-	arg_BL = 3,
-	arg_AH = 4,
-	arg_CH = 5,
-	arg_DH = 6,
-	arg_BH = 7,
-
-	arg_AX = 0,
-	arg_CX = 1,
-	arg_DX = 2,
-	arg_BX = 3,
-	arg_SP = 4,
-	arg_BP = 5,
-	arg_SI = 6,
-	arg_DI = 7,
-#ifdef __amd64__
-	arg_R8  = 8,
-	arg_R9  = 9,
-	arg_R10 = 10,
-	arg_R11 = 11,
-	arg_R12 = 12,
-	arg_R13 = 13,
-	arg_R14 = 14,
-	arg_R15 = 15
-#endif
-};
-
-static unsigned char *get_reg_w8(int no, struct pt_regs *regs)
-{
-	unsigned char *rv = NULL;
-
-	switch (no) {
-	case arg_AL:
-		rv = (unsigned char *)&regs->ax;
-		break;
-	case arg_BL:
-		rv = (unsigned char *)&regs->bx;
-		break;
-	case arg_CL:
-		rv = (unsigned char *)&regs->cx;
-		break;
-	case arg_DL:
-		rv = (unsigned char *)&regs->dx;
-		break;
-	case arg_AH:
-		rv = 1 + (unsigned char *)&regs->ax;
-		break;
-	case arg_BH:
-		rv = 1 + (unsigned char *)&regs->bx;
-		break;
-	case arg_CH:
-		rv = 1 + (unsigned char *)&regs->cx;
-		break;
-	case arg_DH:
-		rv = 1 + (unsigned char *)&regs->dx;
-		break;
-#ifdef __amd64__
-	case arg_R8:
-		rv = (unsigned char *)&regs->r8;
-		break;
-	case arg_R9:
-		rv = (unsigned char *)&regs->r9;
-		break;
-	case arg_R10:
-		rv = (unsigned char *)&regs->r10;
-		break;
-	case arg_R11:
-		rv = (unsigned char *)&regs->r11;
-		break;
-	case arg_R12:
-		rv = (unsigned char *)&regs->r12;
-		break;
-	case arg_R13:
-		rv = (unsigned char *)&regs->r13;
-		break;
-	case arg_R14:
-		rv = (unsigned char *)&regs->r14;
-		break;
-	case arg_R15:
-		rv = (unsigned char *)&regs->r15;
-		break;
-#endif
-	default:
-		printk(KERN_ERR "mmiotrace: Error reg no# %d\n", no);
-		break;
-	}
-	return rv;
-}
-
-static unsigned long *get_reg_w32(int no, struct pt_regs *regs)
-{
-	unsigned long *rv = NULL;
-
-	switch (no) {
-	case arg_AX:
-		rv = &regs->ax;
-		break;
-	case arg_BX:
-		rv = &regs->bx;
-		break;
-	case arg_CX:
-		rv = &regs->cx;
-		break;
-	case arg_DX:
-		rv = &regs->dx;
-		break;
-	case arg_SP:
-		rv = &regs->sp;
-		break;
-	case arg_BP:
-		rv = &regs->bp;
-		break;
-	case arg_SI:
-		rv = &regs->si;
-		break;
-	case arg_DI:
-		rv = &regs->di;
-		break;
-#ifdef __amd64__
-	case arg_R8:
-		rv = &regs->r8;
-		break;
-	case arg_R9:
-		rv = &regs->r9;
-		break;
-	case arg_R10:
-		rv = &regs->r10;
-		break;
-	case arg_R11:
-		rv = &regs->r11;
-		break;
-	case arg_R12:
-		rv = &regs->r12;
-		break;
-	case arg_R13:
-		rv = &regs->r13;
-		break;
-	case arg_R14:
-		rv = &regs->r14;
-		break;
-	case arg_R15:
-		rv = &regs->r15;
-		break;
-#endif
-	default:
-		printk(KERN_ERR "mmiotrace: Error reg no# %d\n", no);
-	}
-
-	return rv;
-}
-
-unsigned long get_ins_reg_val(unsigned long ins_addr, struct pt_regs *regs)
-{
-	unsigned int opcode;
-	unsigned char mod_rm;
-	int reg;
-	unsigned char *p;
-	int i, shorted, enlarged, rexr;
-	unsigned long rv;
-
-	p = (unsigned char *)ins_addr;
-	p += skip_prefix(p, &shorted, &enlarged, &rexr);
-	p += get_opcode(p, &opcode);
-	for (i = 0; i < ARRAY_SIZE(reg_rop); i++)
-		if (reg_rop[i] == opcode) {
-			rv = REG_READ;
-			goto do_work;
-		}
-
-	for (i = 0; i < ARRAY_SIZE(reg_wop); i++)
-		if (reg_wop[i] == opcode) {
-			rv = REG_WRITE;
-			goto do_work;
-		}
-
-	printk(KERN_ERR "mmiotrace: Not a register instruction, opcode "
-							"0x%02x\n", opcode);
-	goto err;
-
-do_work:
-	mod_rm = *p;
-	reg = ((mod_rm >> 3) & 0x7) | (rexr << 3);
-	switch (get_ins_reg_width(ins_addr)) {
-	case 1:
-		return *get_reg_w8(reg, regs);
-
-	case 2:
-		return *(unsigned short *)get_reg_w32(reg, regs);
-
-	case 4:
-		return *(unsigned int *)get_reg_w32(reg, regs);
-
-#ifdef __amd64__
-	case 8:
-		return *(unsigned long *)get_reg_w32(reg, regs);
-#endif
-
-	default:
-		printk(KERN_ERR "mmiotrace: Error width# %d\n", reg);
-	}
-
-err:
-	return 0;
-}
-
-unsigned long get_ins_imm_val(unsigned long ins_addr)
-{
-	unsigned int opcode;
-	unsigned char mod_rm;
-	unsigned char mod;
-	unsigned char *p;
-	int i, shorted, enlarged, rexr;
-	unsigned long rv;
-
-	p = (unsigned char *)ins_addr;
-	p += skip_prefix(p, &shorted, &enlarged, &rexr);
-	p += get_opcode(p, &opcode);
-	for (i = 0; i < ARRAY_SIZE(imm_wop); i++)
-		if (imm_wop[i] == opcode) {
-			rv = IMM_WRITE;
-			goto do_work;
-		}
-
-	printk(KERN_ERR "mmiotrace: Not an immediate instruction, opcode "
-							"0x%02x\n", opcode);
-	goto err;
-
-do_work:
-	mod_rm = *p;
-	mod = mod_rm >> 6;
-	p++;
-	switch (mod) {
-	case 0:
-		/* if r/m is 5 we have a 32 disp (IA32 Manual 3, Table 2-2)  */
-		/* AMD64: XXX Check for address size prefix? */
-		if ((mod_rm & 0x7) == 0x5)
-			p += 4;
-		break;
-
-	case 1:
-		p += 1;
-		break;
-
-	case 2:
-		p += 4;
-		break;
-
-	case 3:
-	default:
-		printk(KERN_ERR "mmiotrace: not a memory access instruction "
-						"at 0x%lx, rm_mod=0x%02x\n",
-						ins_addr, mod_rm);
-	}
-
-	switch (get_ins_reg_width(ins_addr)) {
-	case 1:
-		return *(unsigned char *)p;
-
-	case 2:
-		return *(unsigned short *)p;
-
-	case 4:
-		return *(unsigned int *)p;
-
-#ifdef __amd64__
-	case 8:
-		return *(unsigned long *)p;
-#endif
-
-	default:
-		printk(KERN_ERR "mmiotrace: Error: width.\n");
-	}
-
-err:
-	return 0;
-}
diff --git a/arch/x86/kernel/mmiotrace/pf_in.h b/arch/x86/kernel/mmiotrace/pf_in.h
deleted file mode 100644
index e05341a..0000000
--- a/arch/x86/kernel/mmiotrace/pf_in.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- *  Fault Injection Test harness (FI)
- *  Copyright (C) Intel Crop.
- *
- *  This program is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU General Public License
- *  as published by the Free Software Foundation; either version 2
- *  of the License, or (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
- *  USA.
- *
- */
-
-#ifndef __PF_H_
-#define __PF_H_
-
-enum reason_type {
-	NOT_ME,	/* page fault is not in regions */
-	NOTHING,	/* access others point in regions */
-	REG_READ,	/* read from addr to reg */
-	REG_WRITE,	/* write from reg to addr */
-	IMM_WRITE,	/* write from imm to addr */
-	OTHERS	/* Other instructions can not intercept */
-};
-
-enum reason_type get_ins_type(unsigned long ins_addr);
-unsigned int get_ins_mem_width(unsigned long ins_addr);
-unsigned long get_ins_reg_val(unsigned long ins_addr, struct pt_regs *regs);
-unsigned long get_ins_imm_val(unsigned long ins_addr);
-
-#endif /* __PF_H_ */
diff --git a/arch/x86/kernel/mmiotrace/testmmiotrace.c b/arch/x86/kernel/mmiotrace/testmmiotrace.c
deleted file mode 100644
index 40e66b0..0000000
--- a/arch/x86/kernel/mmiotrace/testmmiotrace.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Written by Pekka Paalanen, 2008 <pq@iki.fi>
- */
-#include <linux/module.h>
-#include <asm/io.h>
-
-extern void __iomem *ioremap_nocache_trace(unsigned long offset,
-						unsigned long size);
-extern void iounmap_trace(volatile void __iomem *addr);
-
-#define MODULE_NAME "testmmiotrace"
-
-static unsigned long mmio_address;
-module_param(mmio_address, ulong, 0);
-MODULE_PARM_DESC(mmio_address, "Start address of the mapping of 16 kB.");
-
-static void do_write_test(void __iomem *p)
-{
-	unsigned int i;
-	for (i = 0; i < 256; i++)
-		iowrite8(i, p + i);
-	for (i = 1024; i < (5 * 1024); i += 2)
-		iowrite16(i * 12 + 7, p + i);
-	for (i = (5 * 1024); i < (16 * 1024); i += 4)
-		iowrite32(i * 212371 + 13, p + i);
-}
-
-static void do_read_test(void __iomem *p)
-{
-	unsigned int i;
-	volatile unsigned int v;
-	for (i = 0; i < 256; i++)
-		v = ioread8(p + i);
-	for (i = 1024; i < (5 * 1024); i += 2)
-		v = ioread16(p + i);
-	for (i = (5 * 1024); i < (16 * 1024); i += 4)
-		v = ioread32(p + i);
-}
-
-static void do_test(void)
-{
-	void __iomem *p = ioremap_nocache_trace(mmio_address, 0x4000);
-	if (!p) {
-		printk(KERN_ERR MODULE_NAME ": could not ioremap IO memory, "
-							"aborting.\n");
-		return;
-	}
-	do_write_test(p);
-	do_read_test(p);
-	iounmap_trace(p);
-}
-
-static int __init init(void)
-{
-	if (mmio_address == 0) {
-		printk(KERN_ERR MODULE_NAME ": you have to use the module "
-						"argument mmio_address.\n");
-		printk(KERN_ERR MODULE_NAME ": DO NOT LOAD THIS MODULE UNLESS"
-				" YOU REALLY KNOW WHAT YOU ARE DOING!\n");
-		return -ENXIO;
-	}
-
-	printk(KERN_WARNING MODULE_NAME ": WARNING: mapping 16 kB @ 0x%08lx "
-					"in PCI address space, and writing "
-					"rubbish in there.\n", mmio_address);
-	do_test();
-	return 0;
-}
-
-static void __exit cleanup(void)
-{
-	printk(KERN_DEBUG MODULE_NAME ": unloaded.\n");
-}
-
-module_init(init);
-module_exit(cleanup);
-MODULE_LICENSE("GPL");
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index 9832910..6fe5727 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -3,3 +3,8 @@ include ${srctree}/arch/x86/mm/Makefile_32
 else
 include ${srctree}/arch/x86/mm/Makefile_64
 endif
+
+obj-$(CONFIG_MMIOTRACE) += mmiotrace.o
+mmiotrace-objs := pf_in.o kmmio.o mmio-mod.o
+
+obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o
diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
new file mode 100644
index 0000000..28411da
--- /dev/null
+++ b/arch/x86/mm/kmmio.c
@@ -0,0 +1,407 @@
+/* Support for MMIO probes.
+ * Benfit many code from kprobes
+ * (C) 2002 Louis Zhuang <louis.zhuang@intel.com>.
+ *     2007 Alexander Eichner
+ *     2008 Pekka Paalanen <pq@iki.fi>
+ */
+
+#include <linux/version.h>
+#include <linux/spinlock.h>
+#include <linux/hash.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/uaccess.h>
+#include <linux/ptrace.h>
+#include <linux/preempt.h>
+#include <asm/io.h>
+#include <asm/cacheflush.h>
+#include <asm/errno.h>
+#include <asm/tlbflush.h>
+#include <asm/pgtable.h>
+
+#include "kmmio.h"
+
+#define KMMIO_HASH_BITS 6
+#define KMMIO_TABLE_SIZE (1 << KMMIO_HASH_BITS)
+#define KMMIO_PAGE_HASH_BITS 4
+#define KMMIO_PAGE_TABLE_SIZE (1 << KMMIO_PAGE_HASH_BITS)
+
+struct kmmio_context {
+	struct kmmio_fault_page *fpage;
+	struct kmmio_probe *probe;
+	unsigned long saved_flags;
+	int active;
+};
+
+static int kmmio_page_fault(struct pt_regs *regs, unsigned long error_code,
+						unsigned long address);
+static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val,
+								void *args);
+
+static DEFINE_SPINLOCK(kmmio_lock);
+
+/* These are protected by kmmio_lock */
+unsigned int kmmio_count;
+static unsigned int handler_registered;
+static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE];
+static LIST_HEAD(kmmio_probes);
+
+static struct kmmio_context kmmio_ctx[NR_CPUS];
+
+static struct pf_handler kmmio_pf_hook = {
+	.handler = kmmio_page_fault
+};
+
+static struct notifier_block nb_die = {
+	.notifier_call = kmmio_die_notifier
+};
+
+int init_kmmio(void)
+{
+	int i;
+	for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++)
+		INIT_LIST_HEAD(&kmmio_page_table[i]);
+
+	register_die_notifier(&nb_die);
+	return 0;
+}
+
+void cleanup_kmmio(void)
+{
+	/*
+	 * Assume the following have been already cleaned by calling
+	 * unregister_kmmio_probe() appropriately:
+	 * kmmio_page_table, kmmio_probes
+	 */
+	if (handler_registered) {
+		unregister_page_fault_handler(&kmmio_pf_hook);
+		synchronize_rcu();
+	}
+	unregister_die_notifier(&nb_die);
+}
+
+/*
+ * this is basically a dynamic stabbing problem:
+ * Could use the existing prio tree code or
+ * Possible better implementations:
+ * The Interval Skip List: A Data Structure for Finding All Intervals That
+ * Overlap a Point (might be simple)
+ * Space Efficient Dynamic Stabbing with Fast Queries - Mikkel Thorup
+ */
+/* Get the kmmio at this addr (if any). You must be holding kmmio_lock. */
+static struct kmmio_probe *get_kmmio_probe(unsigned long addr)
+{
+	struct kmmio_probe *p;
+	list_for_each_entry(p, &kmmio_probes, list) {
+		if (addr >= p->addr && addr <= (p->addr + p->len))
+			return p;
+	}
+	return NULL;
+}
+
+static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page)
+{
+	struct list_head *head, *tmp;
+
+	page &= PAGE_MASK;
+	head = &kmmio_page_table[hash_long(page, KMMIO_PAGE_HASH_BITS)];
+	list_for_each(tmp, head) {
+		struct kmmio_fault_page *p
+			= list_entry(tmp, struct kmmio_fault_page, list);
+		if (p->page == page)
+			return p;
+	}
+
+	return NULL;
+}
+
+static void arm_kmmio_fault_page(unsigned long page, int *page_level)
+{
+	unsigned long address = page & PAGE_MASK;
+	int level;
+	pte_t *pte = lookup_address(address, &level);
+
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, page);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
+		pmd_t *pmd = (pmd_t *)pte;
+		set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_PRESENT));
+	} else {
+		/* PG_LEVEL_4K */
+		set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
+	}
+
+	if (page_level)
+		*page_level = level;
+
+	__flush_tlb_one(page);
+}
+
+static void disarm_kmmio_fault_page(unsigned long page, int *page_level)
+{
+	unsigned long address = page & PAGE_MASK;
+	int level;
+	pte_t *pte = lookup_address(address, &level);
+
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, page);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
+		pmd_t *pmd = (pmd_t *)pte;
+		set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_PRESENT));
+	} else {
+		/* PG_LEVEL_4K */
+		set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
+	}
+
+	if (page_level)
+		*page_level = level;
+
+	__flush_tlb_one(page);
+}
+
+/*
+ * Interrupts are disabled on entry as trap3 is an interrupt gate
+ * and they remain disabled thorough out this function.
+ */
+static int kmmio_handler(struct pt_regs *regs, unsigned long addr)
+{
+	struct kmmio_context *ctx;
+	int cpu;
+
+	/*
+	 * Preemption is now disabled to prevent process switch during
+	 * single stepping. We can only handle one active kmmio trace
+	 * per cpu, so ensure that we finish it before something else
+	 * gets to run.
+	 *
+	 * XXX what if an interrupt occurs between returning from
+	 * do_page_fault() and entering the single-step exception handler?
+	 * And that interrupt triggers a kmmio trap?
+	 */
+	preempt_disable();
+	cpu = smp_processor_id();
+	ctx = &kmmio_ctx[cpu];
+
+	/* interrupts disabled and CPU-local data => atomicity guaranteed. */
+	if (ctx->active) {
+		/*
+		 * This avoids a deadlock with kmmio_lock.
+		 * If this page fault really was due to kmmio trap,
+		 * all hell breaks loose.
+		 */
+		printk(KERN_EMERG "mmiotrace: recursive probe hit on CPU %d, "
+					"for address %lu. Ignoring.\n",
+					cpu, addr);
+		goto no_kmmio;
+	}
+	ctx->active++;
+
+	/*
+	 * Acquire the kmmio lock to prevent changes affecting
+	 * get_kmmio_fault_page() and get_kmmio_probe(), since we save their
+	 * returned pointers.
+	 * The lock is released in post_kmmio_handler().
+	 * XXX: could/should get_kmmio_*() be using RCU instead of spinlock?
+	 */
+	spin_lock(&kmmio_lock);
+
+	ctx->fpage = get_kmmio_fault_page(addr);
+	if (!ctx->fpage) {
+		/* this page fault is not caused by kmmio */
+		goto no_kmmio_locked;
+	}
+
+	ctx->probe = get_kmmio_probe(addr);
+	ctx->saved_flags = (regs->flags & (TF_MASK|IF_MASK));
+
+	if (ctx->probe && ctx->probe->pre_handler)
+		ctx->probe->pre_handler(ctx->probe, regs, addr);
+
+	regs->flags |= TF_MASK;
+	regs->flags &= ~IF_MASK;
+
+	/* We hold lock, now we set present bit in PTE and single step. */
+	disarm_kmmio_fault_page(ctx->fpage->page, NULL);
+
+	return 1;
+
+no_kmmio_locked:
+	spin_unlock(&kmmio_lock);
+	ctx->active--;
+no_kmmio:
+	preempt_enable_no_resched();
+	/* page fault not handled by kmmio */
+	return 0;
+}
+
+/*
+ * Interrupts are disabled on entry as trap1 is an interrupt gate
+ * and they remain disabled thorough out this function.
+ * And we hold kmmio lock.
+ */
+static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
+{
+	int cpu = smp_processor_id();
+	struct kmmio_context *ctx = &kmmio_ctx[cpu];
+
+	if (!ctx->active)
+		return 0;
+
+	if (ctx->probe && ctx->probe->post_handler)
+		ctx->probe->post_handler(ctx->probe, condition, regs);
+
+	arm_kmmio_fault_page(ctx->fpage->page, NULL);
+
+	regs->flags &= ~TF_MASK;
+	regs->flags |= ctx->saved_flags;
+
+	/* These were acquired in kmmio_handler(). */
+	ctx->active--;
+	spin_unlock(&kmmio_lock);
+	preempt_enable_no_resched();
+
+	/*
+	 * if somebody else is singlestepping across a probe point, flags
+	 * will have TF set, in which case, continue the remaining processing
+	 * of do_debug, as if this is not a probe hit.
+	 */
+	if (regs->flags & TF_MASK)
+		return 0;
+
+	return 1;
+}
+
+static int add_kmmio_fault_page(unsigned long page)
+{
+	struct kmmio_fault_page *f;
+
+	page &= PAGE_MASK;
+	f = get_kmmio_fault_page(page);
+	if (f) {
+		f->count++;
+		return 0;
+	}
+
+	f = kmalloc(sizeof(*f), GFP_ATOMIC);
+	if (!f)
+		return -1;
+
+	f->count = 1;
+	f->page = page;
+	list_add(&f->list,
+		 &kmmio_page_table[hash_long(f->page, KMMIO_PAGE_HASH_BITS)]);
+
+	arm_kmmio_fault_page(f->page, NULL);
+
+	return 0;
+}
+
+static void release_kmmio_fault_page(unsigned long page)
+{
+	struct kmmio_fault_page *f;
+
+	page &= PAGE_MASK;
+	f = get_kmmio_fault_page(page);
+	if (!f)
+		return;
+
+	f->count--;
+	if (!f->count) {
+		disarm_kmmio_fault_page(f->page, NULL);
+		list_del(&f->list);
+	}
+}
+
+int register_kmmio_probe(struct kmmio_probe *p)
+{
+	int ret = 0;
+	unsigned long size = 0;
+
+	spin_lock_irq(&kmmio_lock);
+	kmmio_count++;
+	if (get_kmmio_probe(p->addr)) {
+		ret = -EEXIST;
+		goto out;
+	}
+	list_add(&p->list, &kmmio_probes);
+	/*printk("adding fault pages...\n");*/
+	while (size < p->len) {
+		if (add_kmmio_fault_page(p->addr + size))
+			printk(KERN_ERR "mmio: Unable to set page fault.\n");
+		size += PAGE_SIZE;
+	}
+
+	if (!handler_registered) {
+		register_page_fault_handler(&kmmio_pf_hook);
+		handler_registered++;
+	}
+
+out:
+	spin_unlock_irq(&kmmio_lock);
+	/*
+	 * XXX: What should I do here?
+	 * Here was a call to global_flush_tlb(), but it does not exist
+	 * anymore.
+	 */
+	return ret;
+}
+
+void unregister_kmmio_probe(struct kmmio_probe *p)
+{
+	unsigned long size = 0;
+
+	spin_lock_irq(&kmmio_lock);
+	while (size < p->len) {
+		release_kmmio_fault_page(p->addr + size);
+		size += PAGE_SIZE;
+	}
+	list_del(&p->list);
+	kmmio_count--;
+	spin_unlock_irq(&kmmio_lock);
+}
+
+/*
+ * According to 2.6.20, mainly x86_64 arch:
+ * This is being called from do_page_fault(), via the page fault notifier
+ * chain. The chain is called for both user space faults and kernel space
+ * faults (address >= TASK_SIZE64), except not on faults serviced by
+ * vmalloc_fault().
+ *
+ * We may be in an interrupt or a critical section. Also prefecthing may
+ * trigger a page fault. We may be in the middle of process switch.
+ * The page fault hook functionality has put us inside RCU read lock.
+ *
+ * Local interrupts are disabled, so preemption cannot happen.
+ * Do not enable interrupts, do not sleep, and watch out for other CPUs.
+ */
+static int kmmio_page_fault(struct pt_regs *regs, unsigned long error_code,
+						unsigned long address)
+{
+	if (is_kmmio_active())
+		if (kmmio_handler(regs, address) == 1)
+			return -1;
+	return 0;
+}
+
+static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val,
+								void *args)
+{
+	struct die_args *arg = args;
+
+	if (val == DIE_DEBUG)
+		if (post_kmmio_handler(arg->err, arg->regs) == 1)
+			return NOTIFY_STOP;
+
+	return NOTIFY_DONE;
+}
diff --git a/arch/x86/mm/kmmio.h b/arch/x86/mm/kmmio.h
new file mode 100644
index 0000000..85b7f68
--- /dev/null
+++ b/arch/x86/mm/kmmio.h
@@ -0,0 +1,58 @@
+#ifndef _LINUX_KMMIO_H
+#define _LINUX_KMMIO_H
+
+#include <linux/list.h>
+#include <linux/notifier.h>
+#include <linux/smp.h>
+#include <linux/types.h>
+#include <linux/ptrace.h>
+#include <linux/version.h>
+#include <linux/kdebug.h>
+
+struct kmmio_probe;
+struct kmmio_fault_page;
+struct pt_regs;
+
+typedef void (*kmmio_pre_handler_t)(struct kmmio_probe *,
+				struct pt_regs *, unsigned long addr);
+typedef void (*kmmio_post_handler_t)(struct kmmio_probe *,
+				unsigned long condition, struct pt_regs *);
+
+struct kmmio_probe {
+	struct list_head list;
+
+	/* start location of the probe point */
+	unsigned long addr;
+
+	/* length of the probe region */
+	unsigned long len;
+
+	 /* Called before addr is executed. */
+	kmmio_pre_handler_t pre_handler;
+
+	/* Called after addr is executed, unless... */
+	kmmio_post_handler_t post_handler;
+};
+
+struct kmmio_fault_page {
+	struct list_head list;
+
+	/* location of the fault page */
+	unsigned long page;
+
+	int count;
+};
+
+/* kmmio is active by some kmmio_probes? */
+static inline int is_kmmio_active(void)
+{
+	extern unsigned int kmmio_count;
+	return kmmio_count;
+}
+
+int init_kmmio(void);
+void cleanup_kmmio(void);
+int register_kmmio_probe(struct kmmio_probe *p);
+void unregister_kmmio_probe(struct kmmio_probe *p);
+
+#endif /* _LINUX_KMMIO_H */
diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c
new file mode 100644
index 0000000..f492b65
--- /dev/null
+++ b/arch/x86/mm/mmio-mod.c
@@ -0,0 +1,538 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) IBM Corporation, 2005
+ *               Jeff Muizelaar, 2006, 2007
+ *               Pekka Paalanen, 2008 <pq@iki.fi>
+ *
+ * Derived from the read-mod example from relay-examples by Tom Zanussi.
+ */
+#include <linux/module.h>
+#include <linux/relay.h>
+#include <linux/debugfs.h>
+#include <linux/proc_fs.h>
+#include <asm/io.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <asm/pgtable.h>
+#include <linux/mmiotrace.h>
+#include <asm/e820.h> /* for ISA_START_ADDRESS */
+#include <asm/atomic.h>
+
+#include "kmmio.h"
+#include "pf_in.h"
+
+/* This app's relay channel files will appear in /debug/mmio-trace */
+#define APP_DIR		"mmio-trace"
+/* the marker injection file in /proc */
+#define MARKER_FILE     "mmio-marker"
+
+#define MODULE_NAME     "mmiotrace"
+
+struct trap_reason {
+	unsigned long addr;
+	unsigned long ip;
+	enum reason_type type;
+	int active_traces;
+};
+
+/* Accessed per-cpu. */
+static struct trap_reason pf_reason[NR_CPUS];
+static struct mm_io_header_rw cpu_trace[NR_CPUS];
+
+/* Access to this is not per-cpu. */
+static atomic_t dropped[NR_CPUS];
+
+static struct file_operations mmio_fops = {
+	.owner = THIS_MODULE,
+};
+
+static const size_t subbuf_size = 256*1024;
+static struct rchan *chan;
+static struct dentry *dir;
+static struct proc_dir_entry *proc_marker_file;
+
+/* module parameters */
+static unsigned int      n_subbufs = 32*4;
+static unsigned long filter_offset;
+static int                 notrace;
+static int               ISA_trace;
+static int                trace_pc;
+
+module_param(n_subbufs, uint, 0);
+module_param(filter_offset, ulong, 0);
+module_param(notrace, bool, 0);
+module_param(ISA_trace, bool, 0);
+module_param(trace_pc, bool, 0);
+
+MODULE_PARM_DESC(n_subbufs, "Number of 256kB buffers, default 128.");
+MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
+MODULE_PARM_DESC(notrace, "Disable actual MMIO tracing.");
+MODULE_PARM_DESC(ISA_trace, "Do not exclude the low ISA range.");
+MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
+
+static void record_timestamp(struct mm_io_header *header)
+{
+	struct timespec now;
+
+	getnstimeofday(&now);
+	header->sec = now.tv_sec;
+	header->nsec = now.tv_nsec;
+}
+
+/*
+ * Write callback for the /proc entry:
+ * Read a marker and write it to the mmio trace log
+ */
+static int write_marker(struct file *file, const char __user *buffer,
+					unsigned long count, void *data)
+{
+	char *event = NULL;
+	struct mm_io_header *headp;
+	int len = (count > 65535) ? 65535 : count;
+
+	event = kzalloc(sizeof(*headp) + len, GFP_KERNEL);
+	if (!event)
+		return -ENOMEM;
+
+	headp = (struct mm_io_header *)event;
+	headp->type = MMIO_MAGIC | (MMIO_MARKER << MMIO_OPCODE_SHIFT);
+	headp->data_len = len;
+	record_timestamp(headp);
+
+	if (copy_from_user(event + sizeof(*headp), buffer, len)) {
+		kfree(event);
+		return -EFAULT;
+	}
+
+	relay_write(chan, event, sizeof(*headp) + len);
+	kfree(event);
+	return len;
+}
+
+static void print_pte(unsigned long address)
+{
+	int level;
+	pte_t *pte = lookup_address(address, &level);
+
+	if (!pte) {
+		printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
+						__FUNCTION__, address);
+		return;
+	}
+
+	if (level == PG_LEVEL_2M) {
+		printk(KERN_EMERG MODULE_NAME ": 4MB pages are not "
+						"currently supported: %lx\n",
+						address);
+		BUG();
+	}
+	printk(KERN_DEBUG MODULE_NAME ": pte for 0x%lx: 0x%lx 0x%lx\n",
+					address, pte_val(*pte),
+					pte_val(*pte) & _PAGE_PRESENT);
+}
+
+/*
+ * For some reason the pre/post pairs have been called in an
+ * unmatched order. Report and die.
+ */
+static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
+{
+	const unsigned long cpu = smp_processor_id();
+	printk(KERN_EMERG MODULE_NAME ": unexpected fault for address: %lx, "
+					"last fault for address: %lx\n",
+					addr, pf_reason[cpu].addr);
+	print_pte(addr);
+#ifdef __i386__
+	print_symbol(KERN_EMERG "faulting EIP is at %s\n", regs->ip);
+	print_symbol(KERN_EMERG "last faulting EIP was at %s\n",
+							pf_reason[cpu].ip);
+	printk(KERN_EMERG
+			"eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
+			regs->ax, regs->bx, regs->cx, regs->dx);
+	printk(KERN_EMERG
+			"esi: %08lx   edi: %08lx   ebp: %08lx   esp: %08lx\n",
+			regs->si, regs->di, regs->bp, regs->sp);
+#else
+	print_symbol(KERN_EMERG "faulting RIP is at %s\n", regs->ip);
+	print_symbol(KERN_EMERG "last faulting RIP was at %s\n",
+							pf_reason[cpu].ip);
+	printk(KERN_EMERG "rax: %016lx   rcx: %016lx   rdx: %016lx\n",
+					regs->ax, regs->cx, regs->dx);
+	printk(KERN_EMERG "rsi: %016lx   rdi: %016lx   "
+				"rbp: %016lx   rsp: %016lx\n",
+				regs->si, regs->di, regs->bp, regs->sp);
+#endif
+	BUG();
+}
+
+static void pre(struct kmmio_probe *p, struct pt_regs *regs,
+						unsigned long addr)
+{
+	const unsigned long cpu = smp_processor_id();
+	const unsigned long instptr = instruction_pointer(regs);
+	const enum reason_type type = get_ins_type(instptr);
+
+	/* it doesn't make sense to have more than one active trace per cpu */
+	if (pf_reason[cpu].active_traces)
+		die_kmmio_nesting_error(regs, addr);
+	else
+		pf_reason[cpu].active_traces++;
+
+	pf_reason[cpu].type = type;
+	pf_reason[cpu].addr = addr;
+	pf_reason[cpu].ip = instptr;
+
+	cpu_trace[cpu].header.type = MMIO_MAGIC;
+	cpu_trace[cpu].header.pid = 0;
+	cpu_trace[cpu].header.data_len = sizeof(struct mm_io_rw);
+	cpu_trace[cpu].rw.address = addr;
+
+	/*
+	 * Only record the program counter when requested.
+	 * It may taint clean-room reverse engineering.
+	 */
+	if (trace_pc)
+		cpu_trace[cpu].rw.pc = instptr;
+	else
+		cpu_trace[cpu].rw.pc = 0;
+
+	record_timestamp(&cpu_trace[cpu].header);
+
+	switch (type) {
+	case REG_READ:
+		cpu_trace[cpu].header.type |=
+			(MMIO_READ << MMIO_OPCODE_SHIFT) |
+			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
+		break;
+	case REG_WRITE:
+		cpu_trace[cpu].header.type |=
+			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
+			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
+		cpu_trace[cpu].rw.value = get_ins_reg_val(instptr, regs);
+		break;
+	case IMM_WRITE:
+		cpu_trace[cpu].header.type |=
+			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
+			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
+		cpu_trace[cpu].rw.value = get_ins_imm_val(instptr);
+		break;
+	default:
+		{
+			unsigned char *ip = (unsigned char *)instptr;
+			cpu_trace[cpu].header.type |=
+					(MMIO_UNKNOWN_OP << MMIO_OPCODE_SHIFT);
+			cpu_trace[cpu].rw.value = (*ip) << 16 |
+							*(ip + 1) << 8 |
+							*(ip + 2);
+		}
+	}
+}
+
+static void post(struct kmmio_probe *p, unsigned long condition,
+							struct pt_regs *regs)
+{
+	const unsigned long cpu = smp_processor_id();
+
+	/* this should always return the active_trace count to 0 */
+	pf_reason[cpu].active_traces--;
+	if (pf_reason[cpu].active_traces) {
+		printk(KERN_EMERG MODULE_NAME ": unexpected post handler");
+		BUG();
+	}
+
+	switch (pf_reason[cpu].type) {
+	case REG_READ:
+		cpu_trace[cpu].rw.value = get_ins_reg_val(pf_reason[cpu].ip,
+									regs);
+		break;
+	default:
+		break;
+	}
+	relay_write(chan, &cpu_trace[cpu], sizeof(struct mm_io_header_rw));
+}
+
+/*
+ * subbuf_start() relay callback.
+ *
+ * Defined so that we know when events are dropped due to the buffer-full
+ * condition.
+ */
+static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf,
+					void *prev_subbuf, size_t prev_padding)
+{
+	unsigned int cpu = buf->cpu;
+	atomic_t *drop = &dropped[cpu];
+	int count;
+	if (relay_buf_full(buf)) {
+		if (atomic_inc_return(drop) == 1) {
+			printk(KERN_ERR MODULE_NAME ": cpu %d buffer full!\n",
+									cpu);
+		}
+		return 0;
+	} else if ((count = atomic_read(drop))) {
+		printk(KERN_ERR MODULE_NAME
+					": cpu %d buffer no longer full, "
+					"missed %d events.\n",
+					cpu, count);
+		atomic_sub(count, drop);
+	}
+
+	return 1;
+}
+
+/* file_create() callback.  Creates relay file in debugfs. */
+static struct dentry *create_buf_file_handler(const char *filename,
+						struct dentry *parent,
+						int mode,
+						struct rchan_buf *buf,
+						int *is_global)
+{
+	struct dentry *buf_file;
+
+	mmio_fops.read = relay_file_operations.read;
+	mmio_fops.open = relay_file_operations.open;
+	mmio_fops.poll = relay_file_operations.poll;
+	mmio_fops.mmap = relay_file_operations.mmap;
+	mmio_fops.release = relay_file_operations.release;
+	mmio_fops.splice_read = relay_file_operations.splice_read;
+
+	buf_file = debugfs_create_file(filename, mode, parent, buf,
+								&mmio_fops);
+
+	return buf_file;
+}
+
+/* file_remove() default callback.  Removes relay file in debugfs. */
+static int remove_buf_file_handler(struct dentry *dentry)
+{
+	debugfs_remove(dentry);
+	return 0;
+}
+
+static struct rchan_callbacks relay_callbacks = {
+	.subbuf_start = subbuf_start_handler,
+	.create_buf_file = create_buf_file_handler,
+	.remove_buf_file = remove_buf_file_handler,
+};
+
+/*
+ * create_channel - creates channel /debug/APP_DIR/cpuXXX
+ * Returns channel on success, NULL otherwise
+ */
+static struct rchan *create_channel(unsigned size, unsigned n)
+{
+	return relay_open("cpu", dir, size, n, &relay_callbacks, NULL);
+}
+
+/* destroy_channel - destroys channel /debug/APP_DIR/cpuXXX */
+static void destroy_channel(void)
+{
+	if (chan) {
+		relay_close(chan);
+		chan = NULL;
+	}
+}
+
+struct remap_trace {
+	struct list_head list;
+	struct kmmio_probe probe;
+};
+static LIST_HEAD(trace_list);
+static DEFINE_SPINLOCK(trace_list_lock);
+
+static void do_ioremap_trace_core(unsigned long offset, unsigned long size,
+							void __iomem *addr)
+{
+	struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
+	struct mm_io_header_map event = {
+		.header = {
+			.type = MMIO_MAGIC |
+					(MMIO_PROBE << MMIO_OPCODE_SHIFT),
+			.sec = 0,
+			.nsec = 0,
+			.pid = 0,
+			.data_len = sizeof(struct mm_io_map)
+		},
+		.map = {
+			.phys = offset,
+			.addr = (unsigned long)addr,
+			.len  = size,
+			.pc   = 0
+		}
+	};
+	record_timestamp(&event.header);
+
+	*trace = (struct remap_trace) {
+		.probe = {
+			.addr = (unsigned long)addr,
+			.len = size,
+			.pre_handler = pre,
+			.post_handler = post,
+		}
+	};
+
+	relay_write(chan, &event, sizeof(event));
+	spin_lock(&trace_list_lock);
+	list_add_tail(&trace->list, &trace_list);
+	spin_unlock(&trace_list_lock);
+	if (!notrace)
+		register_kmmio_probe(&trace->probe);
+}
+
+static void ioremap_trace_core(unsigned long offset, unsigned long size,
+							void __iomem *addr)
+{
+	if ((filter_offset) && (offset != filter_offset))
+		return;
+
+	/* Don't trace the low PCI/ISA area, it's always mapped.. */
+	if (!ISA_trace && (offset < ISA_END_ADDRESS) &&
+					(offset + size > ISA_START_ADDRESS)) {
+		printk(KERN_NOTICE MODULE_NAME ": Ignoring map of low "
+						"PCI/ISA area (0x%lx-0x%lx)\n",
+						offset, offset + size);
+		return;
+	}
+	do_ioremap_trace_core(offset, size, addr);
+}
+
+void __iomem *ioremap_cache_trace(unsigned long offset, unsigned long size)
+{
+	void __iomem *p = ioremap_cache(offset, size);
+	printk(KERN_DEBUG MODULE_NAME ": ioremap_cache(0x%lx, 0x%lx) = %p\n",
+							offset, size, p);
+	ioremap_trace_core(offset, size, p);
+	return p;
+}
+EXPORT_SYMBOL(ioremap_cache_trace);
+
+void __iomem *ioremap_nocache_trace(unsigned long offset, unsigned long size)
+{
+	void __iomem *p = ioremap_nocache(offset, size);
+	printk(KERN_DEBUG MODULE_NAME ": ioremap_nocache(0x%lx, 0x%lx) = %p\n",
+							offset, size, p);
+	ioremap_trace_core(offset, size, p);
+	return p;
+}
+EXPORT_SYMBOL(ioremap_nocache_trace);
+
+void iounmap_trace(volatile void __iomem *addr)
+{
+	struct mm_io_header_map event = {
+		.header = {
+			.type = MMIO_MAGIC |
+				(MMIO_UNPROBE << MMIO_OPCODE_SHIFT),
+			.sec = 0,
+			.nsec = 0,
+			.pid = 0,
+			.data_len = sizeof(struct mm_io_map)
+		},
+		.map = {
+			.phys = 0,
+			.addr = (unsigned long)addr,
+			.len  = 0,
+			.pc   = 0
+		}
+	};
+	struct remap_trace *trace;
+	struct remap_trace *tmp;
+	printk(KERN_DEBUG MODULE_NAME ": Unmapping %p.\n", addr);
+	record_timestamp(&event.header);
+
+	spin_lock(&trace_list_lock);
+	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
+		if ((unsigned long)addr == trace->probe.addr) {
+			if (!notrace)
+				unregister_kmmio_probe(&trace->probe);
+			list_del(&trace->list);
+			kfree(trace);
+			break;
+		}
+	}
+	spin_unlock(&trace_list_lock);
+	relay_write(chan, &event, sizeof(event));
+	iounmap(addr);
+}
+EXPORT_SYMBOL(iounmap_trace);
+
+static void clear_trace_list(void)
+{
+	struct remap_trace *trace;
+	struct remap_trace *tmp;
+
+	spin_lock(&trace_list_lock);
+	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
+		printk(KERN_WARNING MODULE_NAME ": purging non-iounmapped "
+					"trace @0x%08lx, size 0x%lx.\n",
+					trace->probe.addr, trace->probe.len);
+		if (!notrace)
+			unregister_kmmio_probe(&trace->probe);
+		list_del(&trace->list);
+		kfree(trace);
+		break;
+	}
+	spin_unlock(&trace_list_lock);
+}
+
+static int __init init(void)
+{
+	if (n_subbufs < 2)
+		return -EINVAL;
+
+	dir = debugfs_create_dir(APP_DIR, NULL);
+	if (!dir) {
+		printk(KERN_ERR MODULE_NAME
+				": Couldn't create relay app directory.\n");
+		return -ENOMEM;
+	}
+
+	chan = create_channel(subbuf_size, n_subbufs);
+	if (!chan) {
+		debugfs_remove(dir);
+		printk(KERN_ERR MODULE_NAME
+				": relay app channel creation failed\n");
+		return -ENOMEM;
+	}
+
+	init_kmmio();
+
+	proc_marker_file = create_proc_entry(MARKER_FILE, 0, NULL);
+	if (proc_marker_file)
+		proc_marker_file->write_proc = write_marker;
+
+	printk(KERN_DEBUG MODULE_NAME ": loaded.\n");
+	if (notrace)
+		printk(KERN_DEBUG MODULE_NAME ": MMIO tracing disabled.\n");
+	if (ISA_trace)
+		printk(KERN_WARNING MODULE_NAME
+				": Warning! low ISA range will be traced.\n");
+	return 0;
+}
+
+static void __exit cleanup(void)
+{
+	printk(KERN_DEBUG MODULE_NAME ": unload...\n");
+	clear_trace_list();
+	cleanup_kmmio();
+	remove_proc_entry(MARKER_FILE, NULL);
+	destroy_channel();
+	if (dir)
+		debugfs_remove(dir);
+}
+
+module_init(init);
+module_exit(cleanup);
+MODULE_LICENSE("GPL");
diff --git a/arch/x86/mm/pf_in.c b/arch/x86/mm/pf_in.c
new file mode 100644
index 0000000..67ea520
--- /dev/null
+++ b/arch/x86/mm/pf_in.c
@@ -0,0 +1,489 @@
+/*
+ *  Fault Injection Test harness (FI)
+ *  Copyright (C) Intel Crop.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ *
+ */
+
+/*  $Id: pf_in.c,v 1.1.1.1 2002/11/12 05:56:32 brlock Exp $
+ *  Copyright by Intel Crop., 2002
+ *  Louis Zhuang (louis.zhuang@intel.com)
+ *
+ *  Bjorn Steinbrink (B.Steinbrink@gmx.de), 2007
+ */
+
+#include <linux/module.h>
+#include <linux/ptrace.h> /* struct pt_regs */
+#include "pf_in.h"
+
+#ifdef __i386__
+/* IA32 Manual 3, 2-1 */
+static unsigned char prefix_codes[] = {
+	0xF0, 0xF2, 0xF3, 0x2E, 0x36, 0x3E, 0x26, 0x64,
+	0x65, 0x2E, 0x3E, 0x66, 0x67
+};
+/* IA32 Manual 3, 3-432*/
+static unsigned int reg_rop[] = {
+	0x8A, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
+};
+static unsigned int reg_wop[] = { 0x88, 0x89 };
+static unsigned int imm_wop[] = { 0xC6, 0xC7 };
+/* IA32 Manual 3, 3-432*/
+static unsigned int rw8[] = { 0x88, 0x8A, 0xC6 };
+static unsigned int rw32[] = {
+	0x89, 0x8B, 0xC7, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
+};
+static unsigned int mw8[] = { 0x88, 0x8A, 0xC6, 0xB60F, 0xBE0F };
+static unsigned int mw16[] = { 0xB70F, 0xBF0F };
+static unsigned int mw32[] = { 0x89, 0x8B, 0xC7 };
+static unsigned int mw64[] = {};
+#else /* not __i386__ */
+static unsigned char prefix_codes[] = {
+	0x66, 0x67, 0x2E, 0x3E, 0x26, 0x64, 0x65, 0x36,
+	0xF0, 0xF3, 0xF2,
+	/* REX Prefixes */
+	0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+	0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f
+};
+/* AMD64 Manual 3, Appendix A*/
+static unsigned int reg_rop[] = {
+	0x8A, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
+};
+static unsigned int reg_wop[] = { 0x88, 0x89 };
+static unsigned int imm_wop[] = { 0xC6, 0xC7 };
+static unsigned int rw8[] = { 0xC6, 0x88, 0x8A };
+static unsigned int rw32[] = {
+	0xC7, 0x89, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
+};
+/* 8 bit only */
+static unsigned int mw8[] = { 0xC6, 0x88, 0x8A, 0xB60F, 0xBE0F };
+/* 16 bit only */
+static unsigned int mw16[] = { 0xB70F, 0xBF0F };
+/* 16 or 32 bit */
+static unsigned int mw32[] = { 0xC7 };
+/* 16, 32 or 64 bit */
+static unsigned int mw64[] = { 0x89, 0x8B };
+#endif /* not __i386__ */
+
+static int skip_prefix(unsigned char *addr, int *shorted, int *enlarged,
+								int *rexr)
+{
+	int i;
+	unsigned char *p = addr;
+	*shorted = 0;
+	*enlarged = 0;
+	*rexr = 0;
+
+restart:
+	for (i = 0; i < ARRAY_SIZE(prefix_codes); i++) {
+		if (*p == prefix_codes[i]) {
+			if (*p == 0x66)
+				*shorted = 1;
+#ifdef __amd64__
+			if ((*p & 0xf8) == 0x48)
+				*enlarged = 1;
+			if ((*p & 0xf4) == 0x44)
+				*rexr = 1;
+#endif
+			p++;
+			goto restart;
+		}
+	}
+
+	return (p - addr);
+}
+
+static int get_opcode(unsigned char *addr, unsigned int *opcode)
+{
+	int len;
+
+	if (*addr == 0x0F) {
+		/* 0x0F is extension instruction */
+		*opcode = *(unsigned short *)addr;
+		len = 2;
+	} else {
+		*opcode = *addr;
+		len = 1;
+	}
+
+	return len;
+}
+
+#define CHECK_OP_TYPE(opcode, array, type) \
+	for (i = 0; i < ARRAY_SIZE(array); i++) { \
+		if (array[i] == opcode) { \
+			rv = type; \
+			goto exit; \
+		} \
+	}
+
+enum reason_type get_ins_type(unsigned long ins_addr)
+{
+	unsigned int opcode;
+	unsigned char *p;
+	int shorted, enlarged, rexr;
+	int i;
+	enum reason_type rv = OTHERS;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+
+	CHECK_OP_TYPE(opcode, reg_rop, REG_READ);
+	CHECK_OP_TYPE(opcode, reg_wop, REG_WRITE);
+	CHECK_OP_TYPE(opcode, imm_wop, IMM_WRITE);
+
+exit:
+	return rv;
+}
+#undef CHECK_OP_TYPE
+
+static unsigned int get_ins_reg_width(unsigned long ins_addr)
+{
+	unsigned int opcode;
+	unsigned char *p;
+	int i, shorted, enlarged, rexr;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+
+	for (i = 0; i < ARRAY_SIZE(rw8); i++)
+		if (rw8[i] == opcode)
+			return 1;
+
+	for (i = 0; i < ARRAY_SIZE(rw32); i++)
+		if (rw32[i] == opcode)
+			return (shorted ? 2 : (enlarged ? 8 : 4));
+
+	printk(KERN_ERR "mmiotrace: Unknown opcode 0x%02x\n", opcode);
+	return 0;
+}
+
+unsigned int get_ins_mem_width(unsigned long ins_addr)
+{
+	unsigned int opcode;
+	unsigned char *p;
+	int i, shorted, enlarged, rexr;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+
+	for (i = 0; i < ARRAY_SIZE(mw8); i++)
+		if (mw8[i] == opcode)
+			return 1;
+
+	for (i = 0; i < ARRAY_SIZE(mw16); i++)
+		if (mw16[i] == opcode)
+			return 2;
+
+	for (i = 0; i < ARRAY_SIZE(mw32); i++)
+		if (mw32[i] == opcode)
+			return shorted ? 2 : 4;
+
+	for (i = 0; i < ARRAY_SIZE(mw64); i++)
+		if (mw64[i] == opcode)
+			return shorted ? 2 : (enlarged ? 8 : 4);
+
+	printk(KERN_ERR "mmiotrace: Unknown opcode 0x%02x\n", opcode);
+	return 0;
+}
+
+/*
+ * Define register ident in mod/rm byte.
+ * Note: these are NOT the same as in ptrace-abi.h.
+ */
+enum {
+	arg_AL = 0,
+	arg_CL = 1,
+	arg_DL = 2,
+	arg_BL = 3,
+	arg_AH = 4,
+	arg_CH = 5,
+	arg_DH = 6,
+	arg_BH = 7,
+
+	arg_AX = 0,
+	arg_CX = 1,
+	arg_DX = 2,
+	arg_BX = 3,
+	arg_SP = 4,
+	arg_BP = 5,
+	arg_SI = 6,
+	arg_DI = 7,
+#ifdef __amd64__
+	arg_R8  = 8,
+	arg_R9  = 9,
+	arg_R10 = 10,
+	arg_R11 = 11,
+	arg_R12 = 12,
+	arg_R13 = 13,
+	arg_R14 = 14,
+	arg_R15 = 15
+#endif
+};
+
+static unsigned char *get_reg_w8(int no, struct pt_regs *regs)
+{
+	unsigned char *rv = NULL;
+
+	switch (no) {
+	case arg_AL:
+		rv = (unsigned char *)&regs->ax;
+		break;
+	case arg_BL:
+		rv = (unsigned char *)&regs->bx;
+		break;
+	case arg_CL:
+		rv = (unsigned char *)&regs->cx;
+		break;
+	case arg_DL:
+		rv = (unsigned char *)&regs->dx;
+		break;
+	case arg_AH:
+		rv = 1 + (unsigned char *)&regs->ax;
+		break;
+	case arg_BH:
+		rv = 1 + (unsigned char *)&regs->bx;
+		break;
+	case arg_CH:
+		rv = 1 + (unsigned char *)&regs->cx;
+		break;
+	case arg_DH:
+		rv = 1 + (unsigned char *)&regs->dx;
+		break;
+#ifdef __amd64__
+	case arg_R8:
+		rv = (unsigned char *)&regs->r8;
+		break;
+	case arg_R9:
+		rv = (unsigned char *)&regs->r9;
+		break;
+	case arg_R10:
+		rv = (unsigned char *)&regs->r10;
+		break;
+	case arg_R11:
+		rv = (unsigned char *)&regs->r11;
+		break;
+	case arg_R12:
+		rv = (unsigned char *)&regs->r12;
+		break;
+	case arg_R13:
+		rv = (unsigned char *)&regs->r13;
+		break;
+	case arg_R14:
+		rv = (unsigned char *)&regs->r14;
+		break;
+	case arg_R15:
+		rv = (unsigned char *)&regs->r15;
+		break;
+#endif
+	default:
+		printk(KERN_ERR "mmiotrace: Error reg no# %d\n", no);
+		break;
+	}
+	return rv;
+}
+
+static unsigned long *get_reg_w32(int no, struct pt_regs *regs)
+{
+	unsigned long *rv = NULL;
+
+	switch (no) {
+	case arg_AX:
+		rv = &regs->ax;
+		break;
+	case arg_BX:
+		rv = &regs->bx;
+		break;
+	case arg_CX:
+		rv = &regs->cx;
+		break;
+	case arg_DX:
+		rv = &regs->dx;
+		break;
+	case arg_SP:
+		rv = &regs->sp;
+		break;
+	case arg_BP:
+		rv = &regs->bp;
+		break;
+	case arg_SI:
+		rv = &regs->si;
+		break;
+	case arg_DI:
+		rv = &regs->di;
+		break;
+#ifdef __amd64__
+	case arg_R8:
+		rv = &regs->r8;
+		break;
+	case arg_R9:
+		rv = &regs->r9;
+		break;
+	case arg_R10:
+		rv = &regs->r10;
+		break;
+	case arg_R11:
+		rv = &regs->r11;
+		break;
+	case arg_R12:
+		rv = &regs->r12;
+		break;
+	case arg_R13:
+		rv = &regs->r13;
+		break;
+	case arg_R14:
+		rv = &regs->r14;
+		break;
+	case arg_R15:
+		rv = &regs->r15;
+		break;
+#endif
+	default:
+		printk(KERN_ERR "mmiotrace: Error reg no# %d\n", no);
+	}
+
+	return rv;
+}
+
+unsigned long get_ins_reg_val(unsigned long ins_addr, struct pt_regs *regs)
+{
+	unsigned int opcode;
+	unsigned char mod_rm;
+	int reg;
+	unsigned char *p;
+	int i, shorted, enlarged, rexr;
+	unsigned long rv;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+	for (i = 0; i < ARRAY_SIZE(reg_rop); i++)
+		if (reg_rop[i] == opcode) {
+			rv = REG_READ;
+			goto do_work;
+		}
+
+	for (i = 0; i < ARRAY_SIZE(reg_wop); i++)
+		if (reg_wop[i] == opcode) {
+			rv = REG_WRITE;
+			goto do_work;
+		}
+
+	printk(KERN_ERR "mmiotrace: Not a register instruction, opcode "
+							"0x%02x\n", opcode);
+	goto err;
+
+do_work:
+	mod_rm = *p;
+	reg = ((mod_rm >> 3) & 0x7) | (rexr << 3);
+	switch (get_ins_reg_width(ins_addr)) {
+	case 1:
+		return *get_reg_w8(reg, regs);
+
+	case 2:
+		return *(unsigned short *)get_reg_w32(reg, regs);
+
+	case 4:
+		return *(unsigned int *)get_reg_w32(reg, regs);
+
+#ifdef __amd64__
+	case 8:
+		return *(unsigned long *)get_reg_w32(reg, regs);
+#endif
+
+	default:
+		printk(KERN_ERR "mmiotrace: Error width# %d\n", reg);
+	}
+
+err:
+	return 0;
+}
+
+unsigned long get_ins_imm_val(unsigned long ins_addr)
+{
+	unsigned int opcode;
+	unsigned char mod_rm;
+	unsigned char mod;
+	unsigned char *p;
+	int i, shorted, enlarged, rexr;
+	unsigned long rv;
+
+	p = (unsigned char *)ins_addr;
+	p += skip_prefix(p, &shorted, &enlarged, &rexr);
+	p += get_opcode(p, &opcode);
+	for (i = 0; i < ARRAY_SIZE(imm_wop); i++)
+		if (imm_wop[i] == opcode) {
+			rv = IMM_WRITE;
+			goto do_work;
+		}
+
+	printk(KERN_ERR "mmiotrace: Not an immediate instruction, opcode "
+							"0x%02x\n", opcode);
+	goto err;
+
+do_work:
+	mod_rm = *p;
+	mod = mod_rm >> 6;
+	p++;
+	switch (mod) {
+	case 0:
+		/* if r/m is 5 we have a 32 disp (IA32 Manual 3, Table 2-2)  */
+		/* AMD64: XXX Check for address size prefix? */
+		if ((mod_rm & 0x7) == 0x5)
+			p += 4;
+		break;
+
+	case 1:
+		p += 1;
+		break;
+
+	case 2:
+		p += 4;
+		break;
+
+	case 3:
+	default:
+		printk(KERN_ERR "mmiotrace: not a memory access instruction "
+						"at 0x%lx, rm_mod=0x%02x\n",
+						ins_addr, mod_rm);
+	}
+
+	switch (get_ins_reg_width(ins_addr)) {
+	case 1:
+		return *(unsigned char *)p;
+
+	case 2:
+		return *(unsigned short *)p;
+
+	case 4:
+		return *(unsigned int *)p;
+
+#ifdef __amd64__
+	case 8:
+		return *(unsigned long *)p;
+#endif
+
+	default:
+		printk(KERN_ERR "mmiotrace: Error: width.\n");
+	}
+
+err:
+	return 0;
+}
diff --git a/arch/x86/mm/pf_in.h b/arch/x86/mm/pf_in.h
new file mode 100644
index 0000000..e05341a
--- /dev/null
+++ b/arch/x86/mm/pf_in.h
@@ -0,0 +1,39 @@
+/*
+ *  Fault Injection Test harness (FI)
+ *  Copyright (C) Intel Crop.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ *
+ */
+
+#ifndef __PF_H_
+#define __PF_H_
+
+enum reason_type {
+	NOT_ME,	/* page fault is not in regions */
+	NOTHING,	/* access others point in regions */
+	REG_READ,	/* read from addr to reg */
+	REG_WRITE,	/* write from reg to addr */
+	IMM_WRITE,	/* write from imm to addr */
+	OTHERS	/* Other instructions can not intercept */
+};
+
+enum reason_type get_ins_type(unsigned long ins_addr);
+unsigned int get_ins_mem_width(unsigned long ins_addr);
+unsigned long get_ins_reg_val(unsigned long ins_addr, struct pt_regs *regs);
+unsigned long get_ins_imm_val(unsigned long ins_addr);
+
+#endif /* __PF_H_ */
diff --git a/arch/x86/mm/testmmiotrace.c b/arch/x86/mm/testmmiotrace.c
new file mode 100644
index 0000000..40e66b0
--- /dev/null
+++ b/arch/x86/mm/testmmiotrace.c
@@ -0,0 +1,77 @@
+/*
+ * Written by Pekka Paalanen, 2008 <pq@iki.fi>
+ */
+#include <linux/module.h>
+#include <asm/io.h>
+
+extern void __iomem *ioremap_nocache_trace(unsigned long offset,
+						unsigned long size);
+extern void iounmap_trace(volatile void __iomem *addr);
+
+#define MODULE_NAME "testmmiotrace"
+
+static unsigned long mmio_address;
+module_param(mmio_address, ulong, 0);
+MODULE_PARM_DESC(mmio_address, "Start address of the mapping of 16 kB.");
+
+static void do_write_test(void __iomem *p)
+{
+	unsigned int i;
+	for (i = 0; i < 256; i++)
+		iowrite8(i, p + i);
+	for (i = 1024; i < (5 * 1024); i += 2)
+		iowrite16(i * 12 + 7, p + i);
+	for (i = (5 * 1024); i < (16 * 1024); i += 4)
+		iowrite32(i * 212371 + 13, p + i);
+}
+
+static void do_read_test(void __iomem *p)
+{
+	unsigned int i;
+	volatile unsigned int v;
+	for (i = 0; i < 256; i++)
+		v = ioread8(p + i);
+	for (i = 1024; i < (5 * 1024); i += 2)
+		v = ioread16(p + i);
+	for (i = (5 * 1024); i < (16 * 1024); i += 4)
+		v = ioread32(p + i);
+}
+
+static void do_test(void)
+{
+	void __iomem *p = ioremap_nocache_trace(mmio_address, 0x4000);
+	if (!p) {
+		printk(KERN_ERR MODULE_NAME ": could not ioremap IO memory, "
+							"aborting.\n");
+		return;
+	}
+	do_write_test(p);
+	do_read_test(p);
+	iounmap_trace(p);
+}
+
+static int __init init(void)
+{
+	if (mmio_address == 0) {
+		printk(KERN_ERR MODULE_NAME ": you have to use the module "
+						"argument mmio_address.\n");
+		printk(KERN_ERR MODULE_NAME ": DO NOT LOAD THIS MODULE UNLESS"
+				" YOU REALLY KNOW WHAT YOU ARE DOING!\n");
+		return -ENXIO;
+	}
+
+	printk(KERN_WARNING MODULE_NAME ": WARNING: mapping 16 kB @ 0x%08lx "
+					"in PCI address space, and writing "
+					"rubbish in there.\n", mmio_address);
+	do_test();
+	return 0;
+}
+
+static void __exit cleanup(void)
+{
+	printk(KERN_DEBUG MODULE_NAME ": unloaded.\n");
+}
+
+module_init(init);
+module_exit(cleanup);
+MODULE_LICENSE("GPL");
-- 
1.5.3.7

[RFC PATCH] x86: explicit call to mmiotrace in do_page_fault()

[Pekka Paalanen]

The custom page fault handler list is replaced with a single function
pointer. All related functions and variables are renamed for
mmiotrace.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---
 arch/x86/Kconfig.debug   |   14 ++++-----
 arch/x86/mm/fault.c      |   66 +++++++++++++++++++++++----------------------
 arch/x86/mm/kmmio.c      |   14 +++++-----
 include/asm-x86/kdebug.h |   12 +++-----
 4 files changed, 52 insertions(+), 54 deletions(-)

diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 3daf6c1..0fe4120 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -169,20 +169,18 @@ config IOMMU_LEAK
 	  Add a simple leak tracer to the IOMMU code. This is useful when you
 	  are debugging a buggy device driver that leaks IOMMU mappings.
 
-config PAGE_FAULT_HANDLERS
-	bool "Custom page fault handlers"
-	depends on DEBUG_KERNEL
-	help
-	  Allow the use of custom page fault handlers. A kernel module may
-	  register a function that is called on every page fault. Custom
-	  handlers are used by some debugging and reverse engineering tools.
+config MMIOTRACE_HOOKS
+	bool
+	default n
 
 config MMIOTRACE
 	tristate "Memory mapped IO tracing"
-	depends on DEBUG_KERNEL && PAGE_FAULT_HANDLERS && RELAY && DEBUG_FS
+	depends on DEBUG_KERNEL && RELAY && DEBUG_FS
+	select MMIOTRACE_HOOKS
 	default n
 	help
 	  This will build a kernel module called mmiotrace.
+	  Making this a built-in is heavily discouraged.
 
 	  Mmiotrace traces Memory Mapped I/O access and is meant for debugging
 	  and reverse engineering. The kernel module offers wrapped
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 1d3541d..1100a26 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -49,53 +49,55 @@
 #define PF_RSVD		(1<<3)
 #define PF_INSTR	(1<<4)
 
-#ifdef CONFIG_PAGE_FAULT_HANDLERS
-static HLIST_HEAD(pf_handlers); /* protected by RCU */
-static DEFINE_SPINLOCK(pf_handlers_writer);
+#ifdef CONFIG_MMIOTRACE_HOOKS
+static pf_handler_func mmiotrace_pf_handler; /* protected by RCU */
+static DEFINE_SPINLOCK(mmiotrace_handler_lock);
 
-void register_page_fault_handler(struct pf_handler *new_pfh)
+int mmiotrace_register_pf(pf_handler_func new_pfh)
 {
+	int ret = 0;
 	unsigned long flags;
-	spin_lock_irqsave(&pf_handlers_writer, flags);
-	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
-	spin_unlock_irqrestore(&pf_handlers_writer, flags);
+	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
+	if (mmiotrace_pf_handler)
+		ret = -EBUSY;
+	else
+		mmiotrace_pf_handler = new_pfh;
+	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
+	return ret;
 }
-EXPORT_SYMBOL_GPL(register_page_fault_handler);
+EXPORT_SYMBOL_GPL(mmiotrace_register_pf);
 
 /**
- * unregister_page_fault_handler:
+ * mmiotrace_unregister_pf:
  * The caller must ensure @old_pfh is not in use anymore before freeing it.
- * This function does not guarantee it. The list of handlers is protected by
- * RCU, so you can do this by e.g. calling synchronize_rcu().
+ * This function does not guarantee it. The handler function pointer is
+ * protected by RCU, so you can do this by e.g. calling synchronize_rcu().
  */
-void unregister_page_fault_handler(struct pf_handler *old_pfh)
+int mmiotrace_unregister_pf(pf_handler_func old_pfh)
 {
+	int ret = 0;
 	unsigned long flags;
-	spin_lock_irqsave(&pf_handlers_writer, flags);
-	hlist_del_rcu(&old_pfh->hlist);
-	spin_unlock_irqrestore(&pf_handlers_writer, flags);
+	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
+	if (mmiotrace_pf_handler != old_pfh)
+		ret = -EPERM;
+	else
+		mmiotrace_pf_handler = NULL;
+	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
+	return ret;
 }
-EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
-#endif
+EXPORT_SYMBOL_GPL(mmiotrace_unregister_pf);
+#endif /* CONFIG_MMIOTRACE_HOOKS */
 
 /* returns non-zero if do_page_fault() should return */
-static int handle_custom_pf(struct pt_regs *regs, unsigned long error_code,
-							unsigned long address)
+static inline int call_mmiotrace(struct pt_regs *regs,
+					unsigned long error_code,
+					unsigned long address)
 {
-#ifdef CONFIG_PAGE_FAULT_HANDLERS
+#ifdef CONFIG_MMIOTRACE_HOOKS
 	int ret = 0;
-	struct pf_handler *cur;
-	struct hlist_node *ncur;
-
-	if (hlist_empty(&pf_handlers))
-		return 0;
-
 	rcu_read_lock();
-	hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
-		ret = cur->handler(regs, error_code, address);
-		if (ret)
-			break;
-	}
+	if (mmiotrace_pf_handler)
+		ret = mmiotrace_pf_handler(regs, error_code, address);
 	rcu_read_unlock();
 	return ret;
 #else
@@ -657,7 +659,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 
 	if (notify_page_fault(regs))
 		return;
-	if (handle_custom_pf(regs, error_code, address))
+	if (call_mmiotrace(regs, error_code, address))
 		return;
 
 	/*
diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
index 28411da..e759f7c 100644
--- a/arch/x86/mm/kmmio.c
+++ b/arch/x86/mm/kmmio.c
@@ -51,10 +51,6 @@ static LIST_HEAD(kmmio_probes);
 
 static struct kmmio_context kmmio_ctx[NR_CPUS];
 
-static struct pf_handler kmmio_pf_hook = {
-	.handler = kmmio_page_fault
-};
-
 static struct notifier_block nb_die = {
 	.notifier_call = kmmio_die_notifier
 };
@@ -77,7 +73,8 @@ void cleanup_kmmio(void)
 	 * kmmio_page_table, kmmio_probes
 	 */
 	if (handler_registered) {
-		unregister_page_fault_handler(&kmmio_pf_hook);
+		if (mmiotrace_unregister_pf(&kmmio_page_fault))
+			BUG();
 		synchronize_rcu();
 	}
 	unregister_die_notifier(&nb_die);
@@ -343,8 +340,11 @@ int register_kmmio_probe(struct kmmio_probe *p)
 	}
 
 	if (!handler_registered) {
-		register_page_fault_handler(&kmmio_pf_hook);
-		handler_registered++;
+		if (mmiotrace_register_pf(&kmmio_page_fault))
+			printk(KERN_ERR "mmiotrace: Cannot register page "
+					"fault handler.\n");
+		else
+			handler_registered++;
 	}
 
 out:
diff --git a/include/asm-x86/kdebug.h b/include/asm-x86/kdebug.h
index 7ae2118..4680817 100644
--- a/include/asm-x86/kdebug.h
+++ b/include/asm-x86/kdebug.h
@@ -35,13 +35,11 @@ extern void dump_pagetable(unsigned long);
 extern unsigned long oops_begin(void);
 extern void oops_end(unsigned long, struct pt_regs *, int signr);
 
-struct pf_handler {
-	struct hlist_node hlist;
-	int (*handler)(struct pt_regs *regs, unsigned long error_code,
-						unsigned long address);
-};
+typedef int (*pf_handler_func)(struct pt_regs *regs,
+				unsigned long error_code,
+				unsigned long address);
 
-extern void register_page_fault_handler(struct pf_handler *new_pfh);
-extern void unregister_page_fault_handler(struct pf_handler *old_pfh);
+extern int mmiotrace_register_pf(pf_handler_func new_pfh);
+extern int mmiotrace_unregister_pf(pf_handler_func old_pfh);
 
 #endif
-- 
1.5.3.7

[PATCH] x86 mmiotrace: Use percpu instead of arrays.

[Pekka Paalanen]

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---
 arch/x86/mm/kmmio.c    |   27 +++++++++-------
 arch/x86/mm/mmio-mod.c |   80 +++++++++++++++++++++++++----------------------
 2 files changed, 58 insertions(+), 49 deletions(-)

diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
index e759f7c..5e239d0 100644
--- a/arch/x86/mm/kmmio.c
+++ b/arch/x86/mm/kmmio.c
@@ -16,6 +16,7 @@
 #include <linux/uaccess.h>
 #include <linux/ptrace.h>
 #include <linux/preempt.h>
+#include <linux/percpu.h>
 #include <asm/io.h>
 #include <asm/cacheflush.h>
 #include <asm/errno.h>
@@ -49,7 +50,8 @@ static unsigned int handler_registered;
 static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE];
 static LIST_HEAD(kmmio_probes);
 
-static struct kmmio_context kmmio_ctx[NR_CPUS];
+/* Accessed per-cpu */
+static DEFINE_PER_CPU(struct kmmio_context, kmmio_ctx);
 
 static struct notifier_block nb_die = {
 	.notifier_call = kmmio_die_notifier
@@ -173,8 +175,7 @@ static void disarm_kmmio_fault_page(unsigned long page, int *page_level)
  */
 static int kmmio_handler(struct pt_regs *regs, unsigned long addr)
 {
-	struct kmmio_context *ctx;
-	int cpu;
+	struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx);
 
 	/*
 	 * Preemption is now disabled to prevent process switch during
@@ -187,8 +188,6 @@ static int kmmio_handler(struct pt_regs *regs, unsigned long addr)
 	 * And that interrupt triggers a kmmio trap?
 	 */
 	preempt_disable();
-	cpu = smp_processor_id();
-	ctx = &kmmio_ctx[cpu];
 
 	/* interrupts disabled and CPU-local data => atomicity guaranteed. */
 	if (ctx->active) {
@@ -199,7 +198,7 @@ static int kmmio_handler(struct pt_regs *regs, unsigned long addr)
 		 */
 		printk(KERN_EMERG "mmiotrace: recursive probe hit on CPU %d, "
 					"for address %lu. Ignoring.\n",
-					cpu, addr);
+					smp_processor_id(), addr);
 		goto no_kmmio;
 	}
 	ctx->active++;
@@ -231,6 +230,7 @@ static int kmmio_handler(struct pt_regs *regs, unsigned long addr)
 	/* We hold lock, now we set present bit in PTE and single step. */
 	disarm_kmmio_fault_page(ctx->fpage->page, NULL);
 
+	put_cpu_var(kmmio_ctx);
 	return 1;
 
 no_kmmio_locked:
@@ -238,6 +238,7 @@ no_kmmio_locked:
 	ctx->active--;
 no_kmmio:
 	preempt_enable_no_resched();
+	put_cpu_var(kmmio_ctx);
 	/* page fault not handled by kmmio */
 	return 0;
 }
@@ -249,11 +250,11 @@ no_kmmio:
  */
 static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
 {
-	int cpu = smp_processor_id();
-	struct kmmio_context *ctx = &kmmio_ctx[cpu];
+	int ret = 0;
+	struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx);
 
 	if (!ctx->active)
-		return 0;
+		goto out;
 
 	if (ctx->probe && ctx->probe->post_handler)
 		ctx->probe->post_handler(ctx->probe, condition, regs);
@@ -273,10 +274,12 @@ static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
 	 * will have TF set, in which case, continue the remaining processing
 	 * of do_debug, as if this is not a probe hit.
 	 */
-	if (regs->flags & TF_MASK)
-		return 0;
+	if (!(regs->flags & TF_MASK))
+		ret = 1;
 
-	return 1;
+out:
+	put_cpu_var(kmmio_ctx);
+	return ret;
 }
 
 static int add_kmmio_fault_page(unsigned long page)
diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c
index f492b65..ed62a95 100644
--- a/arch/x86/mm/mmio-mod.c
+++ b/arch/x86/mm/mmio-mod.c
@@ -30,6 +30,7 @@
 #include <linux/mmiotrace.h>
 #include <asm/e820.h> /* for ISA_START_ADDRESS */
 #include <asm/atomic.h>
+#include <linux/percpu.h>
 
 #include "kmmio.h"
 #include "pf_in.h"
@@ -49,11 +50,11 @@ struct trap_reason {
 };
 
 /* Accessed per-cpu. */
-static struct trap_reason pf_reason[NR_CPUS];
-static struct mm_io_header_rw cpu_trace[NR_CPUS];
+static DEFINE_PER_CPU(struct trap_reason, pf_reason);
+static DEFINE_PER_CPU(struct mm_io_header_rw, cpu_trace);
 
 /* Access to this is not per-cpu. */
-static atomic_t dropped[NR_CPUS];
+static DEFINE_PER_CPU(atomic_t, dropped);
 
 static struct file_operations mmio_fops = {
 	.owner = THIS_MODULE,
@@ -150,15 +151,15 @@ static void print_pte(unsigned long address)
  */
 static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
 {
-	const unsigned long cpu = smp_processor_id();
+	const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
 	printk(KERN_EMERG MODULE_NAME ": unexpected fault for address: %lx, "
 					"last fault for address: %lx\n",
-					addr, pf_reason[cpu].addr);
+					addr, my_reason->addr);
 	print_pte(addr);
 #ifdef __i386__
 	print_symbol(KERN_EMERG "faulting EIP is at %s\n", regs->ip);
 	print_symbol(KERN_EMERG "last faulting EIP was at %s\n",
-							pf_reason[cpu].ip);
+							my_reason->ip);
 	printk(KERN_EMERG
 			"eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
 			regs->ax, regs->bx, regs->cx, regs->dx);
@@ -168,100 +169,105 @@ static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
 #else
 	print_symbol(KERN_EMERG "faulting RIP is at %s\n", regs->ip);
 	print_symbol(KERN_EMERG "last faulting RIP was at %s\n",
-							pf_reason[cpu].ip);
+							my_reason->ip);
 	printk(KERN_EMERG "rax: %016lx   rcx: %016lx   rdx: %016lx\n",
 					regs->ax, regs->cx, regs->dx);
 	printk(KERN_EMERG "rsi: %016lx   rdi: %016lx   "
 				"rbp: %016lx   rsp: %016lx\n",
 				regs->si, regs->di, regs->bp, regs->sp);
 #endif
+	put_cpu_var(pf_reason);
 	BUG();
 }
 
 static void pre(struct kmmio_probe *p, struct pt_regs *regs,
 						unsigned long addr)
 {
-	const unsigned long cpu = smp_processor_id();
+	struct trap_reason *my_reason = &get_cpu_var(pf_reason);
+	struct mm_io_header_rw *my_trace = &get_cpu_var(cpu_trace);
 	const unsigned long instptr = instruction_pointer(regs);
 	const enum reason_type type = get_ins_type(instptr);
 
 	/* it doesn't make sense to have more than one active trace per cpu */
-	if (pf_reason[cpu].active_traces)
+	if (my_reason->active_traces)
 		die_kmmio_nesting_error(regs, addr);
 	else
-		pf_reason[cpu].active_traces++;
+		my_reason->active_traces++;
 
-	pf_reason[cpu].type = type;
-	pf_reason[cpu].addr = addr;
-	pf_reason[cpu].ip = instptr;
+	my_reason->type = type;
+	my_reason->addr = addr;
+	my_reason->ip = instptr;
 
-	cpu_trace[cpu].header.type = MMIO_MAGIC;
-	cpu_trace[cpu].header.pid = 0;
-	cpu_trace[cpu].header.data_len = sizeof(struct mm_io_rw);
-	cpu_trace[cpu].rw.address = addr;
+	my_trace->header.type = MMIO_MAGIC;
+	my_trace->header.pid = 0;
+	my_trace->header.data_len = sizeof(struct mm_io_rw);
+	my_trace->rw.address = addr;
 
 	/*
 	 * Only record the program counter when requested.
 	 * It may taint clean-room reverse engineering.
 	 */
 	if (trace_pc)
-		cpu_trace[cpu].rw.pc = instptr;
+		my_trace->rw.pc = instptr;
 	else
-		cpu_trace[cpu].rw.pc = 0;
+		my_trace->rw.pc = 0;
 
-	record_timestamp(&cpu_trace[cpu].header);
+	record_timestamp(&my_trace->header);
 
 	switch (type) {
 	case REG_READ:
-		cpu_trace[cpu].header.type |=
+		my_trace->header.type |=
 			(MMIO_READ << MMIO_OPCODE_SHIFT) |
 			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
 		break;
 	case REG_WRITE:
-		cpu_trace[cpu].header.type |=
+		my_trace->header.type |=
 			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
 			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
-		cpu_trace[cpu].rw.value = get_ins_reg_val(instptr, regs);
+		my_trace->rw.value = get_ins_reg_val(instptr, regs);
 		break;
 	case IMM_WRITE:
-		cpu_trace[cpu].header.type |=
+		my_trace->header.type |=
 			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
 			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
-		cpu_trace[cpu].rw.value = get_ins_imm_val(instptr);
+		my_trace->rw.value = get_ins_imm_val(instptr);
 		break;
 	default:
 		{
 			unsigned char *ip = (unsigned char *)instptr;
-			cpu_trace[cpu].header.type |=
+			my_trace->header.type |=
 					(MMIO_UNKNOWN_OP << MMIO_OPCODE_SHIFT);
-			cpu_trace[cpu].rw.value = (*ip) << 16 |
-							*(ip + 1) << 8 |
-							*(ip + 2);
+			my_trace->rw.value = (*ip) << 16 | *(ip + 1) << 8 |
+								*(ip + 2);
 		}
 	}
+	put_cpu_var(cpu_trace);
+	put_cpu_var(pf_reason);
 }
 
 static void post(struct kmmio_probe *p, unsigned long condition,
 							struct pt_regs *regs)
 {
-	const unsigned long cpu = smp_processor_id();
+	struct trap_reason *my_reason = &get_cpu_var(pf_reason);
+	struct mm_io_header_rw *my_trace = &get_cpu_var(cpu_trace);
 
 	/* this should always return the active_trace count to 0 */
-	pf_reason[cpu].active_traces--;
-	if (pf_reason[cpu].active_traces) {
+	my_reason->active_traces--;
+	if (my_reason->active_traces) {
 		printk(KERN_EMERG MODULE_NAME ": unexpected post handler");
 		BUG();
 	}
 
-	switch (pf_reason[cpu].type) {
+	switch (my_reason->type) {
 	case REG_READ:
-		cpu_trace[cpu].rw.value = get_ins_reg_val(pf_reason[cpu].ip,
-									regs);
+		my_trace->rw.value = get_ins_reg_val(my_reason->ip, regs);
 		break;
 	default:
 		break;
 	}
-	relay_write(chan, &cpu_trace[cpu], sizeof(struct mm_io_header_rw));
+	relay_write(chan, my_trace, sizeof(*my_trace));
+	put_cpu_var(cpu_trace);
+	put_cpu_var(pf_reason);
 }
 
 /*
@@ -274,7 +280,7 @@ static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf,
 					void *prev_subbuf, size_t prev_padding)
 {
 	unsigned int cpu = buf->cpu;
-	atomic_t *drop = &dropped[cpu];
+	atomic_t *drop = &per_cpu(dropped, cpu);
 	int count;
 	if (relay_buf_full(buf)) {
 		if (atomic_inc_return(drop) == 1) {
-- 
1.5.3.7

Re: [RFC PATCH] x86: explicit call to mmiotrace in do_page_fault()

[Arjan van de Ven]

>  	default n
>  	help
>  	  This will build a kernel module called mmiotrace.
> +	  Making this a built-in is heavily discouraged.

why is this? Wouldn't it be nice if distros just shipped with this in their kernel by default
for example?

> -#ifdef CONFIG_PAGE_FAULT_HANDLERS
> -static HLIST_HEAD(pf_handlers); /* protected by RCU */
> -static DEFINE_SPINLOCK(pf_handlers_writer);
> +#ifdef CONFIG_MMIOTRACE_HOOKS
> +static pf_handler_func mmiotrace_pf_handler; /* protected by RCU */
> +static DEFINE_SPINLOCK(mmiotrace_handler_lock);
>  
> -void register_page_fault_handler(struct pf_handler *new_pfh)
> +int mmiotrace_register_pf(pf_handler_func new_pfh)
>  {
> +	int ret = 0;
>  	unsigned long flags;
> -	spin_lock_irqsave(&pf_handlers_writer, flags);
> -	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
> -	spin_unlock_irqrestore(&pf_handlers_writer, flags);
> +	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
> +	if (mmiotrace_pf_handler)
> +		ret = -EBUSY;
> +	else
> +		mmiotrace_pf_handler = new_pfh;
> +	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
> +	return ret;
>  }
> -EXPORT_SYMBOL_GPL(register_page_fault_handler);
> +EXPORT_SYMBOL_GPL(mmiotrace_register_pf);

why not use a weak function instead? should be a ton simpler.

Re: [RFC PATCH] x86: explicit call to mmiotrace in do_page_fault()

[Pekka Paalanen]

On Sat, 9 Feb 2008 10:01:33 -0800
Arjan van de Ven <arjan@infradead.org> wrote:

> >  	default n
> >  	help
> >  	  This will build a kernel module called mmiotrace.
> > +	  Making this a built-in is heavily discouraged.
> 
> why is this? Wouldn't it be nice if distros just shipped with this in their kernel by default
> for example?

It's currently not too nice as a built-in. Originally I did have it depend
on 'm' but maybe Ingo removed it? It's not depending on 'm' anymore in
x86/mm git tree Kconfig.

By default it reserves 32 MB/cpu of relay buffers during module init.
It's this big based on experimentation with the nvidia proprietary driver.
There is no other way than unloading the module to free the buffers, or
even to clear them. This means that every time you start the user space
logger application, it gets old data first. Also poking the module
parameters via /sys is not allowed.

So it's not actually broken as far as I know, it's just not nice IMHO.
And I've never tried it as built-in. I don't think anyone ever has.
I do have a plan to make it a nicer built-in, but it's a long way.

> > +#ifdef CONFIG_MMIOTRACE_HOOKS
> > +static pf_handler_func mmiotrace_pf_handler; /* protected by RCU */
> > +static DEFINE_SPINLOCK(mmiotrace_handler_lock);
> >  
> > +int mmiotrace_register_pf(pf_handler_func new_pfh)
> >  {
> > +	int ret = 0;
> >  	unsigned long flags;
> > +	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
> > +	if (mmiotrace_pf_handler)
> > +		ret = -EBUSY;
> > +	else
> > +		mmiotrace_pf_handler = new_pfh;
> > +	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
> > +	return ret;
> >  }
> > +EXPORT_SYMBOL_GPL(mmiotrace_register_pf);
> 
> why not use a weak function instead? should be a ton simpler.

Ookay, this is the first time I hear about weak functions, and
that I could accomplish something like this with them.
I'll be googling around. These are completely new beasts to me.

Thanks for the keywords, finally I have a clue to follow.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [RFC PATCH] x86: explicit call to mmiotrace in do_page_fault()

[Peter Zijlstra]

On Sat, 2008-02-09 at 19:52 +0200, Pekka Paalanen wrote:

> +int mmiotrace_register_pf(pf_handler_func new_pfh)
>  {
> +	int ret = 0;
>  	unsigned long flags;

> +	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
> +	if (mmiotrace_pf_handler)
> +		ret = -EBUSY;
> +	else
> +		mmiotrace_pf_handler = new_pfh;
> +	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
> +	return ret;
>  }
> +EXPORT_SYMBOL_GPL(mmiotrace_register_pf);
>  
>  /**
> + * mmiotrace_unregister_pf:
>   * The caller must ensure @old_pfh is not in use anymore before freeing it.
> + * This function does not guarantee it. The handler function pointer is
> + * protected by RCU, so you can do this by e.g. calling synchronize_rcu().
>   */
> +int mmiotrace_unregister_pf(pf_handler_func old_pfh)
>  {
> +	int ret = 0;
>  	unsigned long flags;

> +	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
> +	if (mmiotrace_pf_handler != old_pfh)
> +		ret = -EPERM;
> +	else
> +		mmiotrace_pf_handler = NULL;
> +	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
> +	return ret;
>  }

So here you set/unset mmiotrace_pf_handler under protection of a
spinlock.

 
>  /* returns non-zero if do_page_fault() should return */
> +static inline int call_mmiotrace(struct pt_regs *regs,
> +					unsigned long error_code,
> +					unsigned long address)
>  {
> +#ifdef CONFIG_MMIOTRACE_HOOKS
>  	int ret = 0;
>  	rcu_read_lock();
> +	if (mmiotrace_pf_handler)
> +		ret = mmiotrace_pf_handler(regs, error_code, address);
>  	rcu_read_unlock();
>  	return ret;
>  #else

But here you use it under rcu_read_lock(). This doesn't match.

Re: [RFC PATCH] x86: explicit call to mmiotrace in do_page_fault()

[Peter Zijlstra]

On Sat, 2008-02-09 at 10:01 -0800, Arjan van de Ven wrote:
> >  	default n
> >  	help
> >  	  This will build a kernel module called mmiotrace.
> > +	  Making this a built-in is heavily discouraged.
> 
> why is this? Wouldn't it be nice if distros just shipped with this in their kernel by default
> for example?

so default m?

> > -#ifdef CONFIG_PAGE_FAULT_HANDLERS
> > -static HLIST_HEAD(pf_handlers); /* protected by RCU */
> > -static DEFINE_SPINLOCK(pf_handlers_writer);
> > +#ifdef CONFIG_MMIOTRACE_HOOKS
> > +static pf_handler_func mmiotrace_pf_handler; /* protected by RCU */
> > +static DEFINE_SPINLOCK(mmiotrace_handler_lock);
> >  
> > -void register_page_fault_handler(struct pf_handler *new_pfh)
> > +int mmiotrace_register_pf(pf_handler_func new_pfh)
> >  {
> > +	int ret = 0;
> >  	unsigned long flags;
> > -	spin_lock_irqsave(&pf_handlers_writer, flags);
> > -	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
> > -	spin_unlock_irqrestore(&pf_handlers_writer, flags);
> > +	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
> > +	if (mmiotrace_pf_handler)
> > +		ret = -EBUSY;
> > +	else
> > +		mmiotrace_pf_handler = new_pfh;
> > +	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
> > +	return ret;
> >  }
> > -EXPORT_SYMBOL_GPL(register_page_fault_handler);
> > +EXPORT_SYMBOL_GPL(mmiotrace_register_pf);
> 
> why not use a weak function instead? should be a ton simpler.

AFAIC weak functions are a built time thing, not fixed up on module
load. So I'm not seeing the applicability there.

Also, it would force a function call, even for the unlikely case the
tracer is set.

Re: [RFC PATCH] x86: explicit call to mmiotrace in do_page_fault()

[Pekka Enberg]

Hi Pekka,

On Feb 9, 2008 8:23 PM, Pekka Paalanen <pq@iki.fi> wrote:
> By default it reserves 32 MB/cpu of relay buffers during module init.
> It's this big based on experimentation with the nvidia proprietary driver.
> There is no other way than unloading the module to free the buffers, or
> even to clear them. This means that every time you start the user space
> logger application, it gets old data first. Also poking the module
> parameters via /sys is not allowed.

So how is mmiotrace different from, say, blktrace that has an ioctl()
to setup/teardown the relay channels (for mmiotrace it can be a
"status" file that does the setup/teardown when written some
appropriate values)?

                             Pekka

Re: [RFC PATCH] x86: explicit call to mmiotrace in do_page_fault()

[Pekka Paalanen]

On Sat, 9 Feb 2008 20:56:07 +0200
"Pekka Enberg" <penberg@cs.helsinki.fi> wrote:

> Hi Pekka,
> 
> On Feb 9, 2008 8:23 PM, Pekka Paalanen <pq@iki.fi> wrote:
> > By default it reserves 32 MB/cpu of relay buffers during module init.
> > It's this big based on experimentation with the nvidia proprietary driver.
> > There is no other way than unloading the module to free the buffers, or
> > even to clear them. This means that every time you start the user space
> > logger application, it gets old data first. Also poking the module
> > parameters via /sys is not allowed.
> 
> So how is mmiotrace different from, say, blktrace that has an ioctl()
> to setup/teardown the relay channels (for mmiotrace it can be a
> "status" file that does the setup/teardown when written some
> appropriate values)?

No difference. I just have to do it at some point. I can't work on this
every day and when it comes to kernel, I'm a complete newbie. So things
will take time. If people want that I fix this before mmiotrace goes into
mainline, then so be it.

blktrace, thanks for the hint, I should take a look at that.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [RFC PATCH] x86: explicit call to mmiotrace in do_page_fault()

[Pekka Enberg]

Hi,

On Feb 9, 2008 9:11 PM, Pekka Paalanen <pq@iki.fi> wrote:
> No difference. I just have to do it at some point. I can't work on this
> every day and when it comes to kernel, I'm a complete newbie. So things
> will take time. If people want that I fix this before mmiotrace goes into
> mainline, then so be it.

Not my call but please keep in mind that once it's in the kernel we
can't go breaking the ABI (well, we might as this is strictly a
debugging mechanism, but still). Anyway, it shouldn't be that hard to
implement and there are plenty of examples in the kernel and you can
always as for help if you hit a brick wall ;-).

                         Pekka

[RFC PATCH v2] x86: explicit call to mmiotrace in do_page_fault()

[Pekka Paalanen]

The custom page fault handler list is replaced with a single function
pointer. All related functions and variables are renamed for
mmiotrace.

Signed-off-by: Pekka Paalanen <pq@iki.fi>
---
On Sat, 09 Feb 2008 19:39:30 +0100
Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:

> So here you set/unset mmiotrace_pf_handler under protection of a
> spinlock.
..
> But here you use it under rcu_read_lock(). This doesn't match.

Thanks, I fixed it.

Sorry Jan and Pavel, I forgot to CC you in the first go of this
patch. If this makes it into mainline, I don't think it will be left
there for many kernel versions. I plan to make kmmio.h as the API towards
modules in the future, and the page fault callback will "disappear".
I think madwifi could use just fine the kmmio.h functions, if it
works basically just like mmiotrace.

 arch/x86/Kconfig.debug   |   14 ++++-----
 arch/x86/mm/fault.c      |   68 ++++++++++++++++++++++++---------------------
 arch/x86/mm/kmmio.c      |   14 +++++-----
 include/asm-x86/kdebug.h |   12 +++-----
 4 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 3daf6c1..0fe4120 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -169,20 +169,18 @@ config IOMMU_LEAK
 	  Add a simple leak tracer to the IOMMU code. This is useful when you
 	  are debugging a buggy device driver that leaks IOMMU mappings.
 
-config PAGE_FAULT_HANDLERS
-	bool "Custom page fault handlers"
-	depends on DEBUG_KERNEL
-	help
-	  Allow the use of custom page fault handlers. A kernel module may
-	  register a function that is called on every page fault. Custom
-	  handlers are used by some debugging and reverse engineering tools.
+config MMIOTRACE_HOOKS
+	bool
+	default n
 
 config MMIOTRACE
 	tristate "Memory mapped IO tracing"
-	depends on DEBUG_KERNEL && PAGE_FAULT_HANDLERS && RELAY && DEBUG_FS
+	depends on DEBUG_KERNEL && RELAY && DEBUG_FS
+	select MMIOTRACE_HOOKS
 	default n
 	help
 	  This will build a kernel module called mmiotrace.
+	  Making this a built-in is heavily discouraged.
 
 	  Mmiotrace traces Memory Mapped I/O access and is meant for debugging
 	  and reverse engineering. The kernel module offers wrapped
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 1d3541d..ead5db6 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -49,53 +49,57 @@
 #define PF_RSVD		(1<<3)
 #define PF_INSTR	(1<<4)
 
-#ifdef CONFIG_PAGE_FAULT_HANDLERS
-static HLIST_HEAD(pf_handlers); /* protected by RCU */
-static DEFINE_SPINLOCK(pf_handlers_writer);
+#ifdef CONFIG_MMIOTRACE_HOOKS
+static pf_handler_func mmiotrace_pf_handler; /* protected by RCU */
+static DEFINE_SPINLOCK(mmiotrace_handler_lock);
 
-void register_page_fault_handler(struct pf_handler *new_pfh)
+int mmiotrace_register_pf(pf_handler_func new_pfh)
 {
+	int ret = 0;
 	unsigned long flags;
-	spin_lock_irqsave(&pf_handlers_writer, flags);
-	hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
-	spin_unlock_irqrestore(&pf_handlers_writer, flags);
+	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
+	if (mmiotrace_pf_handler)
+		ret = -EBUSY;
+	else
+		rcu_assign_pointer(mmiotrace_pf_handler, new_pfh);
+	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
+	return ret;
 }
-EXPORT_SYMBOL_GPL(register_page_fault_handler);
+EXPORT_SYMBOL_GPL(mmiotrace_register_pf);
 
 /**
- * unregister_page_fault_handler:
+ * mmiotrace_unregister_pf:
  * The caller must ensure @old_pfh is not in use anymore before freeing it.
- * This function does not guarantee it. The list of handlers is protected by
- * RCU, so you can do this by e.g. calling synchronize_rcu().
+ * This function does not guarantee it. The handler function pointer is
+ * protected by RCU, so you can do this by e.g. calling synchronize_rcu().
  */
-void unregister_page_fault_handler(struct pf_handler *old_pfh)
+int mmiotrace_unregister_pf(pf_handler_func old_pfh)
 {
+	int ret = 0;
 	unsigned long flags;
-	spin_lock_irqsave(&pf_handlers_writer, flags);
-	hlist_del_rcu(&old_pfh->hlist);
-	spin_unlock_irqrestore(&pf_handlers_writer, flags);
+	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
+	if (mmiotrace_pf_handler != old_pfh)
+		ret = -EPERM;
+	else
+		rcu_assign_pointer(mmiotrace_pf_handler, NULL);
+	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
+	return ret;
 }
-EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
-#endif
+EXPORT_SYMBOL_GPL(mmiotrace_unregister_pf);
+#endif /* CONFIG_MMIOTRACE_HOOKS */
 
 /* returns non-zero if do_page_fault() should return */
-static int handle_custom_pf(struct pt_regs *regs, unsigned long error_code,
-							unsigned long address)
+static inline int call_mmiotrace(struct pt_regs *regs,
+					unsigned long error_code,
+					unsigned long address)
 {
-#ifdef CONFIG_PAGE_FAULT_HANDLERS
+#ifdef CONFIG_MMIOTRACE_HOOKS
 	int ret = 0;
-	struct pf_handler *cur;
-	struct hlist_node *ncur;
-
-	if (hlist_empty(&pf_handlers))
-		return 0;
-
+	pf_handler_func handler;
 	rcu_read_lock();
-	hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
-		ret = cur->handler(regs, error_code, address);
-		if (ret)
-			break;
-	}
+	handler = rcu_dereference(mmiotrace_pf_handler);
+	if (handler)
+		ret = handler(regs, error_code, address);
 	rcu_read_unlock();
 	return ret;
 #else
@@ -657,7 +661,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 
 	if (notify_page_fault(regs))
 		return;
-	if (handle_custom_pf(regs, error_code, address))
+	if (call_mmiotrace(regs, error_code, address))
 		return;
 
 	/*
diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
index 28411da..e759f7c 100644
--- a/arch/x86/mm/kmmio.c
+++ b/arch/x86/mm/kmmio.c
@@ -51,10 +51,6 @@ static LIST_HEAD(kmmio_probes);
 
 static struct kmmio_context kmmio_ctx[NR_CPUS];
 
-static struct pf_handler kmmio_pf_hook = {
-	.handler = kmmio_page_fault
-};
-
 static struct notifier_block nb_die = {
 	.notifier_call = kmmio_die_notifier
 };
@@ -77,7 +73,8 @@ void cleanup_kmmio(void)
 	 * kmmio_page_table, kmmio_probes
 	 */
 	if (handler_registered) {
-		unregister_page_fault_handler(&kmmio_pf_hook);
+		if (mmiotrace_unregister_pf(&kmmio_page_fault))
+			BUG();
 		synchronize_rcu();
 	}
 	unregister_die_notifier(&nb_die);
@@ -343,8 +340,11 @@ int register_kmmio_probe(struct kmmio_probe *p)
 	}
 
 	if (!handler_registered) {
-		register_page_fault_handler(&kmmio_pf_hook);
-		handler_registered++;
+		if (mmiotrace_register_pf(&kmmio_page_fault))
+			printk(KERN_ERR "mmiotrace: Cannot register page "
+					"fault handler.\n");
+		else
+			handler_registered++;
 	}
 
 out:
diff --git a/include/asm-x86/kdebug.h b/include/asm-x86/kdebug.h
index 7ae2118..4680817 100644
--- a/include/asm-x86/kdebug.h
+++ b/include/asm-x86/kdebug.h
@@ -35,13 +35,11 @@ extern void dump_pagetable(unsigned long);
 extern unsigned long oops_begin(void);
 extern void oops_end(unsigned long, struct pt_regs *, int signr);
 
-struct pf_handler {
-	struct hlist_node hlist;
-	int (*handler)(struct pt_regs *regs, unsigned long error_code,
-						unsigned long address);
-};
+typedef int (*pf_handler_func)(struct pt_regs *regs,
+				unsigned long error_code,
+				unsigned long address);
 
-extern void register_page_fault_handler(struct pf_handler *new_pfh);
-extern void unregister_page_fault_handler(struct pf_handler *old_pfh);
+extern int mmiotrace_register_pf(pf_handler_func new_pfh);
+extern int mmiotrace_unregister_pf(pf_handler_func old_pfh);
 
 #endif
-- 
1.5.3.7

Re: [RFC PATCH v2] x86: explicit call to mmiotrace in do_page_fault()

[Pavel Roskin]

On Sun, 2008-02-10 at 20:05 +0200, Pekka Paalanen wrote:

> Sorry Jan and Pavel, I forgot to CC you in the first go of this
> patch. If this makes it into mainline, I don't think it will be left
> there for many kernel versions. I plan to make kmmio.h as the API towards
> modules in the future, and the page fault callback will "disappear".
> I think madwifi could use just fine the kmmio.h functions, if it
> works basically just like mmiotrace.

In fact, the tracing version of MadWifi has a directory called mmiotrace
with the files lifted from nouveau.  I guess if will just stay in sync
with the upstream.

Many thanks for pushing it into the kernel.

-- 
Regards,
Pavel Roskin

Re: [RFC PATCH v2] x86: explicit call to mmiotrace in do_page_fault()

[Pekka Paalanen]

Hello,

it seems 2.6.25-rc1 is out and mmiotrace did not make it. Thank you all
for your feedback.

Ingo, you can throw away all my patches. I will start from scratch and
come back when I have made mmiotrace built-in-able and got rid of the
page fault hook. It should make a nicer patch set as a whole.

-- 
Pekka Paalanen
http://www.iki.fi/pq/