[RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[Huang Ying]

The testing of Generic Hardware Error Source (GHES) is quite
difficult, because special hardware is needed to trigger the hardware
error. So a software based hardware error injector for GHES is
implemented.

Error notification is not provided in this patch.  So you still need
some NMI/SCI/IRQ injecting support to make it work.

Signed-off-by: Huang Ying <ying.huang@intel.com>
---
 drivers/acpi/apei/Kconfig         |   10 ++
 drivers/acpi/apei/Makefile        |    1 
 drivers/acpi/apei/apei-internal.h |    8 ++
 drivers/acpi/apei/ghes-inj.c      |  132 ++++++++++++++++++++++++++++++++++++++
 drivers/acpi/apei/ghes.c          |   15 ++++
 5 files changed, 165 insertions(+), 1 deletion(-)
 create mode 100644 drivers/acpi/apei/ghes-inj.c

--- a/drivers/acpi/apei/Kconfig
+++ b/drivers/acpi/apei/Kconfig
@@ -54,3 +54,13 @@ config ACPI_APEI_ERST_DEBUG
 	  error information to and from a persistent store. Enable this
 	  if you want to debugging and testing the ERST kernel support
 	  and firmware implementation.
+
+config ACPI_APEI_GHES_INJ
+	tristate "APEI Generic Hardware Error Source (GHES) Injecting Support"
+	depends on ACPI_APEI_GHES
+	help
+	  GHES provides a way to report platform hardware errors (such
+	  as that from chipset).
+
+	  The injector can inject fake hardware error record. This is
+	  used for GHES debugging/testing.
--- a/drivers/acpi/apei/Makefile
+++ b/drivers/acpi/apei/Makefile
@@ -2,5 +2,6 @@ obj-$(CONFIG_ACPI_APEI)		+= apei.o
 obj-$(CONFIG_ACPI_APEI_GHES)	+= ghes.o
 obj-$(CONFIG_ACPI_APEI_EINJ)	+= einj.o
 obj-$(CONFIG_ACPI_APEI_ERST_DEBUG) += erst-dbg.o
+obj-$(CONFIG_ACPI_APEI_GHES_INJ) += ghes-inj.o
 
 apei-y := apei-base.o hest.o cper.o erst.o
--- a/drivers/acpi/apei/apei-internal.h
+++ b/drivers/acpi/apei/apei-internal.h
@@ -33,6 +33,14 @@ struct apei_exec_context {
 	u32 entries;
 };
 
+struct ghes_inject_data {
+	unsigned long error_status_address;
+	u16 source_id;
+	unsigned short valid : 1;
+};
+
+extern struct ghes_inject_data ghes_inject_data;
+
 void apei_exec_ctx_init(struct apei_exec_context *ctx,
 			struct apei_exec_ins_type *ins_table,
 			u32 instructions,
--- /dev/null
+++ b/drivers/acpi/apei/ghes-inj.c
@@ -0,0 +1,132 @@
+/*
+ * APEI Generic Hardware Error Source (GHES) injector support
+ *
+ * Fake hardware error record can be injected. This is used for for
+ * GHES debugging/testing.
+ *
+ * Copyright 2010,2011 Intel Corp.
+ *   Author: Huang Ying <ying.huang@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include <linux/debugfs.h>
+#include <acpi/apei.h>
+
+#include "apei-internal.h"
+
+#define GHES_INJ_PFX		"GHES-INJ: "
+
+#define GHES_INJ_BUF_LEN_MAX	4096
+
+static void *ghes_inj_buf;
+static unsigned int ghes_inj_buf_len;
+
+/* Prevent erst_inj_buf from being accessed concurrently */
+static DEFINE_MUTEX(ghes_inj_mutex);
+
+static ssize_t ghes_inj_write(struct file *filp, const char __user *ubuf,
+			      size_t usize, loff_t *off)
+{
+	int rc;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (*off != 0)
+		return -EINVAL;
+
+	if (usize > GHES_INJ_BUF_LEN_MAX)
+		return -EINVAL;
+
+	if (mutex_lock_interruptible(&ghes_inj_mutex))
+		return -EINTR;
+	ghes_inject_data.valid = 0;
+	/* Wait for all consumers finish using the injecting buffer */
+	synchronize_rcu();
+	if (usize > ghes_inj_buf_len) {
+		void *p;
+		rc = -ENOMEM;
+		p = kmalloc(usize, GFP_KERNEL);
+		if (!p)
+			goto out;
+		kfree(ghes_inj_buf);
+		ghes_inj_buf = p;
+		ghes_inj_buf_len = usize;
+	}
+	rc = copy_from_user(ghes_inj_buf, ubuf, usize);
+	if (rc) {
+		rc = -EFAULT;
+		goto out;
+	}
+	ghes_inject_data.error_status_address = __pa(ghes_inj_buf);
+	/*
+	 * ghes_injiect_data.valid must be set after other fields are
+	 * written
+	 */
+	smp_wmb();
+	ghes_inject_data.valid = 1;
+out:
+	mutex_unlock(&ghes_inj_mutex);
+	return rc ? rc : usize;
+}
+
+static const struct file_operations ghes_inj_fops = {
+	.owner		= THIS_MODULE,
+	.write		= ghes_inj_write,
+};
+
+static struct dentry *ghes_debug_dir;
+
+static __init int ghes_inj_init(void)
+{
+	struct dentry *f;
+	int rc = -ENOMEM;
+
+	ghes_debug_dir = debugfs_create_dir("ghes", apei_get_debugfs_dir());
+	if (!ghes_debug_dir)
+		return rc;
+	f = debugfs_create_file("inject", S_IWUSR, ghes_debug_dir,
+				NULL, &ghes_inj_fops);
+	if (!f)
+		goto err_cleanup;
+	f = debugfs_create_u16("inject_source_id", S_IRUSR | S_IWUSR,
+			       ghes_debug_dir, &ghes_inject_data.source_id);
+	if (!f)
+		goto err_cleanup;
+
+	return 0;
+err_cleanup:
+	debugfs_remove_recursive(ghes_debug_dir);
+	return rc;
+}
+
+static __exit void ghes_inj_exit(void)
+{
+	debugfs_remove_recursive(ghes_debug_dir);
+	ghes_inject_data.valid = 0;
+	/* Wait for all consumers finish using the injecting buffer */
+	synchronize_rcu();
+	kfree(ghes_inj_buf);
+}
+
+module_init(ghes_inj_init);
+module_exit(ghes_inj_exit);
+
+MODULE_AUTHOR("Huang Ying");
+MODULE_DESCRIPTION("APEI Generic Hardware Error Source (GHES) injecting support");
+MODULE_LICENSE("GPL");
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -153,6 +153,9 @@ static unsigned long ghes_estatus_pool_s
 static struct llist_head ghes_estatus_llist;
 static struct irq_work ghes_proc_irq_work;
 
+struct ghes_inject_data ghes_inject_data;
+EXPORT_SYMBOL_GPL(ghes_inject_data);
+
 static int ghes_ioremap_init(void)
 {
 	ghes_ioremap_area = __get_vm_area(PAGE_SIZE * GHES_IOREMAP_PAGES,
@@ -371,7 +374,13 @@ static int ghes_read_estatus(struct ghes
 	u32 len;
 	int rc;
 
-	rc = acpi_atomic_read(&buf_paddr, &g->error_status_address);
+	if (!ghes_inject_data.valid ||
+	    ghes_inject_data.source_id != g->header.source_id)
+		rc = acpi_atomic_read(&buf_paddr, &g->error_status_address);
+	else {
+		buf_paddr = ghes_inject_data.error_status_address;
+		rc = 0;
+	}
 	if (rc) {
 		if (!silent && printk_ratelimit())
 			pr_warning(FW_WARN GHES_PFX
@@ -420,6 +429,10 @@ static void ghes_clear_estatus(struct gh
 	ghes_copy_tofrom_phys(ghes->estatus, ghes->buffer_paddr,
 			      sizeof(ghes->estatus->block_status), 0);
 	ghes->flags &= ~GHES_TO_CLEAR;
+
+	if (ghes_inject_data.valid &&
+	    ghes_inject_data.source_id == ghes->generic->header.source_id)
+		ghes_inject_data.valid = 0;
 }
 
 static void ghes_do_proc(const struct acpi_hest_generic_status *estatus)

Re: [RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[Don Zickus]

On Tue, May 10, 2011 at 11:08:41AM +0800, Huang Ying wrote:
> The testing of Generic Hardware Error Source (GHES) is quite
> difficult, because special hardware is needed to trigger the hardware
> error. So a software based hardware error injector for GHES is
> implemented.
> 
> Error notification is not provided in this patch.  So you still need
> some NMI/SCI/IRQ injecting support to make it work.

Should we add that to this patch, otherwise it seems like the injection
isn't very useful or intuitive from the end-user perspective that they
have to provide their own notification source (ie NMI/SCI/MCE/IRQ).

Cheers,
Don

> 
> Signed-off-by: Huang Ying <ying.huang@intel.com>
> ---
>  drivers/acpi/apei/Kconfig         |   10 ++
>  drivers/acpi/apei/Makefile        |    1 
>  drivers/acpi/apei/apei-internal.h |    8 ++
>  drivers/acpi/apei/ghes-inj.c      |  132 ++++++++++++++++++++++++++++++++++++++
>  drivers/acpi/apei/ghes.c          |   15 ++++
>  5 files changed, 165 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/acpi/apei/ghes-inj.c
> 
> --- a/drivers/acpi/apei/Kconfig
> +++ b/drivers/acpi/apei/Kconfig
> @@ -54,3 +54,13 @@ config ACPI_APEI_ERST_DEBUG
>  	  error information to and from a persistent store. Enable this
>  	  if you want to debugging and testing the ERST kernel support
>  	  and firmware implementation.
> +
> +config ACPI_APEI_GHES_INJ
> +	tristate "APEI Generic Hardware Error Source (GHES) Injecting Support"
> +	depends on ACPI_APEI_GHES
> +	help
> +	  GHES provides a way to report platform hardware errors (such
> +	  as that from chipset).
> +
> +	  The injector can inject fake hardware error record. This is
> +	  used for GHES debugging/testing.
> --- a/drivers/acpi/apei/Makefile
> +++ b/drivers/acpi/apei/Makefile
> @@ -2,5 +2,6 @@ obj-$(CONFIG_ACPI_APEI)		+= apei.o
>  obj-$(CONFIG_ACPI_APEI_GHES)	+= ghes.o
>  obj-$(CONFIG_ACPI_APEI_EINJ)	+= einj.o
>  obj-$(CONFIG_ACPI_APEI_ERST_DEBUG) += erst-dbg.o
> +obj-$(CONFIG_ACPI_APEI_GHES_INJ) += ghes-inj.o
>  
>  apei-y := apei-base.o hest.o cper.o erst.o
> --- a/drivers/acpi/apei/apei-internal.h
> +++ b/drivers/acpi/apei/apei-internal.h
> @@ -33,6 +33,14 @@ struct apei_exec_context {
>  	u32 entries;
>  };
>  
> +struct ghes_inject_data {
> +	unsigned long error_status_address;
> +	u16 source_id;
> +	unsigned short valid : 1;
> +};
> +
> +extern struct ghes_inject_data ghes_inject_data;
> +
>  void apei_exec_ctx_init(struct apei_exec_context *ctx,
>  			struct apei_exec_ins_type *ins_table,
>  			u32 instructions,
> --- /dev/null
> +++ b/drivers/acpi/apei/ghes-inj.c
> @@ -0,0 +1,132 @@
> +/*
> + * APEI Generic Hardware Error Source (GHES) injector support
> + *
> + * Fake hardware error record can be injected. This is used for for
> + * GHES debugging/testing.
> + *
> + * Copyright 2010,2011 Intel Corp.
> + *   Author: Huang Ying <ying.huang@intel.com>
> + *
> + * This program is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU General Public License version
> + * 2 as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; if not, write to the Free Software
> + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/module.h>
> +#include <linux/uaccess.h>
> +#include <linux/debugfs.h>
> +#include <acpi/apei.h>
> +
> +#include "apei-internal.h"
> +
> +#define GHES_INJ_PFX		"GHES-INJ: "
> +
> +#define GHES_INJ_BUF_LEN_MAX	4096
> +
> +static void *ghes_inj_buf;
> +static unsigned int ghes_inj_buf_len;
> +
> +/* Prevent erst_inj_buf from being accessed concurrently */
> +static DEFINE_MUTEX(ghes_inj_mutex);
> +
> +static ssize_t ghes_inj_write(struct file *filp, const char __user *ubuf,
> +			      size_t usize, loff_t *off)
> +{
> +	int rc;
> +
> +	if (!capable(CAP_SYS_ADMIN))
> +		return -EPERM;
> +
> +	if (*off != 0)
> +		return -EINVAL;
> +
> +	if (usize > GHES_INJ_BUF_LEN_MAX)
> +		return -EINVAL;
> +
> +	if (mutex_lock_interruptible(&ghes_inj_mutex))
> +		return -EINTR;
> +	ghes_inject_data.valid = 0;
> +	/* Wait for all consumers finish using the injecting buffer */
> +	synchronize_rcu();
> +	if (usize > ghes_inj_buf_len) {
> +		void *p;
> +		rc = -ENOMEM;
> +		p = kmalloc(usize, GFP_KERNEL);
> +		if (!p)
> +			goto out;
> +		kfree(ghes_inj_buf);
> +		ghes_inj_buf = p;
> +		ghes_inj_buf_len = usize;
> +	}
> +	rc = copy_from_user(ghes_inj_buf, ubuf, usize);
> +	if (rc) {
> +		rc = -EFAULT;
> +		goto out;
> +	}
> +	ghes_inject_data.error_status_address = __pa(ghes_inj_buf);
> +	/*
> +	 * ghes_injiect_data.valid must be set after other fields are
> +	 * written
> +	 */
> +	smp_wmb();
> +	ghes_inject_data.valid = 1;
> +out:
> +	mutex_unlock(&ghes_inj_mutex);
> +	return rc ? rc : usize;
> +}
> +
> +static const struct file_operations ghes_inj_fops = {
> +	.owner		= THIS_MODULE,
> +	.write		= ghes_inj_write,
> +};
> +
> +static struct dentry *ghes_debug_dir;
> +
> +static __init int ghes_inj_init(void)
> +{
> +	struct dentry *f;
> +	int rc = -ENOMEM;
> +
> +	ghes_debug_dir = debugfs_create_dir("ghes", apei_get_debugfs_dir());
> +	if (!ghes_debug_dir)
> +		return rc;
> +	f = debugfs_create_file("inject", S_IWUSR, ghes_debug_dir,
> +				NULL, &ghes_inj_fops);
> +	if (!f)
> +		goto err_cleanup;
> +	f = debugfs_create_u16("inject_source_id", S_IRUSR | S_IWUSR,
> +			       ghes_debug_dir, &ghes_inject_data.source_id);
> +	if (!f)
> +		goto err_cleanup;
> +
> +	return 0;
> +err_cleanup:
> +	debugfs_remove_recursive(ghes_debug_dir);
> +	return rc;
> +}
> +
> +static __exit void ghes_inj_exit(void)
> +{
> +	debugfs_remove_recursive(ghes_debug_dir);
> +	ghes_inject_data.valid = 0;
> +	/* Wait for all consumers finish using the injecting buffer */
> +	synchronize_rcu();
> +	kfree(ghes_inj_buf);
> +}
> +
> +module_init(ghes_inj_init);
> +module_exit(ghes_inj_exit);
> +
> +MODULE_AUTHOR("Huang Ying");
> +MODULE_DESCRIPTION("APEI Generic Hardware Error Source (GHES) injecting support");
> +MODULE_LICENSE("GPL");
> --- a/drivers/acpi/apei/ghes.c
> +++ b/drivers/acpi/apei/ghes.c
> @@ -153,6 +153,9 @@ static unsigned long ghes_estatus_pool_s
>  static struct llist_head ghes_estatus_llist;
>  static struct irq_work ghes_proc_irq_work;
>  
> +struct ghes_inject_data ghes_inject_data;
> +EXPORT_SYMBOL_GPL(ghes_inject_data);
> +
>  static int ghes_ioremap_init(void)
>  {
>  	ghes_ioremap_area = __get_vm_area(PAGE_SIZE * GHES_IOREMAP_PAGES,
> @@ -371,7 +374,13 @@ static int ghes_read_estatus(struct ghes
>  	u32 len;
>  	int rc;
>  
> -	rc = acpi_atomic_read(&buf_paddr, &g->error_status_address);
> +	if (!ghes_inject_data.valid ||
> +	    ghes_inject_data.source_id != g->header.source_id)
> +		rc = acpi_atomic_read(&buf_paddr, &g->error_status_address);
> +	else {
> +		buf_paddr = ghes_inject_data.error_status_address;
> +		rc = 0;
> +	}
>  	if (rc) {
>  		if (!silent && printk_ratelimit())
>  			pr_warning(FW_WARN GHES_PFX
> @@ -420,6 +429,10 @@ static void ghes_clear_estatus(struct gh
>  	ghes_copy_tofrom_phys(ghes->estatus, ghes->buffer_paddr,
>  			      sizeof(ghes->estatus->block_status), 0);
>  	ghes->flags &= ~GHES_TO_CLEAR;
> +
> +	if (ghes_inject_data.valid &&
> +	    ghes_inject_data.source_id == ghes->generic->header.source_id)
> +		ghes_inject_data.valid = 0;
>  }
>  
>  static void ghes_do_proc(const struct acpi_hest_generic_status *estatus)

Re: [RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[Huang Ying]

[-- Attachment #1: Type: text/plain, Size: 859 bytes --]

On 05/17/2011 03:33 AM, Don Zickus wrote:
> On Tue, May 10, 2011 at 11:08:41AM +0800, Huang Ying wrote:
>> The testing of Generic Hardware Error Source (GHES) is quite
>> difficult, because special hardware is needed to trigger the hardware
>> error. So a software based hardware error injector for GHES is
>> implemented.
>>
>> Error notification is not provided in this patch.  So you still need
>> some NMI/SCI/IRQ injecting support to make it work.
> 
> Should we add that to this patch, otherwise it seems like the injection
> isn't very useful or intuitive from the end-user perspective that they
> have to provide their own notification source (ie NMI/SCI/MCE/IRQ).

We can provide the NMI/SCI/IRQ injecting in another patch.  What do you
think about the NMI injecting patch attached?

BTW: MCE will not be processed by GHES.

Best Regards,
Huang Ying

[-- Attachment #2: nmi_inject.patch --]
[-- Type: text/x-patch, Size: 6929 bytes --]

Subject: [RFC] x86, NMI, NMI injecting support

This patch implements trigger NMI on specified CPUs. At the same time,
the NMI reason (contents of port 0x61) can be faked too. This can be
used to debug and test the NMI handler.

Signed-off-by: Huang Ying <ying.huang@intel.com>
---
 arch/x86/Kconfig.debug            |   10 +++
 arch/x86/include/asm/mach_traps.h |    9 ++
 arch/x86/kernel/Makefile          |    1 
 arch/x86/kernel/nmi_inject.c      |  117 ++++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/traps.c           |   34 ++++++++++-
 5 files changed, 167 insertions(+), 4 deletions(-)

--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -304,4 +304,14 @@ config DEBUG_STRICT_USER_COPY_CHECKS
 
 	  If unsure, or if you run an older (pre 4.4) gcc, say N.
 
+config NMI_INJECT
+	tristate "NMI injecting support"
+	depends on DEBUG_KERNEL
+	---help---
+	  This can be used to trigger NMI on specified CPUs. And the
+	  reason of NMI (contents of port 0x61) can be faked
+	  too. This can be used to debug and test the NMI handler.
+
+	  If unsure, say N.
+
 endmenu
--- a/arch/x86/include/asm/mach_traps.h
+++ b/arch/x86/include/asm/mach_traps.h
@@ -17,7 +17,7 @@
 #define NMI_REASON_CLEAR_IOCHK	0x08
 #define NMI_REASON_CLEAR_MASK	0x0f
 
-static inline unsigned char get_nmi_reason(void)
+static inline unsigned char __get_nmi_reason(void)
 {
 	return inb(NMI_REASON_PORT);
 }
@@ -40,4 +40,11 @@ static inline void reassert_nmi(void)
 		unlock_cmos();
 }
 
+struct nmi_reason_inject_data {
+	unsigned char reason;
+	unsigned char valid : 1;
+};
+
+extern struct nmi_reason_inject_data nmi_reason_inject_data;
+
 #endif /* _ASM_X86_MACH_DEFAULT_MACH_TRAPS_H */
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -113,6 +113,7 @@ obj-$(CONFIG_SWIOTLB)			+= pci-swiotlb.o
 obj-$(CONFIG_OF)			+= devicetree.o
 
 obj-y					+= hwerr.o
+obj-$(CONFIG_NMI_INJECT)		+= nmi_inject.o
 
 ###
 # 64 bit specific files
--- /dev/null
+++ b/arch/x86/kernel/nmi_inject.c
@@ -0,0 +1,117 @@
+/*
+ * NMI injector, for NMI handler testing
+ *
+ * Copyright 2010 Intel Corp.
+ *   Author: Huang Ying <ying.huang@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/debugfs.h>
+#include <linux/cpu.h>
+#include <asm/mach_traps.h>
+#include <asm/apic.h>
+
+static int nmi_reason_inject_get(void *data, u64 *val)
+{
+	if (nmi_reason_inject_data.valid)
+		*val = nmi_reason_inject_data.reason;
+	else
+		*val = ~0ULL;
+	return 0;
+}
+
+static int nmi_reason_inject_set(void *data, u64 val)
+{
+	nmi_reason_inject_data.reason = val;
+	nmi_reason_inject_data.valid = 1;
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(nmi_reason_inject_fops, nmi_reason_inject_get,
+			nmi_reason_inject_set, "0x%llx\n");
+
+static int nmi_reason_uninject_set(void *data, u64 val)
+{
+	nmi_reason_inject_data.valid = 0;
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(nmi_reason_uninject_fops, NULL,
+			nmi_reason_uninject_set, "%llu\n");
+
+static int nmi_inject_set(void *data, u64 val)
+{
+	int cpu;
+	cpumask_var_t cpu_mask;
+
+	alloc_cpumask_var(&cpu_mask, GFP_KERNEL);
+	cpumask_clear(cpu_mask);
+	for_each_online_cpu(cpu) {
+		if (cpu >= sizeof(val))
+			continue;
+		if (val & (1ULL << cpu))
+			cpumask_set_cpu(cpu, cpu_mask);
+	}
+	if (!cpumask_empty(cpu_mask))
+		apic->send_IPI_mask(cpu_mask, NMI_VECTOR);
+	free_cpumask_var(cpu_mask);
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(nmi_inject_fops, NULL, nmi_inject_set, "0x%llx\n");
+
+static struct dentry *nmi_debug_dir;
+
+static int __init nmi_inject_init(void)
+{
+	int rc;
+	struct dentry *de;
+
+	rc = -ENOMEM;
+	nmi_debug_dir = debugfs_create_dir("nmi", NULL);
+	if (!nmi_debug_dir)
+		return rc;
+	de = debugfs_create_file("inject", S_IWUSR, nmi_debug_dir,
+				 NULL, &nmi_inject_fops);
+	if (!de)
+		goto err;
+	de = debugfs_create_file("reason_inject", S_IRUSR | S_IWUSR,
+				 nmi_debug_dir, NULL, &nmi_reason_inject_fops);
+	if (!de)
+		goto err;
+	de = debugfs_create_file("reason_uninject", S_IWUSR,
+				 nmi_debug_dir, NULL, &nmi_reason_uninject_fops);
+	if (!de)
+		goto err;
+
+	return 0;
+err:
+	debugfs_remove_recursive(nmi_debug_dir);
+	return rc;
+}
+
+static void __exit nmi_inject_exit(void)
+{
+	debugfs_remove_recursive(nmi_debug_dir);
+}
+
+module_init(nmi_inject_init);
+module_exit(nmi_inject_exit);
+
+MODULE_AUTHOR("Huang Ying");
+MODULE_DESCRIPTION("NMI injecting support");
+MODULE_LICENSE("GPL");
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -316,6 +316,34 @@ static int __init setup_unknown_nmi_pani
 }
 __setup("unknown_nmi_panic", setup_unknown_nmi_panic);
 
+struct nmi_reason_inject_data nmi_reason_inject_data;
+EXPORT_SYMBOL_GPL(nmi_reason_inject_data);
+
+static inline unsigned char get_nmi_reason(void)
+{
+	if (nmi_reason_inject_data.valid)
+		return nmi_reason_inject_data.reason;
+	else
+		return __get_nmi_reason();
+}
+
+static inline void outb_nmi_reason(unsigned char reason)
+{
+	static unsigned char prev_reason;
+
+	if (nmi_reason_inject_data.valid) {
+		if (reason & NMI_REASON_CLEAR_SERR)
+			nmi_reason_inject_data.reason &= ~NMI_REASON_SERR;
+		if (prev_reason == (reason | NMI_REASON_CLEAR_IOCHK) &&
+		    !(reason & NMI_REASON_CLEAR_IOCHK))
+			nmi_reason_inject_data.reason &= ~NMI_REASON_IOCHK;
+		if (!nmi_reason_inject_data.reason)
+			nmi_reason_inject_data.valid = 0;
+		prev_reason = reason;
+	} else
+		outb(reason, NMI_REASON_PORT);
+}
+
 static notrace __kprobes void
 pci_serr_error(unsigned char reason, struct pt_regs *regs)
 {
@@ -340,7 +368,7 @@ pci_serr_error(unsigned char reason, str
 
 	/* Clear and disable the PCI SERR error line. */
 	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
-	outb(reason, NMI_REASON_PORT);
+	outb_nmi_reason(reason);
 }
 
 static notrace __kprobes void
@@ -358,7 +386,7 @@ io_check_error(unsigned char reason, str
 
 	/* Re-enable the IOCK line, wait for a few seconds */
 	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
-	outb(reason, NMI_REASON_PORT);
+	outb_nmi_reason(reason);
 
 	i = 20000;
 	while (--i) {
@@ -367,7 +395,7 @@ io_check_error(unsigned char reason, str
 	}
 
 	reason &= ~NMI_REASON_CLEAR_IOCHK;
-	outb(reason, NMI_REASON_PORT);
+	outb_nmi_reason(reason);
 }
 
 void set_unknown_nmi_as_hwerr(void)

Re: [RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[Don Zickus]

On Tue, May 17, 2011 at 02:41:53PM +0800, Huang Ying wrote:
> On 05/17/2011 03:33 AM, Don Zickus wrote:
> > On Tue, May 10, 2011 at 11:08:41AM +0800, Huang Ying wrote:
> >> The testing of Generic Hardware Error Source (GHES) is quite
> >> difficult, because special hardware is needed to trigger the hardware
> >> error. So a software based hardware error injector for GHES is
> >> implemented.
> >>
> >> Error notification is not provided in this patch.  So you still need
> >> some NMI/SCI/IRQ injecting support to make it work.
> > 
> > Should we add that to this patch, otherwise it seems like the injection
> > isn't very useful or intuitive from the end-user perspective that they
> > have to provide their own notification source (ie NMI/SCI/MCE/IRQ).
> 
> We can provide the NMI/SCI/IRQ injecting in another patch.  What do you
> think about the NMI injecting patch attached?

I understand what the patch is doing and I like the various injection
points, but looking at your other injection modules I start to wonder if
there is a smarter and easier way to do all this.  I believe the software
injection is definitely useful but it does add bloat to the kernel.

I am starting to like Ingo's event filtering idea for stuff like this I
think (though I am still wrapping my head around it).  The beauty of
kprobes and tracepoints and even jump labels was that they were not very
intrusive, they did their work on the side.  It would be nice if we could
figure out a framework for the injection stuff that did something similar.

Perhaps Ingo has some ideas?

Cheers,
Don

> 
> BTW: MCE will not be processed by GHES.
> 
> Best Regards,
> Huang Ying

> Subject: [RFC] x86, NMI, NMI injecting support
> 
> This patch implements trigger NMI on specified CPUs. At the same time,
> the NMI reason (contents of port 0x61) can be faked too. This can be
> used to debug and test the NMI handler.
> 
> Signed-off-by: Huang Ying <ying.huang@intel.com>
> ---
>  arch/x86/Kconfig.debug            |   10 +++
>  arch/x86/include/asm/mach_traps.h |    9 ++
>  arch/x86/kernel/Makefile          |    1 
>  arch/x86/kernel/nmi_inject.c      |  117 ++++++++++++++++++++++++++++++++++++++
>  arch/x86/kernel/traps.c           |   34 ++++++++++-
>  5 files changed, 167 insertions(+), 4 deletions(-)
> 
> --- a/arch/x86/Kconfig.debug
> +++ b/arch/x86/Kconfig.debug
> @@ -304,4 +304,14 @@ config DEBUG_STRICT_USER_COPY_CHECKS
>  
>  	  If unsure, or if you run an older (pre 4.4) gcc, say N.
>  
> +config NMI_INJECT
> +	tristate "NMI injecting support"
> +	depends on DEBUG_KERNEL
> +	---help---
> +	  This can be used to trigger NMI on specified CPUs. And the
> +	  reason of NMI (contents of port 0x61) can be faked
> +	  too. This can be used to debug and test the NMI handler.
> +
> +	  If unsure, say N.
> +
>  endmenu
> --- a/arch/x86/include/asm/mach_traps.h
> +++ b/arch/x86/include/asm/mach_traps.h
> @@ -17,7 +17,7 @@
>  #define NMI_REASON_CLEAR_IOCHK	0x08
>  #define NMI_REASON_CLEAR_MASK	0x0f
>  
> -static inline unsigned char get_nmi_reason(void)
> +static inline unsigned char __get_nmi_reason(void)
>  {
>  	return inb(NMI_REASON_PORT);
>  }
> @@ -40,4 +40,11 @@ static inline void reassert_nmi(void)
>  		unlock_cmos();
>  }
>  
> +struct nmi_reason_inject_data {
> +	unsigned char reason;
> +	unsigned char valid : 1;
> +};
> +
> +extern struct nmi_reason_inject_data nmi_reason_inject_data;
> +
>  #endif /* _ASM_X86_MACH_DEFAULT_MACH_TRAPS_H */
> --- a/arch/x86/kernel/Makefile
> +++ b/arch/x86/kernel/Makefile
> @@ -113,6 +113,7 @@ obj-$(CONFIG_SWIOTLB)			+= pci-swiotlb.o
>  obj-$(CONFIG_OF)			+= devicetree.o
>  
>  obj-y					+= hwerr.o
> +obj-$(CONFIG_NMI_INJECT)		+= nmi_inject.o
>  
>  ###
>  # 64 bit specific files
> --- /dev/null
> +++ b/arch/x86/kernel/nmi_inject.c
> @@ -0,0 +1,117 @@
> +/*
> + * NMI injector, for NMI handler testing
> + *
> + * Copyright 2010 Intel Corp.
> + *   Author: Huang Ying <ying.huang@intel.com>
> + *
> + * This program is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU General Public License version
> + * 2 as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; if not, write to the Free Software
> + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/module.h>
> +#include <linux/debugfs.h>
> +#include <linux/cpu.h>
> +#include <asm/mach_traps.h>
> +#include <asm/apic.h>
> +
> +static int nmi_reason_inject_get(void *data, u64 *val)
> +{
> +	if (nmi_reason_inject_data.valid)
> +		*val = nmi_reason_inject_data.reason;
> +	else
> +		*val = ~0ULL;
> +	return 0;
> +}
> +
> +static int nmi_reason_inject_set(void *data, u64 val)
> +{
> +	nmi_reason_inject_data.reason = val;
> +	nmi_reason_inject_data.valid = 1;
> +	return 0;
> +}
> +
> +DEFINE_SIMPLE_ATTRIBUTE(nmi_reason_inject_fops, nmi_reason_inject_get,
> +			nmi_reason_inject_set, "0x%llx\n");
> +
> +static int nmi_reason_uninject_set(void *data, u64 val)
> +{
> +	nmi_reason_inject_data.valid = 0;
> +	return 0;
> +}
> +
> +DEFINE_SIMPLE_ATTRIBUTE(nmi_reason_uninject_fops, NULL,
> +			nmi_reason_uninject_set, "%llu\n");
> +
> +static int nmi_inject_set(void *data, u64 val)
> +{
> +	int cpu;
> +	cpumask_var_t cpu_mask;
> +
> +	alloc_cpumask_var(&cpu_mask, GFP_KERNEL);
> +	cpumask_clear(cpu_mask);
> +	for_each_online_cpu(cpu) {
> +		if (cpu >= sizeof(val))
> +			continue;
> +		if (val & (1ULL << cpu))
> +			cpumask_set_cpu(cpu, cpu_mask);
> +	}
> +	if (!cpumask_empty(cpu_mask))
> +		apic->send_IPI_mask(cpu_mask, NMI_VECTOR);
> +	free_cpumask_var(cpu_mask);
> +	return 0;
> +}
> +
> +DEFINE_SIMPLE_ATTRIBUTE(nmi_inject_fops, NULL, nmi_inject_set, "0x%llx\n");
> +
> +static struct dentry *nmi_debug_dir;
> +
> +static int __init nmi_inject_init(void)
> +{
> +	int rc;
> +	struct dentry *de;
> +
> +	rc = -ENOMEM;
> +	nmi_debug_dir = debugfs_create_dir("nmi", NULL);
> +	if (!nmi_debug_dir)
> +		return rc;
> +	de = debugfs_create_file("inject", S_IWUSR, nmi_debug_dir,
> +				 NULL, &nmi_inject_fops);
> +	if (!de)
> +		goto err;
> +	de = debugfs_create_file("reason_inject", S_IRUSR | S_IWUSR,
> +				 nmi_debug_dir, NULL, &nmi_reason_inject_fops);
> +	if (!de)
> +		goto err;
> +	de = debugfs_create_file("reason_uninject", S_IWUSR,
> +				 nmi_debug_dir, NULL, &nmi_reason_uninject_fops);
> +	if (!de)
> +		goto err;
> +
> +	return 0;
> +err:
> +	debugfs_remove_recursive(nmi_debug_dir);
> +	return rc;
> +}
> +
> +static void __exit nmi_inject_exit(void)
> +{
> +	debugfs_remove_recursive(nmi_debug_dir);
> +}
> +
> +module_init(nmi_inject_init);
> +module_exit(nmi_inject_exit);
> +
> +MODULE_AUTHOR("Huang Ying");
> +MODULE_DESCRIPTION("NMI injecting support");
> +MODULE_LICENSE("GPL");
> --- a/arch/x86/kernel/traps.c
> +++ b/arch/x86/kernel/traps.c
> @@ -316,6 +316,34 @@ static int __init setup_unknown_nmi_pani
>  }
>  __setup("unknown_nmi_panic", setup_unknown_nmi_panic);
>  
> +struct nmi_reason_inject_data nmi_reason_inject_data;
> +EXPORT_SYMBOL_GPL(nmi_reason_inject_data);
> +
> +static inline unsigned char get_nmi_reason(void)
> +{
> +	if (nmi_reason_inject_data.valid)
> +		return nmi_reason_inject_data.reason;
> +	else
> +		return __get_nmi_reason();
> +}
> +
> +static inline void outb_nmi_reason(unsigned char reason)
> +{
> +	static unsigned char prev_reason;
> +
> +	if (nmi_reason_inject_data.valid) {
> +		if (reason & NMI_REASON_CLEAR_SERR)
> +			nmi_reason_inject_data.reason &= ~NMI_REASON_SERR;
> +		if (prev_reason == (reason | NMI_REASON_CLEAR_IOCHK) &&
> +		    !(reason & NMI_REASON_CLEAR_IOCHK))
> +			nmi_reason_inject_data.reason &= ~NMI_REASON_IOCHK;
> +		if (!nmi_reason_inject_data.reason)
> +			nmi_reason_inject_data.valid = 0;
> +		prev_reason = reason;
> +	} else
> +		outb(reason, NMI_REASON_PORT);
> +}
> +
>  static notrace __kprobes void
>  pci_serr_error(unsigned char reason, struct pt_regs *regs)
>  {
> @@ -340,7 +368,7 @@ pci_serr_error(unsigned char reason, str
>  
>  	/* Clear and disable the PCI SERR error line. */
>  	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
> -	outb(reason, NMI_REASON_PORT);
> +	outb_nmi_reason(reason);
>  }
>  
>  static notrace __kprobes void
> @@ -358,7 +386,7 @@ io_check_error(unsigned char reason, str
>  
>  	/* Re-enable the IOCK line, wait for a few seconds */
>  	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
> -	outb(reason, NMI_REASON_PORT);
> +	outb_nmi_reason(reason);
>  
>  	i = 20000;
>  	while (--i) {
> @@ -367,7 +395,7 @@ io_check_error(unsigned char reason, str
>  	}
>  
>  	reason &= ~NMI_REASON_CLEAR_IOCHK;
> -	outb(reason, NMI_REASON_PORT);
> +	outb_nmi_reason(reason);
>  }
>  
>  void set_unknown_nmi_as_hwerr(void)

Re: [RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[Ingo Molnar]

* Don Zickus <dzickus@redhat.com> wrote:

> On Tue, May 17, 2011 at 02:41:53PM +0800, Huang Ying wrote:
> > On 05/17/2011 03:33 AM, Don Zickus wrote:
> > > On Tue, May 10, 2011 at 11:08:41AM +0800, Huang Ying wrote:
> > >> The testing of Generic Hardware Error Source (GHES) is quite
> > >> difficult, because special hardware is needed to trigger the hardware
> > >> error. So a software based hardware error injector for GHES is
> > >> implemented.
> > >>
> > >> Error notification is not provided in this patch.  So you still need
> > >> some NMI/SCI/IRQ injecting support to make it work.
> > > 
> > > Should we add that to this patch, otherwise it seems like the injection
> > > isn't very useful or intuitive from the end-user perspective that they
> > > have to provide their own notification source (ie NMI/SCI/MCE/IRQ).
> > 
> > We can provide the NMI/SCI/IRQ injecting in another patch.  What do you
> > think about the NMI injecting patch attached?
> 
> I understand what the patch is doing and I like the various injection
> points, but looking at your other injection modules I start to wonder if
> there is a smarter and easier way to do all this.  I believe the software
> injection is definitely useful but it does add bloat to the kernel.
> 
> I am starting to like Ingo's event filtering idea for stuff like this I
> think (though I am still wrapping my head around it).  The beauty of
> kprobes and tracepoints and even jump labels was that they were not very
> intrusive, they did their work on the side.  It would be nice if we could
> figure out a framework for the injection stuff that did something similar.
> 
> Perhaps Ingo has some ideas?

Boris has injection in the EDAC code as well and wants it for RAS purposes and 
i recently outlined to him how event injection could possible look like in the 
not so far future:

---------------->

I think the model we want is to inject actual perf events at the *kernel* 
level, and to add the ability for some events (MCE events here) to also run a 
(optional) callback once user-space does that injection.

So for example [sufficienty privileged] user-space could inject *any* perf 
event - for example a PERF_COUNT_HW_CACHE_MISSES event (for test purposes) and 
any tooling that runs could not tell apart this injected event from a real 
event.

Once we have that, adding a injection callback to MCE events is just another 
step: such a callback could propagate the injected event to the real hardware 
for example, if that is possible. (it would validate, etc. as well)

In the generic case the event just gets injected into the perf event stream.

The ABI for injection could be some obvious extension, either another ioctl 
variant to the perf fd itself, we already have various ways to access it:

 #define PERF_EVENT_IOC_ENABLE           _IO ('$', 0)
 #define PERF_EVENT_IOC_DISABLE          _IO ('$', 1)
 #define PERF_EVENT_IOC_REFRESH          _IO ('$', 2)
 #define PERF_EVENT_IOC_RESET            _IO ('$', 3)
 #define PERF_EVENT_IOC_PERIOD           _IOW('$', 4, __u64)
 #define PERF_EVENT_IOC_SET_OUTPUT       _IO ('$', 5)
 #define PERF_EVENT_IOC_SET_FILTER       _IOW('$', 6, char *)

Or sys_write() access to the perf event fd. The sys_write() one looks like the
conceptually nicest solution to me, because we can read() the fd as well to get
event (counts..) out of it.

I think this model would give us a *lot* of testing power, and we could utilize
arbitrary hardware-injection capabilities as well.

<----------------

That way what would remain in mm/memory-failure.c file is all the useful (and 
interesting!) MM specific knowledge: the method of getting to a list of 
affected tasks for policy action, to collect the tasks that are affected by an 
anonymous page going bad, or by a pagecache page going bad, etc.

These would be offered as filter action functionality, and could be triggered 
from filters straight in the kernel, without having to touch a user-space 
daemon.

The whole boring transport, filtering, enumeration and configuration that is 
duplicated here would go away and would be replaced by EVENT() definitions in 
the places that generate events and callbacks to filter action in 
mm/memory-inject.c.

Now what is somewhat unfortunate as a practical matter is that some of this 
functionality has already been exposed in semi-ABI ways in an ad-hoc fashion, 
so some of the design may be hardcoded. That does not keep me from pointing out 
when i see the mess growing ... :-)

Thanks,

	Ingo

Re: [RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[Borislav Petkov]

On Tue, May 17, 2011 at 09:18:03PM +0200, Ingo Molnar wrote:
> So for example [sufficienty privileged] user-space could inject *any*
> perf event - for example a PERF_COUNT_HW_CACHE_MISSES event (for test
> purposes) and any tooling that runs could not tell apart this injected
> event from a real event.

Yeah about that, I was recently speculating how that would work. So do we do

$ perf record ...

in the one xterm, and, in the other,

$ perf inject

so that while recording, we can inject some events from userspace? Or
do we inject it, it gets buffered somewhere in the meantime and then
the next perf record session sees it along with the remaining injection
events?

The ras inject part would obviously only use the callback and the
injection will happen immediately.

Hmm...

-- 
Regards/Gruss,
    Boris.

Re: [RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[Ingo Molnar]

* Borislav Petkov <bp@alien8.de> wrote:

> On Tue, May 17, 2011 at 09:18:03PM +0200, Ingo Molnar wrote:
> > So for example [sufficienty privileged] user-space could inject *any*
> > perf event - for example a PERF_COUNT_HW_CACHE_MISSES event (for test
> > purposes) and any tooling that runs could not tell apart this injected
> > event from a real event.
> 
> Yeah about that, I was recently speculating how that would work. So do we do
> 
> $ perf record ...
> 
> in the one xterm, and, in the other,
> 
> $ perf inject
> 
> so that while recording, we can inject some events from userspace? Or
> do we inject it, it gets buffered somewhere in the meantime and then
> the next perf record session sees it along with the remaining injection
> events?

Well, for persistent events there would be interim buffering even if there's no 
observation going on anywhere. I.e. there's always an 'observer' of events.

For non-persistent events, if they are injected, then they are like trace 
events for which nobody is interested in: they are lost.

Thanks,

	Ingo

Re: [RFC] ACPI, APEI, Generic Hardware Error Source (GHES) injecting support

[Ingo Molnar]

* Borislav Petkov <bp@alien8.de> wrote:

> On Tue, May 17, 2011 at 09:18:03PM +0200, Ingo Molnar wrote:
> > So for example [sufficienty privileged] user-space could inject *any*
> > perf event - for example a PERF_COUNT_HW_CACHE_MISSES event (for test
> > purposes) and any tooling that runs could not tell apart this injected
> > event from a real event.
> 
> Yeah about that, I was recently speculating how that would work. So do we do
> 
> $ perf record ...
> 
> in the one xterm, and, in the other,
> 
> $ perf inject
> 
> so that while recording, we can inject some events from userspace? Or
> do we inject it, it gets buffered somewhere in the meantime and then
> the next perf record session sees it along with the remaining injection
> events?

Well, for persistent events there would be interim buffering even if there's no 
observation going on anywhere. I.e. there's always an 'observer' of events.

For non-persistent events, if they are injected, then they are like trace 
events for which nobody is interested in: they are lost.

Thanks,

	Ingo