Linux-EDAC Archive on lore.kernel.org
 help / color / Atom feed
From: Robert Richter <rrichter@marvell.com>
To: Borislav Petkov <bp@alien8.de>, Tony Luck <tony.luck@intel.com>,
	"James Morse" <james.morse@arm.com>,
	Mauro Carvalho Chehab <mchehab@kernel.org>
Cc: "linux-edac@vger.kernel.org" <linux-edac@vger.kernel.org>,
	"linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
	Robert Richter <rrichter@marvell.com>
Subject: [PATCH 16/21] EDAC, ghes: Create one memory controller device per node
Date: Wed, 29 May 2019 08:44:39 +0000
Message-ID: <20190529084344.28562-17-rrichter@marvell.com> (raw)
In-Reply-To: <20190529084344.28562-1-rrichter@marvell.com>

Typically for most systems, there is one edac memory controller device
per node. This patch implements the same for the ghes driver. Now,
create multiple mc devices and map the dimms based on the node id.

We need at least one node that is used as fallback if no node
information is available in the error report.

Here a complete and consistent error report from a ThunderX2 system
(zero counter values dropped):

 # find /sys/devices/system/edac/mc/ -name \*count | sort -V | xargs grep . | sed -e '/:0/d'
 /sys/devices/system/edac/mc/mc0/ce_count:11
 /sys/devices/system/edac/mc/mc0/ce_noinfo_count:1
 /sys/devices/system/edac/mc/mc0/csrow2/ce_count:5
 /sys/devices/system/edac/mc/mc0/csrow2/ch0_ce_count:5
 /sys/devices/system/edac/mc/mc0/csrow3/ce_count:3
 /sys/devices/system/edac/mc/mc0/csrow3/ch0_ce_count:3
 /sys/devices/system/edac/mc/mc0/csrow4/ce_count:2
 /sys/devices/system/edac/mc/mc0/csrow4/ch0_ce_count:2
 /sys/devices/system/edac/mc/mc0/dimm2/dimm_ce_count:5
 /sys/devices/system/edac/mc/mc0/dimm3/dimm_ce_count:3
 /sys/devices/system/edac/mc/mc0/dimm4/dimm_ce_count:2
 /sys/devices/system/edac/mc/mc1/ce_count:7
 /sys/devices/system/edac/mc/mc1/csrow2/ce_count:4
 /sys/devices/system/edac/mc/mc1/csrow2/ch0_ce_count:4
 /sys/devices/system/edac/mc/mc1/csrow3/ce_count:1
 /sys/devices/system/edac/mc/mc1/csrow3/ch0_ce_count:1
 /sys/devices/system/edac/mc/mc1/csrow6/ce_count:2
 /sys/devices/system/edac/mc/mc1/csrow6/ch0_ce_count:2
 /sys/devices/system/edac/mc/mc1/dimm2/dimm_ce_count:4
 /sys/devices/system/edac/mc/mc1/dimm3/dimm_ce_count:1
 /sys/devices/system/edac/mc/mc1/dimm6/dimm_ce_count:2

Signed-off-by: Robert Richter <rrichter@marvell.com>
---
 drivers/edac/ghes_edac.c | 126 ++++++++++++++++++++++++++++++++-------
 1 file changed, 104 insertions(+), 22 deletions(-)

diff --git a/drivers/edac/ghes_edac.c b/drivers/edac/ghes_edac.c
index c39cdfdfb8db..e5fa977bcfd9 100644
--- a/drivers/edac/ghes_edac.c
+++ b/drivers/edac/ghes_edac.c
@@ -18,6 +18,7 @@
 #include <ras/ras_event.h>
 
 struct ghes_edac_pvt {
+	struct device dev;
 	struct list_head list;
 	struct ghes *ghes;
 	struct mem_ctl_info *mci;
@@ -28,7 +29,7 @@ struct ghes_edac_pvt {
 };
 
 static atomic_t ghes_init = ATOMIC_INIT(0);
-static struct ghes_edac_pvt *ghes_pvt;
+struct mem_ctl_info *fallback;
 
 /*
  * Sync with other, potentially concurrent callers of
@@ -161,15 +162,15 @@ static void ghes_edac_set_nid(const struct dmi_header *dh, void *arg)
 	}
 }
 
-static int get_dimm_smbios_index(u16 handle)
+static int get_dimm_smbios_index(struct mem_ctl_info *mci, u16 handle)
 {
-	struct mem_ctl_info *mci = ghes_pvt->mci;
 	struct dimm_info *dimm;
 
 	mci_for_each_dimm(mci, dimm) {
 		if (dimm->smbios_handle == handle)
 			return dimm->idx;
 	}
+
 	return -1;
 }
 
@@ -370,6 +371,9 @@ static void mci_add_dimm_info(struct mem_ctl_info *mci)
 	int index = 0;
 
 	for_each_dimm(dimm) {
+		if (mci->mc_idx != dimm->numa_node)
+			continue;
+
 		dmi_dimm = &dimm->dimm_info;
 		mci_dimm = edac_get_dimm_by_index(mci, index);
 
@@ -390,17 +394,35 @@ static void mci_add_dimm_info(struct mem_ctl_info *mci)
 			index, mci->tot_dimms);
 }
 
+static struct mem_ctl_info *get_mc_by_node(int nid)
+{
+	struct mem_ctl_info *mci = edac_mc_find(nid);
+
+	if (mci)
+		return mci;
+
+	if (num_possible_nodes() > 1) {
+		edac_mc_printk(fallback, KERN_WARNING,
+			"Invalid or no node information, falling back to first node: %s",
+			fallback->dev_name);
+	}
+
+	return fallback;
+}
+
 void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 {
 	struct dimm_info *dimm_info;
 	enum hw_event_mc_err_type type;
 	struct edac_raw_error_desc *e;
 	struct mem_ctl_info *mci;
-	struct ghes_edac_pvt *pvt = ghes_pvt;
+	struct ghes_edac_pvt *pvt;
 	unsigned long flags;
 	char *p;
+	int nid = NUMA_NO_NODE;
 
-	if (!pvt)
+	/* We need at least one mc */
+	if (WARN_ON_ONCE(!fallback))
 		return;
 
 	/*
@@ -413,7 +435,11 @@ void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 
 	spin_lock_irqsave(&ghes_lock, flags);
 
-	mci = pvt->mci;
+	/* select the node's mc device */
+	if (mem_err->validation_bits & CPER_MEM_VALID_NODE)
+		nid = mem_err->node;
+	mci = get_mc_by_node(nid);
+	pvt = mci->pvt_info;
 	e = &mci->error_desc;
 
 	/* Cleans the error report buffer */
@@ -546,7 +572,7 @@ void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 			p += sprintf(p, "DIMM DMI handle: 0x%.4x ",
 				     mem_err->mem_dev_handle);
 
-		index = get_dimm_smbios_index(mem_err->mem_dev_handle);
+		index = get_dimm_smbios_index(mci, mem_err->mem_dev_handle);
 		if (index >= 0)
 			e->top_layer = index;
 	}
@@ -645,15 +671,29 @@ static struct acpi_platform_list plat_list[] = {
 	{ } /* End */
 };
 
+void ghes_edac_release(struct device *dev)
+{
+	struct ghes_edac_pvt *ghes_pvt;
+	struct mem_ctl_info *mci;
+
+	ghes_pvt = container_of(dev, struct ghes_edac_pvt, dev);
+
+	mci = ghes_pvt->mci;
+	edac_mc_del_mc(mci->pdev);
+	edac_mc_free(mci);
+}
+
 static int
 ghes_edac_register_one(int nid, struct ghes *ghes, struct device *parent)
 {
+	struct device *dev;
+	struct ghes_edac_pvt *ghes_pvt;
 	int rc;
 	struct mem_ctl_info *mci;
 	struct edac_mc_layer layers[1];
 
 	layers[0].type = EDAC_MC_LAYER_ALL_MEM;
-	layers[0].size = mem_info.num_dimm;
+	layers[0].size = mem_info.num_per_node[nid];
 	layers[0].is_virt_csrow = true;
 
 	mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers,
@@ -667,43 +707,69 @@ ghes_edac_register_one(int nid, struct ghes *ghes, struct device *parent)
 	ghes_pvt->ghes	= ghes;
 	ghes_pvt->mci	= mci;
 
-	mci->pdev = parent;
+	dev		= &ghes_pvt->dev;
+	dev->parent	= parent;
+	dev->release	= ghes_edac_release;
+	dev_set_name(dev, "ghes_mc%d", nid);
+
+	rc = device_register(dev);
+	if (rc) {
+		pr_err("Can't create EDAC device (%d)\n", rc);
+		goto fail;
+	}
+
+	mci->pdev = dev;
 	mci->mtype_cap = MEM_FLAG_EMPTY;
 	mci->edac_ctl_cap = EDAC_FLAG_NONE;
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "ghes_edac.c";
-	mci->ctl_name = "ghes_edac";
-	mci->dev_name = "ghes";
+	mci->ctl_name = "ghes_mc";
+	mci->dev_name = dev_name(dev);
 
 	mci_add_dimm_info(mci);
 
 	rc = edac_mc_add_mc(mci);
 	if (rc < 0) {
-		pr_err("Can't register at EDAC core\n");
-		edac_mc_free(mci);
-		return -ENODEV;
+		pr_err("Can't register at EDAC core (%d)\n", rc);
+		goto fail;
 	}
+
 	return 0;
+fail:
+	put_device(dev);
+	return rc;
+}
+
+static void ghes_edac_unregister_one(struct mem_ctl_info *mci)
+{
+	struct ghes_edac_pvt *pvt = mci->pvt_info;
+
+	put_device(&pvt->dev);
 }
 
 void ghes_edac_unregister(struct ghes *ghes)
 {
 	struct mem_ctl_info *mci;
+	int nid;
 
-	if (!ghes_pvt)
-		return;
-
-	mci = ghes_pvt->mci;
-	edac_mc_del_mc(mci->pdev);
-	edac_mc_free(mci);
+	for_each_node(nid) {
+		mci = edac_mc_find(nid);
+		/* stop fallback at last */
+		if (mci && mci != fallback)
+			ghes_edac_unregister_one(mci);
+	}
 
+	ghes_edac_unregister_one(fallback);
+	fallback = NULL;
 	kfree(mem_info.dimms);
+	atomic_dec(&ghes_init);
 }
 
 int ghes_edac_register(struct ghes *ghes, struct device *dev)
 {
 	bool fake = false;
 	int rc;
+	int nid;
 	int idx = -1;
 
 	if (IS_ENABLED(CONFIG_X86)) {
@@ -743,7 +809,23 @@ int ghes_edac_register(struct ghes *ghes, struct device *dev)
 		pr_info("This system has %d DIMM sockets.\n", mem_info.num_dimm);
 	}
 
-	rc = ghes_edac_register_one(0, ghes, dev);
+	for_each_node(nid) {
+		if (!mem_info.num_per_node[nid])
+			continue;
 
-	return rc;
+		rc = ghes_edac_register_one(nid, ghes, dev);
+		if (rc) {
+			ghes_edac_unregister(ghes);
+			return rc;
+		}
+
+		/*
+		 * use the first node's mc as fallback in case we can
+		 * not detect the node from the error information
+		 */
+		if (!fallback)
+			fallback = edac_mc_find(nid);
+	}
+
+	return 0;
 }
-- 
2.20.1


  parent reply index

Thread overview: 43+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-05-29  8:44 [PATCH 00/21] EDAC, mc, ghes: Fixes and updates to improve memory error reporting Robert Richter
2019-05-29  8:44 ` [PATCH 01/21] EDAC, mc: Fix edac_mc_find() in case no device is found Robert Richter
2019-05-29  8:44 ` [PATCH 02/21] EDAC: Fixes to use put_device() after device_add() errors Robert Richter
2019-06-11 17:28   ` Borislav Petkov
2019-06-12 17:17     ` Robert Richter
2019-05-29  8:44 ` [PATCH 03/21] EDAC: Kill EDAC_DIMM_PTR() macro Robert Richter
2019-05-29  8:44 ` [PATCH 04/21] EDAC: Kill EDAC_DIMM_OFF() macro Robert Richter
2019-05-29  8:44 ` [PATCH 05/21] EDAC: Introduce mci_for_each_dimm() iterator Robert Richter
2019-05-29  8:44 ` [PATCH 06/21] EDAC, mc: Cleanup _edac_mc_free() code Robert Richter
2019-05-29  8:44 ` [PATCH 07/21] EDAC, mc: Remove per layer counters Robert Richter
2019-05-29  8:44 ` [PATCH 08/21] EDAC, mc: Rework edac_raw_mc_handle_error() to use struct dimm_info Robert Richter
2019-05-29  8:44 ` [PATCH 09/21] EDAC, ghes: Use standard kernel macros for page calculations Robert Richter
2019-05-29 15:13   ` James Morse
2019-05-29  8:44 ` [PATCH 10/21] EDAC, ghes: Remove pvt->detail_location string Robert Richter
2019-05-29 15:13   ` James Morse
2019-06-12 18:13     ` Robert Richter
2019-05-29  8:44 ` [PATCH 11/21] EDAC, ghes: Unify trace_mc_event() code with edac_mc driver Robert Richter
2019-05-29 15:12   ` James Morse
2019-06-03 13:10     ` Robert Richter
2019-06-04 17:15       ` James Morse
2019-06-13 22:23         ` Robert Richter
2019-05-29  8:44 ` [PATCH 12/21] EDAC, ghes: Add support for legacy API counters Robert Richter
2019-05-29 15:13   ` James Morse
2019-06-12 18:41     ` Robert Richter
2019-06-19 17:22       ` James Morse
2019-06-20  6:55         ` Robert Richter
2019-06-26  9:33           ` James Morse
2019-06-26 10:27             ` Robert Richter
2019-05-29  8:44 ` [PATCH 13/21] EDAC, ghes: Rework memory hierarchy detection Robert Richter
2019-05-29 15:06   ` James Morse
2019-05-31 13:41     ` Robert Richter
2019-05-29  8:44 ` [PATCH 14/21] EDAC, ghes: Extract numa node information for each dimm Robert Richter
2019-05-29 17:51   ` James Morse
2019-06-13 20:52     ` Robert Richter
2019-05-29  8:44 ` [PATCH 15/21] EDAC, ghes: Moving code around ghes_edac_register() Robert Richter
2019-05-29  8:44 ` Robert Richter [this message]
2019-05-29  8:44 ` [PATCH 17/21] EDAC, ghes: Fill sysfs with the DMI DIMM label information Robert Richter
2019-05-29  8:44 ` [PATCH 18/21] EDAC, mc: Introduce edac_mc_alloc_by_dimm() for per dimm allocation Robert Richter
2019-05-29  8:44 ` [PATCH 19/21] EDAC, ghes: Identify dimm by node, card, module and handle Robert Richter
2019-05-29  8:44 ` [PATCH 20/21] EDAC, ghes: Enable per-layer reporting based on card/module Robert Richter
2019-05-29  8:44 ` [PATCH 21/21] EDAC, Documentation: Describe CPER module definition and DIMM ranks Robert Richter
2019-05-29 14:54 ` [PATCH 00/21] EDAC, mc, ghes: Fixes and updates to improve memory error reporting Borislav Petkov
2019-05-31 14:48   ` Robert Richter

Reply instructions:

You may reply publically to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20190529084344.28562-17-rrichter@marvell.com \
    --to=rrichter@marvell.com \
    --cc=bp@alien8.de \
    --cc=james.morse@arm.com \
    --cc=linux-edac@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mchehab@kernel.org \
    --cc=tony.luck@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Linux-EDAC Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/linux-edac/0 linux-edac/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 linux-edac linux-edac/ https://lore.kernel.org/linux-edac \
		linux-edac@vger.kernel.org
	public-inbox-index linux-edac

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-edac


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git