Linux-EDAC Archive on lore.kernel.org
 help / color / Atom feed
From: Robert Richter <rrichter@marvell.com>
To: Borislav Petkov <bp@alien8.de>,
	Mauro Carvalho Chehab <mchehab@kernel.org>,
	Tony Luck <tony.luck@intel.com>
Cc: James Morse <james.morse@arm.com>,
	Aristeu Rozanski <aris@redhat.com>,
	Robert Richter <rrichter@marvell.com>,
	<linux-edac@vger.kernel.org>, <linux-kernel@vger.kernel.org>
Subject: [PATCH 09/11] EDAC/ghes: Implement DIMM mapping table for SMBIOS handles
Date: Fri,  6 Mar 2020 16:13:16 +0100
Message-ID: <20200306151318.17422-10-rrichter@marvell.com> (raw)
In-Reply-To: <20200306151318.17422-1-rrichter@marvell.com>

GHES uses SMBIOS handles to identify the DIMM that was causing a hw
error. Currently this is stored in smbios_handle of struct dimm_info.
This implementation has several drawbacks. The information is private
to the ghes driver, but struct dimm_info is for general use. The DIMMs
are tied to a *mci struct, which makes a lockup inefficient. It is
hard to dynamically allocate DIMMs and also to meet locking
constraints when adding or removing them. This becomes even more
challenging when having multiple MC instances that group a set of
DIMMs, so the change is needed also to later support multiple MC
instances.

Add an own mapping table for the ghes driver to solve these issues.
The DIMM data is collected in struct ghes_dimm. A data array exists
for all DIMMs found in the system. Lists are used to maintain the DIMM
entries, during initialization they are all added to a pool of
available data structs. Using list entries makes locking easy as only
list operations need to be locked by either the ghes_reg_mutex or
ghes_lock.

There are a couple of helper functions to maintain the dimm list:

 ghes_dimm_init()/ghes_dimm_free():       create/remove the list
 ghes_dimm_alloc()/ghes_dimm_release():   add/remove list entries
 get_default_mci()/find_dimm_by_handle(): lockup a MC or DIMM

The lists are used to identify the corresponding *mci and *dimm_info
structs in the hw error interrupt handler. Esp. the *ghes_pvt pointer
is obsolete now, so *mci can be removed from struct ghes_mci.

The mapping table of the ghes driver makes smbios_handle in struct
dimm_info also obsolete, remove it too.

Signed-off-by: Robert Richter <rrichter@marvell.com>
---
 drivers/edac/ghes_edac.c | 172 +++++++++++++++++++++++++++++++--------
 include/linux/edac.h     |   2 -
 2 files changed, 136 insertions(+), 38 deletions(-)

diff --git a/drivers/edac/ghes_edac.c b/drivers/edac/ghes_edac.c
index 0af7f6a988a9..cd61b8ae49f6 100644
--- a/drivers/edac/ghes_edac.c
+++ b/drivers/edac/ghes_edac.c
@@ -15,9 +15,13 @@
 #include "edac_module.h"
 #include <ras/ras_event.h>
 
-struct ghes_mci {
-	struct mem_ctl_info *mci;
+struct ghes_dimm {
+	struct list_head entry;
+	struct dimm_info *dimm;
+	u16 handle;
+};
 
+struct ghes_mci {
 	/* Buffers for the error handling routine */
 	char other_detail[400];
 	char msg[80];
@@ -26,22 +30,25 @@ struct ghes_mci {
 static refcount_t ghes_refcount = REFCOUNT_INIT(0);
 
 /*
- * Access to ghes_pvt must be protected by ghes_lock. The spinlock
- * also provides the necessary (implicit) memory barrier for the SMP
- * case to make the pointer visible on another CPU.
+ * GHES registration mutex: Serialize registration/unregistration
+ * code, protect ghes_dimm_pool and *dimms.
+ *
  */
-static struct ghes_mci *ghes_pvt;
-
-/* GHES registration mutex */
 static DEFINE_MUTEX(ghes_reg_mutex);
 
 /*
  * Sync with other, potentially concurrent callers of
  * ghes_edac_report_mem_error(). We don't know what the
  * "inventive" firmware would do.
+ *
+ * ghes_dimm_list must be protected by ghes_lock.
  */
 static DEFINE_SPINLOCK(ghes_lock);
 
+static struct ghes_dimm *dimms;
+static LIST_HEAD(ghes_dimm_list);
+static LIST_HEAD(ghes_dimm_pool);
+
 /* "ghes_edac.force_load=1" skips the platform check */
 static bool __read_mostly force_load;
 module_param(force_load, bool, 0);
@@ -72,6 +79,52 @@ struct memdev_dmi_entry {
 	u16 conf_mem_clk_speed;
 } __attribute__((__packed__));
 
+/* ghes_reg_mutex must be held. */
+static int ghes_dimm_init(int num_dimm)
+{
+	struct ghes_dimm *ghes_dimm;
+
+	dimms = kcalloc(num_dimm, sizeof(*dimms), GFP_KERNEL);
+	if (!dimms)
+		return -ENOMEM;
+
+	for (ghes_dimm = dimms; ghes_dimm < dimms + num_dimm; ghes_dimm++)
+		list_add(&ghes_dimm->entry, &ghes_dimm_pool);
+
+	return 0;
+}
+
+/* ghes_reg_mutex must be held. */
+static void ghes_dimm_free(void)
+{
+	INIT_LIST_HEAD(&ghes_dimm_pool);
+	kfree(dimms);
+}
+
+/* ghes_reg_mutex must be held. */
+static struct ghes_dimm *ghes_dimm_alloc(struct dimm_info *dimm, u16 handle)
+{
+	struct ghes_dimm *ghes_dimm;
+
+	ghes_dimm = list_first_entry_or_null(&ghes_dimm_pool,
+					struct ghes_dimm, entry);
+
+	/* should be always non-zero */
+	if (!WARN_ON_ONCE(!ghes_dimm)) {
+		ghes_dimm->dimm = dimm;
+		ghes_dimm->handle = handle;
+		list_del(&ghes_dimm->entry);
+	}
+
+	return ghes_dimm;
+}
+
+/* ghes_reg_mutex must be held. */
+static void ghes_dimm_release(struct list_head *dimms)
+{
+	list_splice(dimms, &ghes_dimm_pool);
+}
+
 static void ghes_edac_count_dimms(const struct dmi_header *dh, void *arg)
 {
 	int *num_dimm = arg;
@@ -80,14 +133,30 @@ static void ghes_edac_count_dimms(const struct dmi_header *dh, void *arg)
 		(*num_dimm)++;
 }
 
-static struct dimm_info *find_dimm_by_handle(struct mem_ctl_info *mci,
-					u16 handle)
+/*
+ * Make first DIMM's MC the default to report errors in case
+ * no DIMM for the mem device handle is found.
+ *
+ * ghes_lock must be held.
+ */
+static struct mem_ctl_info *get_default_mci(void)
 {
-	struct dimm_info *dimm;
+	struct ghes_dimm *ghes_dimm;
+
+	ghes_dimm = list_first_entry_or_null(&ghes_dimm_list,
+					struct ghes_dimm, entry);
+
+	return ghes_dimm ? ghes_dimm->dimm->mci : NULL;
+}
+
+/* ghes_lock must be held as long as struct ghes_info is used */
+static struct dimm_info *find_dimm_by_handle(u16 handle)
+{
+	struct ghes_dimm *ghes_dimm;
 
-	mci_for_each_dimm(mci, dimm) {
-		if (dimm->smbios_handle == handle)
-			return dimm;
+	list_for_each_entry(ghes_dimm, &ghes_dimm_list, entry) {
+		if (ghes_dimm->handle == handle)
+			return ghes_dimm->dimm;
 	}
 
 	return NULL;
@@ -109,6 +178,7 @@ static void ghes_dimm_setup_label(struct dimm_info *dimm, u16 handle)
 }
 
 struct ghes_dimm_fill {
+	struct list_head dimms;
 	struct mem_ctl_info *mci;
 	int index;
 };
@@ -121,6 +191,11 @@ static void ghes_edac_dmidecode(const struct dmi_header *dh, void *arg)
 	if (dh->type == DMI_ENTRY_MEM_DEVICE) {
 		struct memdev_dmi_entry *entry = (struct memdev_dmi_entry *)dh;
 		struct dimm_info *dimm = edac_get_dimm(mci, dimm_fill->index, 0, 0);
+		struct ghes_dimm *ghes_dimm;
+
+		ghes_dimm = ghes_dimm_alloc(dimm, entry->handle);
+		if (ghes_dimm)
+			list_add_tail(&ghes_dimm->entry, &dimm_fill->dimms);
 
 		if (entry->size == 0xffff) {
 			pr_info("Can't get DIMM%i size\n",
@@ -205,8 +280,6 @@ static void ghes_edac_dmidecode(const struct dmi_header *dh, void *arg)
 				entry->total_width, entry->data_width);
 		}
 
-		dimm->smbios_handle = entry->handle;
-
 		dimm_fill->index++;
 	}
 }
@@ -214,6 +287,7 @@ static void ghes_edac_dmidecode(const struct dmi_header *dh, void *arg)
 void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 {
 	struct edac_raw_error_desc *e;
+	struct dimm_info *dimm = NULL;
 	struct mem_ctl_info *mci;
 	struct ghes_mci *pvt;
 	unsigned long flags;
@@ -229,11 +303,14 @@ void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 
 	spin_lock_irqsave(&ghes_lock, flags);
 
-	pvt = ghes_pvt;
-	if (!pvt)
+	if (mem_err->validation_bits & CPER_MEM_VALID_MODULE_HANDLE)
+		dimm = find_dimm_by_handle(mem_err->mem_dev_handle);
+
+	mci = dimm ? dimm->mci : get_default_mci();
+	if (!mci)
 		goto unlock;
 
-	mci = pvt->mci;
+	pvt = mci->pvt_info;
 	e = &mci->error_desc;
 
 	/* Cleans the error report buffer */
@@ -356,9 +433,6 @@ void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 	if (mem_err->validation_bits & CPER_MEM_VALID_BIT_POSITION)
 		p += sprintf(p, "bit_pos:%d ", mem_err->bit_pos);
 	if (mem_err->validation_bits & CPER_MEM_VALID_MODULE_HANDLE) {
-		struct dimm_info *dimm;
-
-		dimm = find_dimm_by_handle(mci, mem_err->mem_dev_handle);
 		if (dimm) {
 			e->top_layer = dimm->idx;
 			strcpy(e->label, dimm->label);
@@ -477,8 +551,7 @@ static struct mem_ctl_info *ghes_mc_create(struct device *dev, int mc_idx,
 	if (!mci)
 		return NULL;
 
-	pvt		= mci->pvt_info;
-	pvt->mci	= mci;
+	pvt = mci->pvt_info;
 
 	mci->pdev = dev;
 	mci->mtype_cap = MEM_FLAG_EMPTY;
@@ -491,19 +564,21 @@ static struct mem_ctl_info *ghes_mc_create(struct device *dev, int mc_idx,
 	return mci;
 }
 
-static int ghes_mc_add_or_free(struct mem_ctl_info *mci)
+static int ghes_mc_add_or_free(struct mem_ctl_info *mci,
+			struct list_head *dimms)
 {
 	unsigned long flags;
 	int rc;
 
 	rc = edac_mc_add_mc(mci);
 	if (rc < 0) {
+		ghes_dimm_release(dimms);
 		edac_mc_free(mci);
 		return rc;
 	}
 
 	spin_lock_irqsave(&ghes_lock, flags);
-	ghes_pvt = mci->pvt_info;
+	list_splice_tail(dimms, &ghes_dimm_list);
 	spin_unlock_irqrestore(&ghes_lock, flags);
 
 	return 0;
@@ -511,17 +586,25 @@ static int ghes_mc_add_or_free(struct mem_ctl_info *mci)
 
 static void ghes_mc_free(void)
 {
-	struct mem_ctl_info *mci;
+	struct ghes_dimm *ghes_dimm, *tmp;
+	struct mem_ctl_info *mci = NULL;
+	LIST_HEAD(dimm_list);
 	unsigned long flags;
 
-	/*
-	 * Wait for the irq handler being finished.
-	 */
 	spin_lock_irqsave(&ghes_lock, flags);
-	mci = ghes_pvt ? ghes_pvt->mci : NULL;
-	ghes_pvt = NULL;
+
+	list_for_each_entry_safe(ghes_dimm, tmp, &ghes_dimm_list, entry) {
+		mci = mci ?: ghes_dimm->dimm->mci;
+		WARN_ON_ONCE(mci != ghes_dimm->dimm->mci);
+		list_move_tail(&ghes_dimm->entry, &dimm_list);
+	}
+
+	WARN_ON_ONCE(!list_empty(&ghes_dimm_list));
+
 	spin_unlock_irqrestore(&ghes_lock, flags);
 
+	ghes_dimm_release(&dimm_list);
+
 	if (!mci)
 		return;
 
@@ -532,8 +615,10 @@ static void ghes_mc_free(void)
 
 static int ghes_edac_register_fake(struct device *dev)
 {
+	struct ghes_dimm *ghes_dimm;
 	struct mem_ctl_info *mci;
 	struct dimm_info *dimm;
+	LIST_HEAD(dimm_list);
 
 	mci = ghes_mc_create(dev, 0, 1);
 	if (!mci)
@@ -549,7 +634,11 @@ static int ghes_edac_register_fake(struct device *dev)
 
 	snprintf(dimm->label, sizeof(dimm->label), "unknown memory");
 
-	return ghes_mc_add_or_free(mci);
+	ghes_dimm = ghes_dimm_alloc(dimm, 0xffff);
+	if (ghes_dimm)
+		list_add_tail(&ghes_dimm->entry, &dimm_list);
+
+	return ghes_mc_add_or_free(mci, &dimm_list);
 }
 
 static int ghes_edac_register_one(struct device *dev, int mc_idx, int num_dimm)
@@ -563,9 +652,11 @@ static int ghes_edac_register_one(struct device *dev, int mc_idx, int num_dimm)
 
 	dimm_fill.index = 0;
 	dimm_fill.mci = mci;
+	INIT_LIST_HEAD(&dimm_fill.dimms);
+
 	dmi_walk(ghes_edac_dmidecode, &dimm_fill);
 
-	return ghes_mc_add_or_free(mci);
+	return ghes_mc_add_or_free(mci, &dimm_fill.dimms);
 }
 
 int ghes_edac_register(struct ghes *ghes, struct device *dev)
@@ -594,6 +685,10 @@ int ghes_edac_register(struct ghes *ghes, struct device *dev)
 	/* Get the number of DIMMs */
 	dmi_walk(ghes_edac_count_dimms, &num_dimm);
 
+	rc = ghes_dimm_init(num_dimm ?: 1);
+	if (rc)
+		goto unlock;
+
 	if (!num_dimm) {
 		/*
 		 * Bogus BIOS: Ignore DMI topology and use a single MC
@@ -629,8 +724,11 @@ int ghes_edac_register(struct ghes *ghes, struct device *dev)
 	refcount_set(&ghes_refcount, 1);
 
 unlock:
-	if (rc)
+	if (rc) {
+		ghes_mc_free();
+		ghes_dimm_free();
 		pr_err("Can't register at EDAC core: %d\n", rc);
+	}
 
 	mutex_unlock(&ghes_reg_mutex);
 
@@ -641,8 +739,10 @@ void ghes_edac_unregister(struct ghes *ghes)
 {
 	mutex_lock(&ghes_reg_mutex);
 
-	if (refcount_dec_and_test(&ghes_refcount))
+	if (refcount_dec_and_test(&ghes_refcount)) {
 		ghes_mc_free();
+		ghes_dimm_free();
+	}
 
 	mutex_unlock(&ghes_reg_mutex);
 }
diff --git a/include/linux/edac.h b/include/linux/edac.h
index 0f20b986b0ab..6b7f594782c0 100644
--- a/include/linux/edac.h
+++ b/include/linux/edac.h
@@ -382,8 +382,6 @@ struct dimm_info {
 
 	unsigned int csrow, cschannel;	/* Points to the old API data */
 
-	u16 smbios_handle;              /* Handle for SMBIOS type 17 */
-
 	u32 ce_count;
 	u32 ue_count;
 };
-- 
2.20.1


  parent reply index

Thread overview: 27+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-03-06 15:13 [PATCH 00/11] EDAC/ghes: Cleanup, rework and improvement of memory reporting Robert Richter
2020-03-06 15:13 ` [PATCH 01/11] EDAC/mc: Use int type for parameters of edac_mc_alloc() Robert Richter
2020-04-06 11:38   ` Matthias Brugger
2020-03-06 15:13 ` [PATCH 02/11] EDAC/ghes: Setup DIMM label from DMI and use it in error reports Robert Richter
2020-03-06 15:13 ` [PATCH 03/11] EDAC/ghes: Remove local variable rdr_mask in ghes_edac_dmidecode() Robert Richter
2020-04-06 11:51   ` Matthias Brugger
2020-03-06 15:13 ` [PATCH 04/11] EDAC/ghes: Remove unused members of struct ghes_edac_pvt, rename it to ghes_mci Robert Richter
2020-03-06 15:13 ` [PATCH 05/11] EDAC/ghes: Cleanup struct ghes_edac_dimm_fill, rename it to ghes_dimm_fill Robert Richter
2020-03-16  9:29   ` Borislav Petkov
2020-03-06 15:13 ` [PATCH 06/11] EDAC/ghes: Carve out MC device handling into separate functions Robert Richter
2020-03-16  9:31   ` Borislav Petkov
2020-03-16 12:12     ` Robert Richter
2020-03-06 15:13 ` [PATCH 07/11] EDAC/ghes: Have a separate code path for creating the fake MC Robert Richter
2020-03-06 15:13 ` [PATCH 08/11] EDAC/ghes: Carve out code into ghes_edac_register_{one,fake}() Robert Richter
2020-03-06 15:13 ` Robert Richter [this message]
2020-03-16  9:40   ` [PATCH 09/11] EDAC/ghes: Implement DIMM mapping table for SMBIOS handles Borislav Petkov
2020-03-06 15:13 ` [PATCH 10/11] EDAC/ghes: Create an own device for each mci Robert Richter
2020-03-16  9:45   ` Borislav Petkov
2020-03-06 15:13 ` [PATCH 11/11] EDAC/ghes: Create one memory controller per physical memory array Robert Richter
2020-03-16  9:51   ` Borislav Petkov
2020-03-17 16:34     ` John Garry
2020-03-17 22:14       ` Kani, Toshi
2020-03-17 22:50         ` Borislav Petkov
2020-03-17 22:53           ` Kani, Toshi
2020-03-18  0:10             ` Robert Richter
2020-03-24 11:32       ` Xiaofei Tan
2020-03-10 20:18 ` [PATCH 00/11] EDAC/ghes: Cleanup, rework and improvement of memory reporting Aristeu Rozanski

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200306151318.17422-10-rrichter@marvell.com \
    --to=rrichter@marvell.com \
    --cc=aris@redhat.com \
    --cc=bp@alien8.de \
    --cc=james.morse@arm.com \
    --cc=linux-edac@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mchehab@kernel.org \
    --cc=tony.luck@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Linux-EDAC Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/linux-edac/0 linux-edac/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 linux-edac linux-edac/ https://lore.kernel.org/linux-edac \
		linux-edac@vger.kernel.org
	public-inbox-index linux-edac

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-edac


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git