[PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

[PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

[Laurent Dufour]

All the LMB from the same set of ibm,dynamic-memory-v2 property are
sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>
---
 arch/powerpc/mm/drmem.c | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index b2eeea39684c..3819c523c65b 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c
@@ -397,7 +397,7 @@ static void __init init_drmem_v1_lmbs(const __be32 *prop)
 
 static void __init init_drmem_v2_lmbs(const __be32 *prop)
 {
-	struct drmem_lmb *lmb;
+	struct drmem_lmb *lmb, *first;
 	struct of_drconf_cell_v2 dr_cell;
 	const __be32 *p;
 	u32 i, j, lmb_sets;
@@ -422,10 +422,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	/* second pass, read in the LMB information */
 	lmb_index = 0;
 	p = prop;
+	first = NULL;
 
 	for (i = 0; i < lmb_sets; i++) {
 		read_drconf_v2_cell(&dr_cell, &p);
 
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+
 		for (j = 0; j < dr_cell.seq_lmbs; j++) {
 			lmb = &drmem_info->lmbs[lmb_index++];
 
@@ -438,7 +446,16 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 			lmb->aa_index = dr_cell.aa_index;
 			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
 		}
 	}
 }
-- 
2.28.0

[PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

[Laurent Dufour]

All the LMB from the same set of ibm,dynamic-memory-v2 property are
sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>
---
 arch/powerpc/mm/drmem.c | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index b2eeea39684c..3819c523c65b 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c
@@ -397,7 +397,7 @@ static void __init init_drmem_v1_lmbs(const __be32 *prop)
 
 static void __init init_drmem_v2_lmbs(const __be32 *prop)
 {
-	struct drmem_lmb *lmb;
+	struct drmem_lmb *lmb, *first;
 	struct of_drconf_cell_v2 dr_cell;
 	const __be32 *p;
 	u32 i, j, lmb_sets;
@@ -422,10 +422,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	/* second pass, read in the LMB information */
 	lmb_index = 0;
 	p = prop;
+	first = NULL;
 
 	for (i = 0; i < lmb_sets; i++) {
 		read_drconf_v2_cell(&dr_cell, &p);
 
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+
 		for (j = 0; j < dr_cell.seq_lmbs; j++) {
 			lmb = &drmem_info->lmbs[lmb_index++];
 
@@ -438,7 +446,16 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 			lmb->aa_index = dr_cell.aa_index;
 			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
 		}
 	}
 }
-- 
2.28.0

Re: [PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 3560 bytes --]

Hi Laurent,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on linux/master linus/master v5.8 next-20200804]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-mpc885_ads_defconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':
>> arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |        ^~
   arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |                     ^~

vim +457 arch/powerpc/mm/drmem.c

   397	
   398	static void __init init_drmem_v2_lmbs(const __be32 *prop)
   399	{
   400		struct drmem_lmb *lmb, *first;
   401		struct of_drconf_cell_v2 dr_cell;
   402		const __be32 *p;
   403		u32 i, j, lmb_sets;
   404		int lmb_index;
   405	
   406		lmb_sets = of_read_number(prop++, 1);
   407		if (lmb_sets == 0)
   408			return;
   409	
   410		/* first pass, calculate the number of LMBs */
   411		p = prop;
   412		for (i = 0; i < lmb_sets; i++) {
   413			read_drconf_v2_cell(&dr_cell, &p);
   414			drmem_info->n_lmbs += dr_cell.seq_lmbs;
   415		}
   416	
   417		drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
   418					   GFP_KERNEL);
   419		if (!drmem_info->lmbs)
   420			return;
   421	
   422		/* second pass, read in the LMB information */
   423		lmb_index = 0;
   424		p = prop;
   425		first = NULL;
   426	
   427		for (i = 0; i < lmb_sets; i++) {
   428			read_drconf_v2_cell(&dr_cell, &p);
   429	
   430			/*
   431			 * Fetch the NUMA node id for the fist set or if the
   432			 * associativity index is different from the previous set.
   433			 */
   434			if (first && dr_cell.aa_index != first->aa_index)
   435				first = NULL;
   436	
   437			for (j = 0; j < dr_cell.seq_lmbs; j++) {
   438				lmb = &drmem_info->lmbs[lmb_index++];
   439	
   440				lmb->base_addr = dr_cell.base_addr;
   441				dr_cell.base_addr += drmem_info->lmb_size;
   442	
   443				lmb->drc_index = dr_cell.drc_index;
   444				dr_cell.drc_index++;
   445	
   446				lmb->aa_index = dr_cell.aa_index;
   447				lmb->flags = dr_cell.flags;
   448	
   449				/*
   450				 * All the LMB in the set share the same NUMA
   451				 * associativity property. So read that node only once.
   452				 */
   453				if (!first) {
   454					lmb_set_nid(lmb);
   455					first = lmb;
   456				} else {
 > 457					lmb->nid = first->nid;
   458				}
   459			}
   460		}
   461	}
   462	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 10853 bytes --]

Re: [PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 3560 bytes --]

Hi Laurent,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on linux/master linus/master v5.8 next-20200804]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-mpc885_ads_defconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':
>> arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |        ^~
   arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |                     ^~

vim +457 arch/powerpc/mm/drmem.c

   397	
   398	static void __init init_drmem_v2_lmbs(const __be32 *prop)
   399	{
   400		struct drmem_lmb *lmb, *first;
   401		struct of_drconf_cell_v2 dr_cell;
   402		const __be32 *p;
   403		u32 i, j, lmb_sets;
   404		int lmb_index;
   405	
   406		lmb_sets = of_read_number(prop++, 1);
   407		if (lmb_sets == 0)
   408			return;
   409	
   410		/* first pass, calculate the number of LMBs */
   411		p = prop;
   412		for (i = 0; i < lmb_sets; i++) {
   413			read_drconf_v2_cell(&dr_cell, &p);
   414			drmem_info->n_lmbs += dr_cell.seq_lmbs;
   415		}
   416	
   417		drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
   418					   GFP_KERNEL);
   419		if (!drmem_info->lmbs)
   420			return;
   421	
   422		/* second pass, read in the LMB information */
   423		lmb_index = 0;
   424		p = prop;
   425		first = NULL;
   426	
   427		for (i = 0; i < lmb_sets; i++) {
   428			read_drconf_v2_cell(&dr_cell, &p);
   429	
   430			/*
   431			 * Fetch the NUMA node id for the fist set or if the
   432			 * associativity index is different from the previous set.
   433			 */
   434			if (first && dr_cell.aa_index != first->aa_index)
   435				first = NULL;
   436	
   437			for (j = 0; j < dr_cell.seq_lmbs; j++) {
   438				lmb = &drmem_info->lmbs[lmb_index++];
   439	
   440				lmb->base_addr = dr_cell.base_addr;
   441				dr_cell.base_addr += drmem_info->lmb_size;
   442	
   443				lmb->drc_index = dr_cell.drc_index;
   444				dr_cell.drc_index++;
   445	
   446				lmb->aa_index = dr_cell.aa_index;
   447				lmb->flags = dr_cell.flags;
   448	
   449				/*
   450				 * All the LMB in the set share the same NUMA
   451				 * associativity property. So read that node only once.
   452				 */
   453				if (!first) {
   454					lmb_set_nid(lmb);
   455					first = lmb;
   456				} else {
 > 457					lmb->nid = first->nid;
   458				}
   459			}
   460		}
   461	}
   462	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 10853 bytes --]

Re: [PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 3667 bytes --]

Hi Laurent,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on linux/master linus/master v5.8 next-20200804]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-mpc885_ads_defconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':
>> arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |        ^~
   arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |                     ^~

vim +457 arch/powerpc/mm/drmem.c

   397	
   398	static void __init init_drmem_v2_lmbs(const __be32 *prop)
   399	{
   400		struct drmem_lmb *lmb, *first;
   401		struct of_drconf_cell_v2 dr_cell;
   402		const __be32 *p;
   403		u32 i, j, lmb_sets;
   404		int lmb_index;
   405	
   406		lmb_sets = of_read_number(prop++, 1);
   407		if (lmb_sets == 0)
   408			return;
   409	
   410		/* first pass, calculate the number of LMBs */
   411		p = prop;
   412		for (i = 0; i < lmb_sets; i++) {
   413			read_drconf_v2_cell(&dr_cell, &p);
   414			drmem_info->n_lmbs += dr_cell.seq_lmbs;
   415		}
   416	
   417		drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
   418					   GFP_KERNEL);
   419		if (!drmem_info->lmbs)
   420			return;
   421	
   422		/* second pass, read in the LMB information */
   423		lmb_index = 0;
   424		p = prop;
   425		first = NULL;
   426	
   427		for (i = 0; i < lmb_sets; i++) {
   428			read_drconf_v2_cell(&dr_cell, &p);
   429	
   430			/*
   431			 * Fetch the NUMA node id for the fist set or if the
   432			 * associativity index is different from the previous set.
   433			 */
   434			if (first && dr_cell.aa_index != first->aa_index)
   435				first = NULL;
   436	
   437			for (j = 0; j < dr_cell.seq_lmbs; j++) {
   438				lmb = &drmem_info->lmbs[lmb_index++];
   439	
   440				lmb->base_addr = dr_cell.base_addr;
   441				dr_cell.base_addr += drmem_info->lmb_size;
   442	
   443				lmb->drc_index = dr_cell.drc_index;
   444				dr_cell.drc_index++;
   445	
   446				lmb->aa_index = dr_cell.aa_index;
   447				lmb->flags = dr_cell.flags;
   448	
   449				/*
   450				 * All the LMB in the set share the same NUMA
   451				 * associativity property. So read that node only once.
   452				 */
   453				if (!first) {
   454					lmb_set_nid(lmb);
   455					first = lmb;
   456				} else {
 > 457					lmb->nid = first->nid;
   458				}
   459			}
   460		}
   461	}
   462	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 10853 bytes --]

Re: [PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

[Laurent Dufour]

Le 05/08/2020 à 12:43, kernel test robot a écrit :
> Hi Laurent,
> 
> Thank you for the patch! Yet something to improve:
> 
> [auto build test ERROR on powerpc/next]
> [also build test ERROR on linux/master linus/master v5.8 next-20200804]
> [If your patch is applied to the wrong git tree, kindly drop us a note.
> And when submitting patch, we suggest to use '--base' as documented in
> https://git-scm.com/docs/git-format-patch]
> 
> url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
> base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
> config: powerpc-mpc885_ads_defconfig (attached as .config)
> compiler: powerpc-linux-gcc (GCC) 9.3.0
> reproduce (this is a W=1 build):
>          wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
>          chmod +x ~/bin/make.cross
>          # save the attached .config to linux build tree
>          COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc
> 
> If you fix the issue, kindly add following tag as appropriate
> Reported-by: kernel test robot <lkp@intel.com>
> 
> All errors (new ones prefixed by >>):
> 
>     arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':
>>> arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'
>       457 |     lmb->nid = first->nid;
>           |        ^~
>     arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
>       457 |     lmb->nid = first->nid;
>           |                     ^~

My mistake, the nid member is only present when CONFIG_MEMORY_HOTPLUG is set.

I'll send a new version fixing this.


> vim +457 arch/powerpc/mm/drmem.c
> 
>     397	
>     398	static void __init init_drmem_v2_lmbs(const __be32 *prop)
>     399	{
>     400		struct drmem_lmb *lmb, *first;
>     401		struct of_drconf_cell_v2 dr_cell;
>     402		const __be32 *p;
>     403		u32 i, j, lmb_sets;
>     404		int lmb_index;
>     405	
>     406		lmb_sets = of_read_number(prop++, 1);
>     407		if (lmb_sets == 0)
>     408			return;
>     409	
>     410		/* first pass, calculate the number of LMBs */
>     411		p = prop;
>     412		for (i = 0; i < lmb_sets; i++) {
>     413			read_drconf_v2_cell(&dr_cell, &p);
>     414			drmem_info->n_lmbs += dr_cell.seq_lmbs;
>     415		}
>     416	
>     417		drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
>     418					   GFP_KERNEL);
>     419		if (!drmem_info->lmbs)
>     420			return;
>     421	
>     422		/* second pass, read in the LMB information */
>     423		lmb_index = 0;
>     424		p = prop;
>     425		first = NULL;
>     426	
>     427		for (i = 0; i < lmb_sets; i++) {
>     428			read_drconf_v2_cell(&dr_cell, &p);
>     429	
>     430			/*
>     431			 * Fetch the NUMA node id for the fist set or if the
>     432			 * associativity index is different from the previous set.
>     433			 */
>     434			if (first && dr_cell.aa_index != first->aa_index)
>     435				first = NULL;
>     436	
>     437			for (j = 0; j < dr_cell.seq_lmbs; j++) {
>     438				lmb = &drmem_info->lmbs[lmb_index++];
>     439	
>     440				lmb->base_addr = dr_cell.base_addr;
>     441				dr_cell.base_addr += drmem_info->lmb_size;
>     442	
>     443				lmb->drc_index = dr_cell.drc_index;
>     444				dr_cell.drc_index++;
>     445	
>     446				lmb->aa_index = dr_cell.aa_index;
>     447				lmb->flags = dr_cell.flags;
>     448	
>     449				/*
>     450				 * All the LMB in the set share the same NUMA
>     451				 * associativity property. So read that node only once.
>     452				 */
>     453				if (!first) {
>     454					lmb_set_nid(lmb);
>     455					first = lmb;
>     456				} else {
>   > 457					lmb->nid = first->nid;
>     458				}
>     459			}
>     460		}
>     461	}
>     462	
> 
> ---
> 0-DAY CI Kernel Test Service, Intel Corporation
> https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
> 

Re: [PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

[Laurent Dufour]

Le 05/08/2020 à 12:43, kernel test robot a écrit :
> Hi Laurent,
> 
> Thank you for the patch! Yet something to improve:
> 
> [auto build test ERROR on powerpc/next]
> [also build test ERROR on linux/master linus/master v5.8 next-20200804]
> [If your patch is applied to the wrong git tree, kindly drop us a note.
> And when submitting patch, we suggest to use '--base' as documented in
> https://git-scm.com/docs/git-format-patch]
> 
> url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
> base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
> config: powerpc-mpc885_ads_defconfig (attached as .config)
> compiler: powerpc-linux-gcc (GCC) 9.3.0
> reproduce (this is a W=1 build):
>          wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
>          chmod +x ~/bin/make.cross
>          # save the attached .config to linux build tree
>          COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc
> 
> If you fix the issue, kindly add following tag as appropriate
> Reported-by: kernel test robot <lkp@intel.com>
> 
> All errors (new ones prefixed by >>):
> 
>     arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':
>>> arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'
>       457 |     lmb->nid = first->nid;
>           |        ^~
>     arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
>       457 |     lmb->nid = first->nid;
>           |                     ^~

My mistake, the nid member is only present when CONFIG_MEMORY_HOTPLUG is set.

I'll send a new version fixing this.


> vim +457 arch/powerpc/mm/drmem.c
> 
>     397	
>     398	static void __init init_drmem_v2_lmbs(const __be32 *prop)
>     399	{
>     400		struct drmem_lmb *lmb, *first;
>     401		struct of_drconf_cell_v2 dr_cell;
>     402		const __be32 *p;
>     403		u32 i, j, lmb_sets;
>     404		int lmb_index;
>     405	
>     406		lmb_sets = of_read_number(prop++, 1);
>     407		if (lmb_sets == 0)
>     408			return;
>     409	
>     410		/* first pass, calculate the number of LMBs */
>     411		p = prop;
>     412		for (i = 0; i < lmb_sets; i++) {
>     413			read_drconf_v2_cell(&dr_cell, &p);
>     414			drmem_info->n_lmbs += dr_cell.seq_lmbs;
>     415		}
>     416	
>     417		drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
>     418					   GFP_KERNEL);
>     419		if (!drmem_info->lmbs)
>     420			return;
>     421	
>     422		/* second pass, read in the LMB information */
>     423		lmb_index = 0;
>     424		p = prop;
>     425		first = NULL;
>     426	
>     427		for (i = 0; i < lmb_sets; i++) {
>     428			read_drconf_v2_cell(&dr_cell, &p);
>     429	
>     430			/*
>     431			 * Fetch the NUMA node id for the fist set or if the
>     432			 * associativity index is different from the previous set.
>     433			 */
>     434			if (first && dr_cell.aa_index != first->aa_index)
>     435				first = NULL;
>     436	
>     437			for (j = 0; j < dr_cell.seq_lmbs; j++) {
>     438				lmb = &drmem_info->lmbs[lmb_index++];
>     439	
>     440				lmb->base_addr = dr_cell.base_addr;
>     441				dr_cell.base_addr += drmem_info->lmb_size;
>     442	
>     443				lmb->drc_index = dr_cell.drc_index;
>     444				dr_cell.drc_index++;
>     445	
>     446				lmb->aa_index = dr_cell.aa_index;
>     447				lmb->flags = dr_cell.flags;
>     448	
>     449				/*
>     450				 * All the LMB in the set share the same NUMA
>     451				 * associativity property. So read that node only once.
>     452				 */
>     453				if (!first) {
>     454					lmb_set_nid(lmb);
>     455					first = lmb;
>     456				} else {
>   > 457					lmb->nid = first->nid;
>     458				}
>     459			}
>     460		}
>     461	}
>     462	
> 
> ---
> 0-DAY CI Kernel Test Service, Intel Corporation
> https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
> 

[PATCH v2] powerpc/drmem: Don't compute the NUMA node for each LMB

[Laurent Dufour]

All the LMB from the same set of ibm,dynamic-memory-v2 property are
sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid
field is not present in the drmem_lmb structure.

Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>
---
 arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index b2eeea39684c..c11b6ec99ea3 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c
@@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	const __be32 *p;
 	u32 i, j, lmb_sets;
 	int lmb_index;
+#ifdef CONFIG_MEMORY_HOTPLUG
+	struct drmem_lmb *first = NULL;
+#endif
 
 	lmb_sets = of_read_number(prop++, 1);
 	if (lmb_sets == 0)
@@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	for (i = 0; i < lmb_sets; i++) {
 		read_drconf_v2_cell(&dr_cell, &p);
 
+#ifdef CONFIG_MEMORY_HOTPLUG
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+#endif
+
 		for (j = 0; j < dr_cell.seq_lmbs; j++) {
 			lmb = &drmem_info->lmbs[lmb_index++];
 
@@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 			lmb->aa_index = dr_cell.aa_index;
 			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+#ifdef CONFIG_MEMORY_HOTPLUG
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
+#endif
 		}
 	}
 }
-- 
2.28.0

[PATCH v2] powerpc/drmem: Don't compute the NUMA node for each LMB

[Laurent Dufour]

All the LMB from the same set of ibm,dynamic-memory-v2 property are
sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid
field is not present in the drmem_lmb structure.

Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>
---
 arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index b2eeea39684c..c11b6ec99ea3 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c
@@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	const __be32 *p;
 	u32 i, j, lmb_sets;
 	int lmb_index;
+#ifdef CONFIG_MEMORY_HOTPLUG
+	struct drmem_lmb *first = NULL;
+#endif
 
 	lmb_sets = of_read_number(prop++, 1);
 	if (lmb_sets == 0)
@@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	for (i = 0; i < lmb_sets; i++) {
 		read_drconf_v2_cell(&dr_cell, &p);
 
+#ifdef CONFIG_MEMORY_HOTPLUG
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+#endif
+
 		for (j = 0; j < dr_cell.seq_lmbs; j++) {
 			lmb = &drmem_info->lmbs[lmb_index++];
 
@@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 			lmb->aa_index = dr_cell.aa_index;
 			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+#ifdef CONFIG_MEMORY_HOTPLUG
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
+#endif
 		}
 	}
 }
-- 
2.28.0

Re: [PATCH v2] powerpc/drmem: Don't compute the NUMA node for each LMB

[Christophe Leroy]

Le 05/08/2020 à 15:35, Laurent Dufour a écrit :
> All the LMB from the same set of ibm,dynamic-memory-v2 property are
> sharing the same NUMA node. Don't compute that node for each one.
> 
> Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
> lmb_set_nid() have been made instead of 1022.
> 
> This should prevent some soft lockups when starting large guests
> 
> Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid
> field is not present in the drmem_lmb structure.
> 
> Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>

It looks like this patch was superseded by e5e179aa3a39 ("pseries/drmem: 
don't cache node id in drmem_lmb struct").

If not, anyway it conflicts with that patch so it has to be rebased.

Thanks
Christophe


> ---
>   arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-
>   1 file changed, 24 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
> index b2eeea39684c..c11b6ec99ea3 100644
> --- a/arch/powerpc/mm/drmem.c
> +++ b/arch/powerpc/mm/drmem.c
> @@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
>   	const __be32 *p;
>   	u32 i, j, lmb_sets;
>   	int lmb_index;
> +#ifdef CONFIG_MEMORY_HOTPLUG
> +	struct drmem_lmb *first = NULL;
> +#endif
>   
>   	lmb_sets = of_read_number(prop++, 1);
>   	if (lmb_sets == 0)
> @@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
>   	for (i = 0; i < lmb_sets; i++) {
>   		read_drconf_v2_cell(&dr_cell, &p);
>   
> +#ifdef CONFIG_MEMORY_HOTPLUG
> +		/*
> +		 * Fetch the NUMA node id for the fist set or if the
> +		 * associativity index is different from the previous set.
> +		 */
> +		if (first && dr_cell.aa_index != first->aa_index)
> +			first = NULL;
> +#endif
> +
>   		for (j = 0; j < dr_cell.seq_lmbs; j++) {
>   			lmb = &drmem_info->lmbs[lmb_index++];
>   
> @@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
>   			lmb->aa_index = dr_cell.aa_index;
>   			lmb->flags = dr_cell.flags;
>   
> -			lmb_set_nid(lmb);
> +#ifdef CONFIG_MEMORY_HOTPLUG
> +			/*
> +			 * All the LMB in the set share the same NUMA
> +			 * associativity property. So read that node only once.
> +			 */
> +			if (!first) {
> +				lmb_set_nid(lmb);
> +				first = lmb;
> +			} else {
> +				lmb->nid = first->nid;
> +			}
> +#endif
>   		}
>   	}
>   }

Re: [PATCH v2] powerpc/drmem: Don't compute the NUMA node for each LMB

[Laurent Dufour]

On 02/04/2022, 18:35:15, Christophe Leroy wrote:
> 
> 
> Le 05/08/2020 à 15:35, Laurent Dufour a écrit :
>> All the LMB from the same set of ibm,dynamic-memory-v2 property are
>> sharing the same NUMA node. Don't compute that node for each one.
>>
>> Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
>> lmb_set_nid() have been made instead of 1022.
>>
>> This should prevent some soft lockups when starting large guests
>>
>> Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid
>> field is not present in the drmem_lmb structure.
>>
>> Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>
> 
> It looks like this patch was superseded by e5e179aa3a39 ("pseries/drmem:
> don't cache node id in drmem_lmb struct").

That patch has been superseded and can be dropped.

Thanks,
Laurent.


> If not, anyway it conflicts with that patch so it has to be rebased.
> 
> Thanks
> Christophe
> 
> 
>> ---
>>   arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-
>>   1 file changed, 24 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
>> index b2eeea39684c..c11b6ec99ea3 100644
>> --- a/arch/powerpc/mm/drmem.c
>> +++ b/arch/powerpc/mm/drmem.c
>> @@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32
>> *prop)
>>       const __be32 *p;
>>       u32 i, j, lmb_sets;
>>       int lmb_index;
>> +#ifdef CONFIG_MEMORY_HOTPLUG
>> +    struct drmem_lmb *first = NULL;
>> +#endif
>>         lmb_sets = of_read_number(prop++, 1);
>>       if (lmb_sets == 0)
>> @@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32
>> *prop)
>>       for (i = 0; i < lmb_sets; i++) {
>>           read_drconf_v2_cell(&dr_cell, &p);
>>   +#ifdef CONFIG_MEMORY_HOTPLUG
>> +        /*
>> +         * Fetch the NUMA node id for the fist set or if the
>> +         * associativity index is different from the previous set.
>> +         */
>> +        if (first && dr_cell.aa_index != first->aa_index)
>> +            first = NULL;
>> +#endif
>> +
>>           for (j = 0; j < dr_cell.seq_lmbs; j++) {
>>               lmb = &drmem_info->lmbs[lmb_index++];
>>   @@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32
>> *prop)
>>               lmb->aa_index = dr_cell.aa_index;
>>               lmb->flags = dr_cell.flags;
>>   -            lmb_set_nid(lmb);
>> +#ifdef CONFIG_MEMORY_HOTPLUG
>> +            /*
>> +             * All the LMB in the set share the same NUMA
>> +             * associativity property. So read that node only once.
>> +             */
>> +            if (!first) {
>> +                lmb_set_nid(lmb);
>> +                first = lmb;
>> +            } else {
>> +                lmb->nid = first->nid;
>> +            }
>> +#endif
>>           }
>>       }
>>   }