[PATCH v4 1/4] Numachip: Fix 16-bit APIC ID truncation

[PATCH v4 1/4] Numachip: Fix 16-bit APIC ID truncation

[Daniel J Blueman]

Prevent 16-bit APIC IDs being truncated by using correct mask. This fixes
booting large systems, where the wrong core would receive the startup and
init IPIs, causing hanging.

Signed-off-by: Daniel J Blueman <daniel@numascale.com>
---
 apic_numachip.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 81d70ba..bd083c0 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -40,7 +40,7 @@ static unsigned int get_apic_id(unsigned long x)
 	unsigned int id;
 
 	rdmsrl(MSR_FAM10H_NODE_ID, value);
-	id = ((x >> 24) & 0xffU) | ((value << 2) & 0x3f00U);
+	id = ((x >> 24) & 0xffU) | ((value << 2) & 0xff00U);
 
 	return id;
 }
-- 
1.9.1

[PATCH v4 2/4] Numachip: Elide self-IPI ICR polling

[Daniel J Blueman]

The default self-IPI path polls the ICR to delay sending the IPI until
there is no IPI in progress. This is redundant on x86-86 APICs, since
IPIs are queued. See the AMD64 Architecture Programmer's Manual, vol 2,
p525.

Signed-off-by: Daniel J Blueman <daniel@numascale.com>
---
 apic_numachip.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 81d70ba..bd083c0 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -145,7 +145,7 @@ static void numachip_send_IPI_all(int vector)
 
 static void numachip_send_IPI_self(int vector)
 {
-	__default_send_IPI_shortcut(APIC_DEST_SELF, vector, APIC_DEST_PHYSICAL);
+	apic_write(APIC_SELF_IPI, vector);
 }
 
 static int __init numachip_probe(void)
-- 
1.9.1

[tip:x86/platform] x86: numachip: Elide self-IPI ICR polling

[tip-bot for Daniel J Blueman]

Commit-ID:  25e5a76bae106e1673887db09e22b19cb1a86c45
Gitweb:     http://git.kernel.org/tip/25e5a76bae106e1673887db09e22b19cb1a86c45
Author:     Daniel J Blueman <daniel@numascale.com>
AuthorDate: Tue, 4 Nov 2014 16:29:42 +0800
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 4 Nov 2014 18:17:27 +0100

x86: numachip: Elide self-IPI ICR polling

The default self-IPI path polls the ICR to delay sending the IPI until
there is no IPI in progress. This is redundant on x86-86 APICs, since
IPIs are queued. See the AMD64 Architecture Programmer's Manual, vol 2,
p525.

Signed-off-by: Daniel J Blueman <daniel@numascale.com>
Cc: Steffen Persvold <sp@numascale.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Link: http://lkml.kernel.org/r/1415089784-28779-2-git-send-email-daniel@numascale.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/apic/apic_numachip.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 2aaee79..7a31912 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -145,7 +145,7 @@ static void numachip_send_IPI_all(int vector)
 
 static void numachip_send_IPI_self(int vector)
 {
-	__default_send_IPI_shortcut(APIC_DEST_SELF, vector, APIC_DEST_PHYSICAL);
+	apic_write(APIC_SELF_IPI, vector);
 }
 
 static int __init numachip_probe(void)

[PATCH v4 3/4] Numachip: APIC driver cleanups

[Daniel J Blueman]

Drop printing that serves no purpose, as it's printing fixed or known
values, and mark constant structure appropriately.

Signed-off-by: Daniel J Blueman <daniel@numascale.com>
---
 arch/x86/kernel/apic/apic_numachip.c | 22 +++-------------------
 arch/x86/pci/numachip.c              |  2 +-
 2 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index c965b69..6374d94 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -153,20 +153,8 @@ static int __init numachip_probe(void)
 	return apic == &apic_numachip;
 }
 
-static void __init map_csrs(void)
-{
-	printk(KERN_INFO "NumaChip: Mapping local CSR space (%016llx - %016llx)\n",
-		NUMACHIP_LCSR_BASE, NUMACHIP_LCSR_BASE + NUMACHIP_LCSR_SIZE - 1);
-	init_extra_mapping_uc(NUMACHIP_LCSR_BASE, NUMACHIP_LCSR_SIZE);
-
-	printk(KERN_INFO "NumaChip: Mapping global CSR space (%016llx - %016llx)\n",
-		NUMACHIP_GCSR_BASE, NUMACHIP_GCSR_BASE + NUMACHIP_GCSR_SIZE - 1);
-	init_extra_mapping_uc(NUMACHIP_GCSR_BASE, NUMACHIP_GCSR_SIZE);
-}
-
 static void fixup_cpu_id(struct cpuinfo_x86 *c, int node)
 {
-
 	if (c->phys_proc_id != node) {
 		c->phys_proc_id = node;
 		per_cpu(cpu_llc_id, smp_processor_id()) = node;
@@ -180,19 +168,15 @@ bool is_numachip_system(void)
 
 static int __init numachip_system_init(void)
 {
-	unsigned int val;
-
 	if (!numachip_system)
 		return 0;
 
+	init_extra_mapping_uc(NUMACHIP_LCSR_BASE, NUMACHIP_LCSR_SIZE);
+	init_extra_mapping_uc(NUMACHIP_GCSR_BASE, NUMACHIP_GCSR_SIZE);
+
 	x86_cpuinit.fixup_cpu_id = fixup_cpu_id;
 	x86_init.pci.arch_init = pci_numachip_init;
 
-	map_csrs();
-
-	val = read_lcsr(CSR_G0_NODE_IDS);
-	printk(KERN_INFO "NumaChip: Local NodeID = %08x\n", val);
-
 	return 0;
 }
 early_initcall(numachip_system_init);
diff --git a/arch/x86/pci/numachip.c b/arch/x86/pci/numachip.c
index 7307d9d..2e565e6 100644
--- a/arch/x86/pci/numachip.c
+++ b/arch/x86/pci/numachip.c
@@ -103,7 +103,7 @@ static int pci_mmcfg_write_numachip(unsigned int seg, unsigned int bus,
 	return 0;
 }
 
-const struct pci_raw_ops pci_mmcfg_numachip = {
+static const struct pci_raw_ops pci_mmcfg_numachip = {
 	.read = pci_mmcfg_read_numachip,
 	.write = pci_mmcfg_write_numachip,
 };
-- 
1.9.1

[tip:x86/platform] x86: numachip: APIC driver cleanups

[tip-bot for Daniel J Blueman]

Commit-ID:  b980dcf25d0ee1f0f8c7b6afc0e715a2f5da5ec4
Gitweb:     http://git.kernel.org/tip/b980dcf25d0ee1f0f8c7b6afc0e715a2f5da5ec4
Author:     Daniel J Blueman <daniel@numascale.com>
AuthorDate: Tue, 4 Nov 2014 16:29:43 +0800
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 4 Nov 2014 18:17:27 +0100

x86: numachip: APIC driver cleanups

Drop printing that serves no purpose, as it's printing fixed or known
values, and mark constant structure appropriately.

Signed-off-by: Daniel J Blueman <daniel@numascale.com>
Cc: Steffen Persvold <sp@numascale.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Link: http://lkml.kernel.org/r/1415089784-28779-3-git-send-email-daniel@numascale.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/apic/apic_numachip.c | 22 +++-------------------
 arch/x86/pci/numachip.c              |  2 +-
 2 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 7a31912..c2fd21f 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -153,20 +153,8 @@ static int __init numachip_probe(void)
 	return apic == &apic_numachip;
 }
 
-static void __init map_csrs(void)
-{
-	printk(KERN_INFO "NumaChip: Mapping local CSR space (%016llx - %016llx)\n",
-		NUMACHIP_LCSR_BASE, NUMACHIP_LCSR_BASE + NUMACHIP_LCSR_SIZE - 1);
-	init_extra_mapping_uc(NUMACHIP_LCSR_BASE, NUMACHIP_LCSR_SIZE);
-
-	printk(KERN_INFO "NumaChip: Mapping global CSR space (%016llx - %016llx)\n",
-		NUMACHIP_GCSR_BASE, NUMACHIP_GCSR_BASE + NUMACHIP_GCSR_SIZE - 1);
-	init_extra_mapping_uc(NUMACHIP_GCSR_BASE, NUMACHIP_GCSR_SIZE);
-}
-
 static void fixup_cpu_id(struct cpuinfo_x86 *c, int node)
 {
-
 	if (c->phys_proc_id != node) {
 		c->phys_proc_id = node;
 		per_cpu(cpu_llc_id, smp_processor_id()) = node;
@@ -175,19 +163,15 @@ static void fixup_cpu_id(struct cpuinfo_x86 *c, int node)
 
 static int __init numachip_system_init(void)
 {
-	unsigned int val;
-
 	if (!numachip_system)
 		return 0;
 
+	init_extra_mapping_uc(NUMACHIP_LCSR_BASE, NUMACHIP_LCSR_SIZE);
+	init_extra_mapping_uc(NUMACHIP_GCSR_BASE, NUMACHIP_GCSR_SIZE);
+
 	x86_cpuinit.fixup_cpu_id = fixup_cpu_id;
 	x86_init.pci.arch_init = pci_numachip_init;
 
-	map_csrs();
-
-	val = read_lcsr(CSR_G0_NODE_IDS);
-	printk(KERN_INFO "NumaChip: Local NodeID = %08x\n", val);
-
 	return 0;
 }
 early_initcall(numachip_system_init);
diff --git a/arch/x86/pci/numachip.c b/arch/x86/pci/numachip.c
index 7307d9d..2e565e6 100644
--- a/arch/x86/pci/numachip.c
+++ b/arch/x86/pci/numachip.c
@@ -103,7 +103,7 @@ static int pci_mmcfg_write_numachip(unsigned int seg, unsigned int bus,
 	return 0;
 }
 
-const struct pci_raw_ops pci_mmcfg_numachip = {
+static const struct pci_raw_ops pci_mmcfg_numachip = {
 	.read = pci_mmcfg_read_numachip,
 	.write = pci_mmcfg_write_numachip,
 };

[PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Daniel J Blueman]

On large-memory x86-64 systems of 64GB or more with memory hot-plug
enabled, use a 2GB memory block size. Eg with 64GB memory, this reduces
the number of directories in /sys/devices/system/memory from 512 to 32,
making it more manageable, and reducing the creation time accordingly.

This caveat is that the memory can't be offlined (for hotplug or otherwise)
with finer 128MB granularity, but this is unimportant due to the high
memory densities generally used with such large-memory systems, where
eg a single DIMM is the order of 16GB. 

Signed-off-by: Daniel J Blueman <daniel@numascale.com>
---
 init_64.c |    7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index df1a992..9622ab2 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -52,7 +52,6 @@
 #include <asm/numa.h>
 #include <asm/cacheflush.h>
 #include <asm/init.h>
-#include <asm/uv/uv.h>
 #include <asm/setup.h>
 
 #include "mm_internal.h"
@@ -1234,12 +1233,10 @@ static unsigned long probe_memory_block_size(void)
 	/* start from 2g */
 	unsigned long bz = 1UL<<31;
 
-#ifdef CONFIG_X86_UV
-	if (is_uv_system()) {
-		printk(KERN_INFO "UV: memory block size 2GB\n");
+	if (totalram_pages >= (64ULL << (30 - PAGE_SHIFT))) {
+		pr_info("Using 2GB memory block size for large-memory system\n");
 		return 2UL * 1024 * 1024 * 1024;
 	}
-#endif
 
 	/* less than 64g installed */
 	if ((max_pfn << PAGE_SHIFT) < (16UL << 32))

[tip:x86/mm] x86: mm: Use 2GB memory block size on large-memory x86-64 systems

[tip-bot for Daniel J Blueman]

Commit-ID:  bdee237c0343a5d1a6cf72c7ea68e88338b26e08
Gitweb:     http://git.kernel.org/tip/bdee237c0343a5d1a6cf72c7ea68e88338b26e08
Author:     Daniel J Blueman <daniel@numascale.com>
AuthorDate: Tue, 4 Nov 2014 16:29:44 +0800
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 4 Nov 2014 18:19:27 +0100

x86: mm: Use 2GB memory block size on large-memory x86-64 systems

On large-memory x86-64 systems of 64GB or more with memory hot-plug
enabled, use a 2GB memory block size. Eg with 64GB memory, this reduces
the number of directories in /sys/devices/system/memory from 512 to 32,
making it more manageable, and reducing the creation time accordingly.

This caveat is that the memory can't be offlined (for hotplug or
otherwise) with the finer default 128MB granularity, but this is
unimportant due to the high memory densities generally used with such
large-memory systems, where eg a single DIMM is the order of 16GB.

Signed-off-by: Daniel J Blueman <daniel@numascale.com>
Cc: Steffen Persvold <sp@numascale.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Link: http://lkml.kernel.org/r/1415089784-28779-4-git-send-email-daniel@numascale.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/mm/init_64.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 4cb8763..ebca30f 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -52,7 +52,6 @@
 #include <asm/numa.h>
 #include <asm/cacheflush.h>
 #include <asm/init.h>
-#include <asm/uv/uv.h>
 #include <asm/setup.h>
 
 #include "mm_internal.h"
@@ -1247,12 +1246,10 @@ static unsigned long probe_memory_block_size(void)
 	/* start from 2g */
 	unsigned long bz = 1UL<<31;
 
-#ifdef CONFIG_X86_UV
-	if (is_uv_system()) {
-		printk(KERN_INFO "UV: memory block size 2GB\n");
+	if (totalram_pages >= (64ULL << (30 - PAGE_SHIFT))) {
+		pr_info("Using 2GB memory block size for large-memory system\n");
 		return 2UL * 1024 * 1024 * 1024;
 	}
-#endif
 
 	/* less than 64g installed */
 	if ((max_pfn << PAGE_SHIFT) < (16UL << 32))

Re: [tip:x86/mm] x86: mm: Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

On Tue, Nov 4, 2014 at 9:22 AM, tip-bot for Daniel J Blueman
<tipbot@zytor.com> wrote:
> Commit-ID:  bdee237c0343a5d1a6cf72c7ea68e88338b26e08
> Gitweb:     http://git.kernel.org/tip/bdee237c0343a5d1a6cf72c7ea68e88338b26e08
> Author:     Daniel J Blueman <daniel@numascale.com>
> AuthorDate: Tue, 4 Nov 2014 16:29:44 +0800
> Committer:  Thomas Gleixner <tglx@linutronix.de>
> CommitDate: Tue, 4 Nov 2014 18:19:27 +0100
>
> x86: mm: Use 2GB memory block size on large-memory x86-64 systems
>
> On large-memory x86-64 systems of 64GB or more with memory hot-plug
> enabled, use a 2GB memory block size. Eg with 64GB memory, this reduces
> the number of directories in /sys/devices/system/memory from 512 to 32,
> making it more manageable, and reducing the creation time accordingly.
>
> This caveat is that the memory can't be offlined (for hotplug or
> otherwise) with the finer default 128MB granularity, but this is
> unimportant due to the high memory densities generally used with such
> large-memory systems, where eg a single DIMM is the order of 16GB.
>
> Signed-off-by: Daniel J Blueman <daniel@numascale.com>
> Cc: Steffen Persvold <sp@numascale.com>
> Cc: Bjorn Helgaas <bhelgaas@google.com>
> Link: http://lkml.kernel.org/r/1415089784-28779-4-git-send-email-daniel@numascale.com
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> ---
>  arch/x86/mm/init_64.c | 7 ++-----
>  1 file changed, 2 insertions(+), 5 deletions(-)
>
> diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
> index 4cb8763..ebca30f 100644
> --- a/arch/x86/mm/init_64.c
> +++ b/arch/x86/mm/init_64.c
> @@ -52,7 +52,6 @@
>  #include <asm/numa.h>
>  #include <asm/cacheflush.h>
>  #include <asm/init.h>
> -#include <asm/uv/uv.h>
>  #include <asm/setup.h>
>
>  #include "mm_internal.h"
> @@ -1247,12 +1246,10 @@ static unsigned long probe_memory_block_size(void)
>         /* start from 2g */
>         unsigned long bz = 1UL<<31;
>
> -#ifdef CONFIG_X86_UV
> -       if (is_uv_system()) {
> -               printk(KERN_INFO "UV: memory block size 2GB\n");
> +       if (totalram_pages >= (64ULL << (30 - PAGE_SHIFT))) {
> +               pr_info("Using 2GB memory block size for large-memory system\n");
>                 return 2UL * 1024 * 1024 * 1024;
>         }
> -#endif
>
>         /* less than 64g installed */
>         if ((max_pfn << PAGE_SHIFT) < (16UL << 32))
.....

Come on. Can you remove the following dead lines caused by this patch
the same time ?

Yinghai

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Luck, Tony]

On Tue, Nov 04, 2014 at 04:29:44PM +0800, Daniel J Blueman wrote:
> On large-memory x86-64 systems of 64GB or more with memory hot-plug
> enabled, use a 2GB memory block size. Eg with 64GB memory, this reduces
> the number of directories in /sys/devices/system/memory from 512 to 32,
> making it more manageable, and reducing the creation time accordingly.
> 
> This caveat is that the memory can't be offlined (for hotplug or otherwise)
> with finer 128MB granularity, but this is unimportant due to the high
> memory densities generally used with such large-memory systems, where
> eg a single DIMM is the order of 16GB. 

git bisect points to this commit as the cause of a panic on my
machine:

[    4.518415] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[    4.525882] PCI: MMCONFIG for domain 0000 [bus 00-ff] at [mem 0x80000000-0x8fffffff] (base 0x80000000)
[    4.536280] PCI: MMCONFIG at [mem 0x80000000-0x8fffffff] reserved in E820
[    4.544344] PCI: Using configuration type 1 for base access
[    4.550778] BUG: unable to handle kernel paging request at ffffea0078000020
[    4.558572] IP: [<ffffffff8142ab0d>] register_mem_sect_under_node+0x6d/0xe0
[    4.566366] PGD 1dfffcc067 PUD 1dfffca067 PMD 0
[    4.571554] Oops: 0000 [#1] SMP
[    4.575181] Modules linked in:
[    4.578604] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 3.18.0-rc2+ #17
[    4.585800] Hardware name: Intel Corporation BRICKLAND/BRICKLAND, BIOS BRBDXSD1.86B.0326.D03.1508171454 08/17/2015
[    4.597347] task: ffff883b84960000 ti: ffff881d7ea14000 task.ti: ffff881d7ea14000
[    4.605705] RIP: 0010:[<ffffffff8142ab0d>]  [<ffffffff8142ab0d>] register_mem_sect_under_node+0x6d/0xe0
[    4.616205] RSP: 0000:ffff881d7ea17d68  EFLAGS: 00010206
[    4.622135] RAX: ffffea0078000020 RBX: 0000000000000001 RCX: 0000000001e00000
[    4.630102] RDX: 0000000078000000 RSI: 0000000000000001 RDI: ffff881d7ccb6400
[    4.638069] RBP: ffff881d7ea17d78 R08: 0000000001e7ffff R09: 0000000003c00000
[    4.646035] R10: ffffffff813043a0 R11: ffffea0169efa600 R12: 0000000000000001
[    4.654003] R13: 0000000000000001 R14: ffff881d7ccb6400 R15: 0000000000000000
[    4.661972] FS:  0000000000000000(0000) GS:ffff881d8b400000(0000) knlGS:0000000000000000
[    4.670996] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[    4.677411] CR2: ffffea0078000020 CR3: 00000000019a0000 CR4: 00000000003407f0
[    4.685381] Stack:
[    4.687627]  0000000001e70000 0000000000000001 ffff881d7ea17dc8 ffffffff8142af0a
[    4.695926]  ffff881d7ea17de8 0000000003c00000 ffff881d00000018 0000000000000002
[    4.704225]  0000000000000400 0000000000000000 ffffffff81b101c5 0000000000000000
[    4.712524] Call Trace:
[    4.715261]  [<ffffffff8142af0a>] register_one_node+0x18a/0x2b0
[    4.721871]  [<ffffffff81b101c5>] ? pci_iommu_alloc+0x6e/0x6e
[    4.728287]  [<ffffffff81b10201>] topology_init+0x3c/0x95
[    4.734321]  [<ffffffff81002144>] do_one_initcall+0xd4/0x210
[    4.740645]  [<ffffffff8109b515>] ? parse_args+0x245/0x480
[    4.746774]  [<ffffffff810bddc8>] ? __wake_up+0x48/0x60
[    4.752611]  [<ffffffff81b062f9>] kernel_init_freeable+0x19d/0x23c
[    4.759511]  [<ffffffff81b059e3>] ? initcall_blacklist+0xb6/0xb6
[    4.766226]  [<ffffffff816580d0>] ? rest_init+0x80/0x80
[    4.772059]  [<ffffffff816580de>] kernel_init+0xe/0xf0
[    4.777803]  [<ffffffff8167057c>] ret_from_fork+0x7c/0xb0
[    4.783831]  [<ffffffff816580d0>] ? rest_init+0x80/0x80
[    4.789655] Code: 39 c1 77 59 48 c1 e2 15 48 b8 00 00 00 00 00 ea ff ff 48 8d 44 02 20 eb 12 0f 1f 44 00 00 48 83 c1 01 48 83 c0 40 49 39 c8 72 5b <48> 83 38 00 74 ed 48 8b 50 e0 48 c1 ea 36 39 d6 75 e1 48 8b 04
[    4.811356] RIP  [<ffffffff8142ab0d>] register_mem_sect_under_node+0x6d/0xe0
[    4.819238]  RSP <ffff881d7ea17d68>
[    4.823132] CR2: ffffea0078000020
[    4.826836] ---[ end trace 10b7bb944b11529f ]---
[    4.831989] Kernel panic - not syncing: Fatal exception
[    4.837866] ---[ end Kernel panic - not syncing: Fatal exception

reverting the commit indeed makes the problem go away.

Now the root problem for me is that I have an insane BIOS
that handed me an e820 table that is full of holes (for entries
above 4GB) ... and ends with an entry that is only 256M aligned:


[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000008dfff] usable
[    0.000000] BIOS-e820: [mem 0x000000000008e000-0x000000000008ffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000090000-0x000000000009ffff] usable
[    0.000000] BIOS-e820: [mem 0x00000000000a0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000005cc0afff] usable
[    0.000000] BIOS-e820: [mem 0x000000005cc0b000-0x000000005e108fff] reserved
[    0.000000] BIOS-e820: [mem 0x000000005e109000-0x000000006035cfff] ACPI NVS
[    0.000000] BIOS-e820: [mem 0x000000006035d000-0x00000000604fcfff] ACPI data
[    0.000000] BIOS-e820: [mem 0x00000000604fd000-0x000000007bafffff] usable
[    0.000000] BIOS-e820: [mem 0x000000007bb00000-0x000000008fffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fed1c000-0x00000000fed1ffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000100000000-0x000000118fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000001200000000-0x0000001dffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000001e70000000-0x0000001f3fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000002000000000-0x0000002cffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000002da0000000-0x0000002e6fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000002f00000000-0x0000003bffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000003cd0000000-0x0000003d9fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000003e00000000-0x0000004ccfffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000004d00000000-0x0000005affffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000005b30000000-0x0000005bffffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000005c00000000-0x00000069ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000006a60000000-0x0000006b2fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000006c00000000-0x000000798fffffff] usable

so the older code will look at max_pfn and set memory block size:

[    3.021752] memory block size : 256MB

I think the problem is more connected to the strange max_pfn rather
than the holes ... but will defer to wiser heads.

If the problem is with max_pfn ... I don't think it is a safe assumption
that systems with >64GB memory will have 2GB aligned max_pfn.

-Tony

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

[-- Attachment #1: Type: text/plain, Size: 1789 bytes --]

On Fri, Aug 21, 2015 at 11:19 AM, Luck, Tony <tony.luck@intel.com> wrote:
> On Tue, Nov 04, 2014 at 04:29:44PM +0800, Daniel J Blueman wrote:
>> On large-memory x86-64 systems of 64GB or more with memory hot-plug
>> enabled, use a 2GB memory block size. Eg with 64GB memory, this reduces
>> the number of directories in /sys/devices/system/memory from 512 to 32,
>> making it more manageable, and reducing the creation time accordingly.
>>
>> This caveat is that the memory can't be offlined (for hotplug or otherwise)
>> with finer 128MB granularity, but this is unimportant due to the high
>> memory densities generally used with such large-memory systems, where
>> eg a single DIMM is the order of 16GB.
>
> git bisect points to this commit as the cause of a panic on my
> machine:
>
> [    4.518415] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
> [    4.525882] PCI: MMCONFIG for domain 0000 [bus 00-ff] at [mem 0x80000000-0x8fffffff] (base 0x80000000)
> [    4.536280] PCI: MMCONFIG at [mem 0x80000000-0x8fffffff] reserved in E820
> [    4.544344] PCI: Using configuration type 1 for base access
> [    4.550778] BUG: unable to handle kernel paging request at ffffea0078000020
> [    4.558572] IP: [<ffffffff8142ab0d>] register_mem_sect_under_
...
> so the older code will look at max_pfn and set memory block size:
>
> [    3.021752] memory block size : 256MB
>
> I think the problem is more connected to the strange max_pfn rather
> than the holes ... but will defer to wiser heads.
>
> If the problem is with max_pfn ... I don't think it is a safe assumption
> that systems with >64GB memory will have 2GB aligned max_pfn.

That commit could be reverted.
According to
https://lkml.org/lkml/2014/11/10/123

I had attached patch for my test setups for a while.

Yinghai

[-- Attachment #2: revert_commit_bdee237.patch --]
[-- Type: text/x-patch, Size: 1521 bytes --]

Subject: [PATCH] x86, mm: put memory block size probing back

commit bdee237c0343 ("x86: mm: Use 2GB memory block size on large-memory x86-64 systems")
let system with more than 64GiB ram just use 2G as memory block
size without probing.

found one system: has memory map like:
[0x00000000-0x60000000)
[0x100000000-0x20a0000000)

We should use 0x20000000 in this case. So can not assume system with big
memory have 2g tail anymore.

So revert it to put probing back.

Fixes: bdee237c0343 ("x86: mm: Use 2GB memory block size on large-memory x86-64 systems")
Signed-off-by: Yinghai Lu <yinghai@kernel.org>

---
 arch/x86/mm/init_64.c |    7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

Index: linux-2.6/arch/x86/mm/init_64.c
===================================================================
--- linux-2.6.orig/arch/x86/mm/init_64.c
+++ linux-2.6/arch/x86/mm/init_64.c
@@ -52,6 +52,7 @@
 #include <asm/numa.h>
 #include <asm/cacheflush.h>
 #include <asm/init.h>
+#include <asm/uv/uv.h>
 #include <asm/setup.h>
 
 #include "mm_internal.h"
@@ -1204,10 +1205,12 @@ static unsigned long probe_memory_block_
 	/* start from 2g */
 	unsigned long bz = 1UL<<31;
 
-	if (totalram_pages >= (64ULL << (30 - PAGE_SHIFT))) {
-		pr_info("Using 2GB memory block size for large-memory system\n");
+#ifdef CONFIG_X86_UV
+	if (is_uv_system()) {
+		printk(KERN_INFO "UV: memory block size 2GB\n");
 		return 2UL * 1024 * 1024 * 1024;
 	}
+#endif
 
 	/* less than 64g installed */
 	if ((max_pfn << PAGE_SHIFT) < (16UL << 32))

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Luck, Tony]

On Fri, Aug 21, 2015 at 11:38:13AM -0700, Yinghai Lu wrote:
> That commit could be reverted.
> According to
> https://lkml.org/lkml/2014/11/10/123

Do we really need to force the MIN_MEMORY_BLOCK_SIZE on small
systems?

What about this patch - which just uses max_pfn to choose
the block size.

It seems that many systems with large amounts of memory
will have a nicely aligned max_pfn ... so they will get
the 2GB block size.  If they don't have a well aligned
max_pfn, then they need to use a smaller size to avoid
the crash I saw.

-Tony


diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 3fba623e3ba5..e14e90fd1cf8 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1195,15 +1195,6 @@ static unsigned long probe_memory_block_size(void)
 	/* start from 2g */
 	unsigned long bz = 1UL<<31;
 
-	if (totalram_pages >= (64ULL << (30 - PAGE_SHIFT))) {
-		pr_info("Using 2GB memory block size for large-memory system\n");
-		return 2UL * 1024 * 1024 * 1024;
-	}
-
-	/* less than 64g installed */
-	if ((max_pfn << PAGE_SHIFT) < (16UL << 32))
-		return MIN_MEMORY_BLOCK_SIZE;
-
 	/* get the tail size */
 	while (bz > MIN_MEMORY_BLOCK_SIZE) {
 		if (!((max_pfn << PAGE_SHIFT) & (bz - 1)))

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

On Fri, Aug 21, 2015 at 1:27 PM, Luck, Tony <tony.luck@intel.com> wrote:
> On Fri, Aug 21, 2015 at 11:38:13AM -0700, Yinghai Lu wrote:
>> That commit could be reverted.
>> According to
>> https://lkml.org/lkml/2014/11/10/123
>
> Do we really need to force the MIN_MEMORY_BLOCK_SIZE on small
> systems?

That is introduced in commit 982792c7 ("x86, mm: probe memory block
size for generic x86 64bit
").
that patch is used to make boot faster why create less entries
in /sys/device/system/memory/.
On system with less 64G ram, that will not have too many entries
even with MIN_MEMORY_BLOCK_SIZE.

>
> What about this patch - which just uses max_pfn to choose
> the block size.
>
> It seems that many systems with large amounts of memory
> will have a nicely aligned max_pfn ... so they will get
> the 2GB block size.  If they don't have a well aligned
> max_pfn, then they need to use a smaller size to avoid
> the crash I saw.

Good to me.

> diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
> index 3fba623e3ba5..e14e90fd1cf8 100644
> --- a/arch/x86/mm/init_64.c
> +++ b/arch/x86/mm/init_64.c
> @@ -1195,15 +1195,6 @@ static unsigned long probe_memory_block_size(void)
>         /* start from 2g */
>         unsigned long bz = 1UL<<31;
>
> -       if (totalram_pages >= (64ULL << (30 - PAGE_SHIFT))) {
> -               pr_info("Using 2GB memory block size for large-memory system\n");
> -               return 2UL * 1024 * 1024 * 1024;
> -       }
> -
> -       /* less than 64g installed */
> -       if ((max_pfn << PAGE_SHIFT) < (16UL << 32))
> -               return MIN_MEMORY_BLOCK_SIZE;
> -
>         /* get the tail size */
>         while (bz > MIN_MEMORY_BLOCK_SIZE) {
>                 if (!((max_pfn << PAGE_SHIFT) & (bz - 1)))

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Tony Luck]

On Fri, Aug 21, 2015 at 1:50 PM, Yinghai Lu <yinghai@kernel.org> wrote:
>> It seems that many systems with large amounts of memory
>> will have a nicely aligned max_pfn ... so they will get
>> the 2GB block size.  If they don't have a well aligned
>> max_pfn, then they need to use a smaller size to avoid
>> the crash I saw.
>
> Good to me.

Still stuff going on that I don't understand here. I increased the amount of
mirrored memory in this machine which moved max_pfn to 0x7560000
and probe_memory_block_size() picked 512MB as the memory_block_size,
which seemed plausible.

But my kernel still crashed during boot with this value. :-(
Forcing the block size to 128M made the system boot.

Maybe all the holes in the e820 map matter too (specifically the
alignment of the holes)?

-Tony

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

On Fri, Aug 21, 2015 at 4:54 PM, Tony Luck <tony.luck@gmail.com> wrote:
> On Fri, Aug 21, 2015 at 1:50 PM, Yinghai Lu <yinghai@kernel.org> wrote:
>
> Still stuff going on that I don't understand here. I increased the amount of
> mirrored memory in this machine which moved max_pfn to 0x7560000
> and probe_memory_block_size() picked 512MB as the memory_block_size,
> which seemed plausible.
>
> But my kernel still crashed during boot with this value. :-(
> Forcing the block size to 128M made the system boot.
>
> Maybe all the holes in the e820 map matter too (specifically the
> alignment of the holes)?

Then, what does the E820 look like?

Yinghai

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Tony Luck]

[-- Attachment #1: Type: text/plain, Size: 171 bytes --]

On Mon, Aug 24, 2015 at 10:46 AM, Yinghai Lu <yinghai@kernel.org> wrote:
> Then, what does the E820 look like?

See attached serial console log of the latest crash

-Tony

[-- Attachment #2: dmesg --]
[-- Type: application/octet-stream, Size: 47246 bytes --]

[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Initializing cgroup subsys cpuacct
[    0.000000] Linux version 4.2.0-rc8 (aegl@brk-bdx-01.sc.intel.com) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-4) (GCC) ) #1 SMP Mon Aug 24 11:45:01 PDT 2015
[    0.000000] Command line: BOOT_IMAGE=/vmlinuz-4.2.0-rc8 root=/dev/mapper/rhel_brk--bdx--01-root ro rd.lvm.lv=rhel_brk-bdx-01/root rd.lvm.lv=rhel_brk-bdx-01/swap console=tty1 console=ttyS0,115200 LANG=en_US.UTF-8
[    0.000000] x86/fpu: xstate_offset[2]: 0240, xstate_sizes[2]: 0100
[    0.000000] x86/fpu: Supporting XSAVE feature 0x01: 'x87 floating point registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x02: 'SSE registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x04: 'AVX registers'
[    0.000000] x86/fpu: Enabled xstate features 0x7, context size is 0x340 bytes, using 'standard' format.
[    0.000000] x86/fpu: Using 'eager' FPU context switches.
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000008dfff] usable
[    0.000000] BIOS-e820: [mem 0x000000000008e000-0x000000000008ffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000090000-0x000000000009ffff] usable
[    0.000000] BIOS-e820: [mem 0x00000000000a0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000005cc0afff] usable
[    0.000000] BIOS-e820: [mem 0x000000005cc0b000-0x000000005e108fff] reserved
[    0.000000] BIOS-e820: [mem 0x000000005e109000-0x000000006035cfff] ACPI NVS
[    0.000000] BIOS-e820: [mem 0x000000006035d000-0x00000000604fcfff] ACPI data
[    0.000000] BIOS-e820: [mem 0x00000000604fd000-0x000000007bafffff] usable
[    0.000000] BIOS-e820: [mem 0x000000007bb00000-0x000000008fffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fed1c000-0x00000000fed1ffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000100000000-0x000000128fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000001300000000-0x0000001cffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000001d70000000-0x0000001ec7ffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000001f00000000-0x0000002bffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000002c18000000-0x0000002d6fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000002e00000000-0x00000039ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000003ac0000000-0x0000003c17ffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000003d00000000-0x00000048ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000004968000000-0x0000004abfffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000004b00000000-0x00000057ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000005810000000-0x0000005967ffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000005a00000000-0x00000065ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x00000066b8000000-0x000000680fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000006900000000-0x000000755fffffff] usable
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] extended physical RAM map:
[    0.000000] reserve setup_data: [mem 0x0000000000000000-0x000000000008dfff] usable
[    0.000000] reserve setup_data: [mem 0x000000000008e000-0x000000000008ffff] reserved
[    0.000000] reserve setup_data: [mem 0x0000000000090000-0x000000000009ffff] usable
[    0.000000] reserve setup_data: [mem 0x00000000000a0000-0x00000000000fffff] reserved
[    0.000000] reserve setup_data: [mem 0x0000000000100000-0x000000005b754017] usable
[    0.000000] reserve setup_data: [mem 0x000000005b754018-0x000000005b75c057] usable
[    0.000000] reserve setup_data: [mem 0x000000005b75c058-0x000000005b75d017] usable
[    0.000000] reserve setup_data: [mem 0x000000005b75d018-0x000000005b781e57] usable
[    0.000000] reserve setup_data: [mem 0x000000005b781e58-0x000000005b782017] usable
[    0.000000] reserve setup_data: [mem 0x000000005b782018-0x000000005b7a6e57] usable
[    0.000000] reserve setup_data: [mem 0x000000005b7a6e58-0x000000005b7a7017] usable
[    0.000000] reserve setup_data: [mem 0x000000005b7a7018-0x000000005b7ee457] usable
[    0.000000] reserve setup_data: [mem 0x000000005b7ee458-0x000000005cc0afff] usable
[    0.000000] reserve setup_data: [mem 0x000000005cc0b000-0x000000005e108fff] reserved
[    0.000000] reserve setup_data: [mem 0x000000005e109000-0x000000006035cfff] ACPI NVS
[    0.000000] reserve setup_data: [mem 0x000000006035d000-0x00000000604fcfff] ACPI data
[    0.000000] reserve setup_data: [mem 0x00000000604fd000-0x000000007bafffff] usable
[    0.000000] reserve setup_data: [mem 0x000000007bb00000-0x000000008fffffff] reserved
[    0.000000] reserve setup_data: [mem 0x00000000fed1c000-0x00000000fed1ffff] reserved
[    0.000000] reserve setup_data: [mem 0x0000000100000000-0x000000128fffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000001300000000-0x0000001cffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000001d70000000-0x0000001ec7ffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000001f00000000-0x0000002bffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000002c18000000-0x0000002d6fffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000002e00000000-0x00000039ffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000003ac0000000-0x0000003c17ffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000003d00000000-0x00000048ffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000004968000000-0x0000004abfffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000004b00000000-0x00000057ffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000005810000000-0x0000005967ffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000005a00000000-0x00000065ffffffff] usable
[    0.000000] reserve setup_data: [mem 0x00000066b8000000-0x000000680fffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000006900000000-0x000000755fffffff] usable
[    0.000000] efi: EFI v2.40 by EDK II
[    0.000000]                              BIOS ID: BRBDXSD1.86B.0326.D03.1508171454
[    0.000000] efi:  ACPI=0x604fc000  ACPI 2.0=0x604fc014  SMBIOS=0x5d007000 
[    0.000000] SMBIOS 3.0 present.
[    0.000000] e820: last_pfn = 0x7560000 max_arch_pfn = 0x400000000
[    0.000000] x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WC  UC- WT  
[    0.000000] e820: last_pfn = 0x7bb00 max_arch_pfn = 0x400000000
[    0.000000] efi: Memory: 45441M/452863M mirrored memory
[    0.000000] Using GB pages for direct mapping
[    0.000000] init_memory_mapping: [mem 0x00000000-0x000fffff]
[    0.000000] init_memory_mapping: [mem 0x680fc00000-0x680fdfffff]
[    0.000000] init_memory_mapping: [mem 0x6800000000-0x680fbfffff]
[    0.000000] init_memory_mapping: [mem 0x67e0000000-0x67ffffffff]
[    0.000000] init_memory_mapping: [mem 0x6000000000-0x65ffffffff]
[    0.000000] init_memory_mapping: [mem 0x66b8000000-0x67dfffffff]
[    0.000000] init_memory_mapping: [mem 0x4000000000-0x48ffffffff]
[    0.000000] init_memory_mapping: [mem 0x4968000000-0x4abfffefff]
[    0.000000] init_memory_mapping: [mem 0x4b00000000-0x57ffffffff]
[    0.000000] init_memory_mapping: [mem 0x5810000000-0x5967ffefff]
[    0.000000] init_memory_mapping: [mem 0x5a00000000-0x5fffffffff]
[    0.000000] init_memory_mapping: [mem 0x00100000-0x5cc0afff]
[    0.000000] init_memory_mapping: [mem 0x604fd000-0x7bafffff]
[    0.000000] init_memory_mapping: [mem 0x100000000-0xfffffffff]
[    0.000000] init_memory_mapping: [mem 0x1000000000-0x128fffefff]
[    0.000000] init_memory_mapping: [mem 0x1300000000-0x1cffffffff]
[    0.000000] init_memory_mapping: [mem 0x1d70000000-0x1ec7ffefff]
[    0.000000] init_memory_mapping: [mem 0x1f00000000-0x2bffffffff]
[    0.000000] init_memory_mapping: [mem 0x2c18000000-0x2d6fffefff]
[    0.000000] init_memory_mapping: [mem 0x2e00000000-0x39ffffffff]
[    0.000000] init_memory_mapping: [mem 0x3ac0000000-0x3c17ffefff]
[    0.000000] init_memory_mapping: [mem 0x3d00000000-0x3fffffffff]
[    0.000000] init_memory_mapping: [mem 0x680fe00000-0x680fffefff]
[    0.000000] init_memory_mapping: [mem 0x6900000000-0x755fffffff]
[    0.000000] RAMDISK: [mem 0x2bbcc000-0x2ecf0fff]
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000604FC014 000024 (v02 INTEL )
[    0.000000] ACPI: XSDT 0x00000000604FB0E8 0000BC (v01 INTEL  TIANO    00000000      01000013)
[    0.000000] ACPI: FACP 0x00000000604F8000 0000F4 (v04 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: DSDT 0x00000000604C6000 02503B (v02 INTEL  TIANO    00000003 MSFT 01000013)
[    0.000000] ACPI: FACS 0x000000005EABC000 000040
[    0.000000] ACPI: TCPA 0x00000000604FA000 000064 (v02 INTEL  EDK2     00000002      01000013)
[    0.000000] ACPI: BDAT 0x00000000604F9000 000030 (v01 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: HPET 0x00000000604F7000 000038 (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: APIC 0x00000000604F6000 000AFC (v03 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: MCFG 0x00000000604F5000 00003C (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: MSCT 0x00000000604F4000 000090 (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: PCCT 0x00000000604F3000 0000AC (v01 INTEL  TIANO    00000002 MSFT 01000013)
[    0.000000] ACPI: PMCT 0x00000000604F2000 000060 (v01 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: RASF 0x00000000604F1000 000030 (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: SLIT 0x00000000604F0000 00006C (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: SRAT 0x00000000604EE000 001130 (v03 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: SVOS 0x00000000604ED000 000032 (v01 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: WDDT 0x00000000604EC000 000040 (v01 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: SSDT 0x00000000603B6000 10F622 (v02 INTEL  SSDT  PM 00004000 INTL 20090521)
[    0.000000] ACPI: SSDT 0x00000000603B3000 00246F (v02 INTEL  SpsNm    00000002 INTL 20090521)
[    0.000000] ACPI: SSDT 0x00000000603B2000 000040 (v02 INTEL  SpsNvs   00000002 INTL 20090521)
[    0.000000] ACPI: PRAD 0x00000000603B1000 000125 (v02 INTEL  SpsPrAgg 00000002 INTL 20090521)
[    0.000000] ACPI: SPCR 0x00000000603B0000 000050 (v01                 00000000      00000000)
[    0.000000] SRAT: PXM 0 -> APIC 0x00 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x02 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x04 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x06 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x08 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x10 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x12 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x14 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x16 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x20 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x22 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x24 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x26 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x28 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x30 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x32 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x34 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x36 -> Node 0
[    0.000000] SRAT: PXM 1 -> APIC 0x40 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x42 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x44 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x46 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x48 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x50 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x52 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x54 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x56 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x60 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x62 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x64 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x66 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x68 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x70 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x72 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x74 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x76 -> Node 1
[    0.000000] SRAT: PXM 2 -> APIC 0x80 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x82 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x84 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x86 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x88 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x90 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x92 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x94 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x96 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa0 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa2 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa4 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa6 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa8 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb0 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb2 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb4 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb6 -> Node 2
[    0.000000] SRAT: PXM 3 -> APIC 0xc0 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc2 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc4 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc6 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc8 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd0 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd2 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd4 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd6 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe0 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe2 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe4 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe6 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe8 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf0 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf2 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf4 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf6 -> Node 3
[    0.000000] SRAT: PXM 0 -> APIC 0x01 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x03 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x05 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x07 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x09 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x11 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x13 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x15 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x17 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x21 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x23 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x25 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x27 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x29 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x31 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x33 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x35 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x37 -> Node 0
[    0.000000] SRAT: PXM 1 -> APIC 0x41 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x43 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x45 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x47 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x49 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x51 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x53 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x55 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x57 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x61 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x63 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x65 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x67 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x69 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x71 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x73 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x75 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x77 -> Node 1
[    0.000000] SRAT: PXM 2 -> APIC 0x81 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x83 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x85 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x87 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x89 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x91 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x93 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x95 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x97 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa1 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa3 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa5 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa7 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa9 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb1 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb3 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb5 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb7 -> Node 2
[    0.000000] SRAT: PXM 3 -> APIC 0xc1 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc3 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc5 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc7 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc9 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd1 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd3 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd5 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd7 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe1 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe3 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe5 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe7 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe9 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf1 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf3 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf5 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf7 -> Node 3
[    0.000000] SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
[    0.000000] SRAT: Node 0 PXM 0 [mem 0x100000000-0xfffffffff]
[    0.000000] SRAT: Node 0 PXM 0 [mem 0x1000000000-0x1d6fffffff]
[    0.000000] SRAT: Node 1 PXM 1 [mem 0x1d70000000-0x2c17ffffff]
[    0.000000] SRAT: Node 1 PXM 1 [mem 0x2c18000000-0x3abfffffff]
[    0.000000] SRAT: Node 2 PXM 2 [mem 0x3ac0000000-0x4967ffffff]
[    0.000000] SRAT: Node 2 PXM 2 [mem 0x4968000000-0x580fffffff]
[    0.000000] SRAT: Node 3 PXM 3 [mem 0x5810000000-0x66b7ffffff]
[    0.000000] SRAT: Node 3 PXM 3 [mem 0x66b8000000-0x755fffffff]
[    0.000000] NUMA: Node 0 [mem 0x00000000-0x7fffffff] + [mem 0x100000000-0xfffffffff] -> [mem 0x00000000-0xfffffffff]
[    0.000000] NUMA: Node 0 [mem 0x00000000-0xfffffffff] + [mem 0x1000000000-0x1d6fffffff] -> [mem 0x00000000-0x1d6fffffff]
[    0.000000] NUMA: Node 1 [mem 0x1d70000000-0x2c17ffffff] + [mem 0x2c18000000-0x3abfffffff] -> [mem 0x1d70000000-0x3abfffffff]
[    0.000000] NUMA: Node 2 [mem 0x3ac0000000-0x4967ffffff] + [mem 0x4968000000-0x580fffffff] -> [mem 0x3ac0000000-0x580fffffff]
[    0.000000] NUMA: Node 3 [mem 0x5810000000-0x66b7ffffff] + [mem 0x66b8000000-0x755fffffff] -> [mem 0x5810000000-0x755fffffff]
[    0.000000] NODE_DATA(0) allocated [mem 0x128ffd8000-0x128fffefff]
[    0.000000] NODE_DATA(1) allocated [mem 0x2d6ffd8000-0x2d6fffefff]
[    0.000000] NODE_DATA(2) allocated [mem 0x4abffd8000-0x4abfffefff]
[    0.000000] NODE_DATA(3) allocated [mem 0x680ffd6000-0x680fffcfff]
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.000000]   DMA32    [mem 0x0000000001000000-0x00000000ffffffff]
[    0.000000]   Normal   [mem 0x0000000100000000-0x000000755fffffff]
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x0000000000001000-0x000000000008dfff]
[    0.000000]   node   0: [mem 0x0000000000090000-0x000000000009ffff]
[    0.000000]   node   0: [mem 0x0000000000100000-0x000000005cc0afff]
[    0.000000]   node   0: [mem 0x00000000604fd000-0x000000007bafffff]
[    0.000000]   node   0: [mem 0x0000000100000000-0x0000000fffffffff]
[    0.000000]   node   0: [mem 0x0000001000000000-0x000000128fffefff]
[    0.000000]   node   0: [mem 0x0000001300000000-0x0000001cffffffff]
[    0.000000]   node   1: [mem 0x0000001d70000000-0x0000001ec7ffefff]
[    0.000000]   node   1: [mem 0x0000001f00000000-0x0000002bffffffff]
[    0.000000]   node   1: [mem 0x0000002c18000000-0x0000002d6fffefff]
[    0.000000]   node   1: [mem 0x0000002e00000000-0x00000039ffffffff]
[    0.000000]   node   2: [mem 0x0000003ac0000000-0x0000003c17ffefff]
[    0.000000]   node   2: [mem 0x0000003d00000000-0x00000048ffffffff]
[    0.000000]   node   2: [mem 0x0000004968000000-0x0000004abfffefff]
[    0.000000]   node   2: [mem 0x0000004b00000000-0x00000057ffffffff]
[    0.000000]   node   3: [mem 0x0000005810000000-0x0000005967ffefff]
[    0.000000]   node   3: [mem 0x0000005a00000000-0x00000065ffffffff]
[    0.000000]   node   3: [mem 0x00000066b8000000-0x000000680fffefff]
[    0.000000]   node   3: [mem 0x0000006900000000-0x000000755fffffff]
[    0.000000] Initmem setup node 0 [mem 0x0000000000001000-0x0000001cffffffff]
[    0.000000] Initmem setup node 1 [mem 0x0000001d70000000-0x00000039ffffffff]
[    0.000000] Initmem setup node 2 [mem 0x0000003ac0000000-0x00000057ffffffff]
[    0.000000] Initmem setup node 3 [mem 0x0000005810000000-0x000000755fffffff]
[    0.000000] tboot: non-0 tboot_addr but it is not of type E820_RESERVED
[    0.000000] ACPI: PM-Timer IO Port: 0x408
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x00] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x01] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x02] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x03] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x04] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x05] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x06] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x07] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x08] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x09] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x10] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x11] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x12] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x13] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x14] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x15] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x16] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x17] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x18] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x19] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x20] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x21] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x22] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x23] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x24] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x25] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x26] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x27] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x28] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x29] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x30] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x31] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x32] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x33] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x34] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x35] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x36] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x37] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x38] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x39] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x40] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x41] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x42] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x43] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x44] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x45] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x46] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x47] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x48] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x49] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x50] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x51] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x52] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x53] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x54] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x55] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x56] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x57] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x58] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x59] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x60] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x61] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x62] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x63] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x64] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x65] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x66] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x67] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x68] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x69] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x70] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x71] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x72] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x73] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x74] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x75] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x76] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x77] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x78] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x79] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x80] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x81] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x82] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x83] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x84] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x85] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x86] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x87] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x88] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x89] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x90] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x91] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x92] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x93] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x94] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x95] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x96] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x97] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x98] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x99] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa0] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa1] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa2] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa3] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa4] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa5] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa6] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa7] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa8] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa9] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xaa] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xab] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xac] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xad] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xae] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xaf] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb0] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb1] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb2] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb3] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb4] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb5] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb6] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb7] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb8] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb9] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xba] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbb] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbc] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbd] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbe] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbf] high level lint[0x1])
[    0.000000] IOAPIC[0]: apic_id 8, version 32, address 0xfec00000, GSI 0-23
[    0.000000] IOAPIC[1]: apic_id 9, version 32, address 0xfec01000, GSI 24-47
[    0.000000] IOAPIC[2]: apic_id 10, version 32, address 0xfec40000, GSI 48-71
[    0.000000] IOAPIC[3]: apic_id 11, version 32, address 0xfec80000, GSI 72-95
[    0.000000] IOAPIC[4]: apic_id 12, version 32, address 0xfecc0000, GSI 96-119
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a301 base: 0xfed00000
[    0.000000] smpboot: Allowing 192 CPUs, 48 hotplug CPUs
[    0.000000] PM: Registered nosave memory: [mem 0x00000000-0x00000fff]
[    0.000000] PM: Registered nosave memory: [mem 0x0008e000-0x0008ffff]
[    0.000000] PM: Registered nosave memory: [mem 0x000a0000-0x000fffff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b754000-0x5b754fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b75c000-0x5b75cfff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b75d000-0x5b75dfff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b781000-0x5b781fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b782000-0x5b782fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b7a6000-0x5b7a6fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b7a7000-0x5b7a7fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b7ee000-0x5b7eefff]
[    0.000000] PM: Registered nosave memory: [mem 0x5cc0b000-0x5e108fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5e109000-0x6035cfff]
[    0.000000] PM: Registered nosave memory: [mem 0x6035d000-0x604fcfff]
[    0.000000] PM: Registered nosave memory: [mem 0x7bb00000-0x8fffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x90000000-0xfed1bfff]
[    0.000000] PM: Registered nosave memory: [mem 0xfed1c000-0xfed1ffff]
[    0.000000] PM: Registered nosave memory: [mem 0xfed20000-0xffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x128ffff000-0x12ffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x1d00000000-0x1d6fffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x1ec7fff000-0x1effffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x2c00000000-0x2c17ffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x2d6ffff000-0x2dffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x3a00000000-0x3abfffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x3c17fff000-0x3cffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x4900000000-0x4967ffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x4abffff000-0x4affffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x5800000000-0x580fffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x5967fff000-0x59ffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x6600000000-0x66b7ffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x680ffff000-0x68ffffffff]
[    0.000000] e820: [mem 0x90000000-0xfed1bfff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on bare hardware
[    0.000000] clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 1910969940391419 ns
[    0.000000] setup_percpu: NR_CPUS:8192 nr_cpumask_bits:192 nr_cpu_ids:192 nr_node_ids:4
[    0.000000] PERCPU: Embedded 33 pages/cpu @ffff88121ee00000 s96984 r8192 d29992 u262144
[    0.000000] Built 4 zonelists in Node order, mobility grouping on.  Total pages: 113977983
[    0.000000] Policy zone: Normal
[    0.000000] Kernel command line: BOOT_IMAGE=/vmlinuz-4.2.0-rc8 root=/dev/mapper/rhel_brk--bdx--01-root ro rd.lvm.lv=rhel_brk-bdx-01/root rd.lvm.lv=rhel_brk-bdx-01/swap console=tty1 console=ttyS0,115200 LANG=en_US.UTF-8
[    0.000000] log_buf_len individual max cpu contribution: 4096 bytes
[    0.000000] log_buf_len total cpu_extra contributions: 782336 bytes
[    0.000000] log_buf_len min size: 1048576 bytes
[    0.000000] log_buf_len: 2097152 bytes
[    0.000000] early log buf free: 1002168(95%)
[    0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    0.000000] Memory: 455606716K/463341200K available (6685K kernel code, 1324K rwdata, 3220K rodata, 1744K init, 3204K bss, 7734484K reserved, 0K cma-reserved)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=192, Nodes=4
[    0.000000] Hierarchical RCU implementation.
[    0.000000] 	Build-time adjustment of leaf fanout to 64.
[    0.000000] 	RCU restricting CPUs from NR_CPUS=8192 to nr_cpu_ids=192.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=192
[    0.000000] NR_IRQS:524544 nr_irqs:3592 16
[    0.000000] 	Offload RCU callbacks from all CPUs
[    0.000000] 	Offload RCU callbacks from CPUs: 0-191.
[    0.000000] Console: colour dummy device 80x25
[    0.000000] console [tty1] enabled
[    0.000000] console [ttyS0] enabled
[    0.000000] mempolicy: Enabling automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl
[    0.000000] clocksource: hpet: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 133484882848 ns
[    0.000000] tsc: Fast TSC calibration using PIT
[    0.000000] tsc: Detected 1695.766 MHz processor
[    0.000111] Calibrating delay loop (skipped), value calculated using timer frequency.. 3391.53 BogoMIPS (lpj=1695766)
[    0.011991] pid_max: default: 196608 minimum: 1536
[    0.017409] ACPI: Core revision 20150619
[    0.182935] ACPI: All ACPI Tables successfully acquired
[    0.191430] Security Framework initialized
[    0.196015] SELinux:  Initializing.
[    0.253287] Dentry cache hash table entries: 67108864 (order: 17, 536870912 bytes)
[    0.460326] Inode-cache hash table entries: 33554432 (order: 16, 268435456 bytes)
[    0.555488] Mount-cache hash table entries: 1048576 (order: 11, 8388608 bytes)
[    0.564399] Mountpoint-cache hash table entries: 1048576 (order: 11, 8388608 bytes)
[    0.579012] Initializing cgroup subsys blkio
[    0.583793] Initializing cgroup subsys memory
[    0.588747] Initializing cgroup subsys devices
[    0.593720] Initializing cgroup subsys freezer
[    0.598691] Initializing cgroup subsys net_cls
[    0.603664] Initializing cgroup subsys perf_event
[    0.608974] Initializing cgroup subsys hugetlb
[    0.614089] CPU: Physical Processor ID: 0
[    0.618573] CPU: Processor Core ID: 0
[    0.623580] mce: CPU supports 22 MCE banks
[    0.628216] CPU0: Thermal monitoring enabled (TM1)
[    0.633615] process: using mwait in idle threads
[    0.638783] Last level iTLB entries: 4KB 64, 2MB 8, 4MB 8
[    0.644820] Last level dTLB entries: 4KB 64, 2MB 0, 4MB 0, 1GB 4
[    0.652070] Freeing SMP alternatives memory: 24K (ffffffff81cb0000 - ffffffff81cb6000)
[    0.676890] ftrace: allocating 26061 entries in 102 pages
[    0.699118] x2apic: IRQ remapping doesn't support X2APIC mode
[    0.705681] Switched APIC routing to physical flat.
[    0.712023] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.728780] smpboot: CPU0: Genuine Intel(R) CPU 0000 @ 1.70GHz (fam: 06, model: 4f, stepping: 00)
[    0.738794] Performance Events: PEBS fmt2+, 16-deep LBR, Broadwell events, full-width counters, Intel PMU driver.
[    0.750341] ... version:                3
[    0.754816] ... bit width:              48
[    0.759390] ... generic registers:      4
[    0.763865] ... value mask:             0000ffffffffffff
[    0.769799] ... max period:             0000ffffffffffff
[    0.775734] ... fixed-purpose events:   3
[    0.780211] ... event mask:             000000070000000f
[    0.790598] x86: Booting SMP configuration:
[    0.795273] .... node  #0, CPUs:          #1
[    0.821246] NMI watchdog: enabled on all CPUs, permanently consumes one hw-PMU counter.
[    0.830566]    #2   #3   #4   #5   #6   #7   #8   #9  #10  #11  #12  #13  #14  #15  #16  #17
[    0.942434] .... node  #1, CPUs:    #18  #19  #20  #21  #22  #23  #24  #25  #26  #27  #28  #29  #30  #31  #32  #33  #34  #35
[    1.146885] .... node  #2, CPUs:    #36  #37  #38  #39  #40  #41  #42  #43  #44  #45  #46  #47  #48  #49  #50  #51  #52  #53
[    1.351032] .... node  #3, CPUs:    #54  #55  #56  #57  #58  #59  #60  #61  #62  #63  #64  #65  #66  #67  #68  #69  #70  #71
[    1.555045] .... node  #0, CPUs:    #72  #73  #74  #75  #76  #77  #78  #79  #80  #81  #82  #83  #84  #85  #86  #87  #88  #89
[    1.681234] .... node  #1, CPUs:    #90  #91  #92  #93  #94  #95  #96  #97  #98  #99 #100 #101 #102 #103 #104 #105 #106 #107
[    1.807818] .... node  #2, CPUs:   #108 #109 #110 #111 #112 #113 #114 #115 #116 #117 #118 #119 #120 #121 #122 #123 #124 #125
[    1.933394] .... node  #3, CPUs:   #126 #127 #128 #129 #130 #131 #132 #133 #134 #135 #136 #137 #138 #139 #140 #141 #142 #143
[    2.058521] x86: Booted up 4 nodes, 144 CPUs
[    2.063508] smpboot: Total of 144 processors activated (489048.04 BogoMIPS)
[    3.034499] devtmpfs: initialized
[    3.038292] Using 2GB memory block size for large-memory system
[    3.049725] evm: security.selinux
[    3.053428] evm: security.ima
[    3.056743] evm: security.capability
[    3.060875] PM: Registering ACPI NVS region [mem 0x5e109000-0x6035cfff] (35995648 bytes)
[    3.071173] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 1911260446275000 ns
[    3.083139] atomic64_test: passed for x86-64 platform with CX8 and with SSE
[    3.090920] pinctrl core: initialized pinctrl subsystem
[    3.097379] NET: Registered protocol family 16
[    3.108095] cpuidle: using governor menu
[    3.113103] ACPI FADT declares the system doesn't support PCIe ASPM, so disable it
[    3.121562] ACPI: bus type PCI registered
[    3.126041] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[    3.133508] PCI: MMCONFIG for domain 0000 [bus 00-ff] at [mem 0x80000000-0x8fffffff] (base 0x80000000)
[    3.143909] PCI: MMCONFIG at [mem 0x80000000-0x8fffffff] reserved in E820
[    3.151514] PCI: Using configuration type 1 for base access
[    3.158011] BUG: unable to handle kernel paging request at ffffea0074000020
[    3.165808] IP: [<ffffffff81670527>] get_nid_for_pfn+0x17/0x40
[    3.172344] PGD 128ffcb067 PUD 128ffc9067 PMD 0 
[    3.177539] Oops: 0000 [#1] SMP 
[    3.181166] Modules linked in:
[    3.184583] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.2.0-rc8 #1
[    3.191488] Hardware name: Intel Corporation BRICKLAND/BRICKLAND, BIOS BRBDXSD1.86B.0326.D03.1508171454 08/17/2015
[    3.203048] task: ffff881cf3618000 ti: ffff881cf3640000 task.ti: ffff881cf3640000
[    3.211408] RIP: 0010:[<ffffffff81670527>]  [<ffffffff81670527>] get_nid_for_pfn+0x17/0x40
[    3.220657] RSP: 0000:ffff881cf3643d38  EFLAGS: 00010286
[    3.226590] RAX: ffffea0000000000 RBX: 0000000001d00000 RCX: 0000000000000000
[    3.234559] RDX: 00000000ffffffff RSI: 0000000000000001 RDI: ffffea0074000000
[    3.242519] RBP: ffff881cf3643d78 R08: 00000000000003ae R09: 0000000003a00000
[    3.250487] R10: ffffffff81317817 R11: ffffea015fc99c00 R12: 0000000000000001
[    3.258453] R13: 0000000001d7ffff R14: ffff8839f2495800 R15: 0000000000000001
[    3.266421] FS:  0000000000000000(0000) GS:ffff88121ee00000(0000) knlGS:0000000000000000
[    3.275457] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[    3.281865] CR2: ffffea0074000020 CR3: 00000000019bc000 CR4: 00000000003406f0
[    3.289824] Stack:
[    3.292071]  ffff881cf3643d78 ffffffff81453b56 0000000000000001 0000000001d70000
[    3.300374]  0000000000000001 0000000000000001 ffff8839f2495800 0000000000000000
[    3.308673]  ffff881cf3643dc8 ffffffff81453eeb ffff881cf3643de8 0000000003a00000
[    3.316973] Call Trace:
[    3.319711]  [<ffffffff81453b56>] ? register_mem_sect_under_node+0x66/0xe0
[    3.327388]  [<ffffffff81453eeb>] register_one_node+0x17b/0x240
[    3.334001]  [<ffffffff81b1f1ed>] ? pci_iommu_alloc+0x6e/0x6e
[    3.340418]  [<ffffffff81b1f229>] topology_init+0x3c/0x95
[    3.346446]  [<ffffffff8100213d>] do_one_initcall+0xcd/0x1f0
[    3.352773]  [<ffffffff8109f500>] ? parse_args+0x200/0x470
[    3.358905]  [<ffffffff810c3fa8>] ? __wake_up+0x48/0x60
[    3.364746]  [<ffffffff81b15342>] kernel_init_freeable+0x1a5/0x249
[    3.371650]  [<ffffffff81b149dd>] ? initcall_blacklist+0xb6/0xb6
[    3.378365]  [<ffffffff8166c610>] ? rest_init+0x80/0x80
[    3.384198]  [<ffffffff8166c61e>] kernel_init+0xe/0xe0
[    3.389945]  [<ffffffff81682a9f>] ret_from_fork+0x3f/0x70
[    3.395965]  [<ffffffff8166c610>] ? rest_init+0x80/0x80
[    3.401798] Code: 5c 41 5d 41 5e 41 5f 5d c3 41 be ed ff ff ff eb e6 0f 1f 00 0f 1f 44 00 00 48 c1 e7 06 48 b8 00 00 00 00 00 ea ff ff 55 48 01 c7 <48> 83 7f 20 00 48 89 e5 74 09 48 8b 07 5d 48 c1 e8 36 c3 b8 ff 
[    3.423506] RIP  [<ffffffff81670527>] get_nid_for_pfn+0x17/0x40
[    3.430128]  RSP <ffff881cf3643d38>
[    3.434022] CR2: ffffea0074000020
[    3.437726] ---[ end trace 3395ea2cce2f2a27 ]---
[    3.442882] Kernel panic - not syncing: Fatal exception
[    3.448760] ---[ end Kernel panic - not syncing: Fatal exception

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

On Mon, Aug 24, 2015 at 1:41 PM, Tony Luck <tony.luck@gmail.com> wrote:
> On Mon, Aug 24, 2015 at 10:46 AM, Yinghai Lu <yinghai@kernel.org> wrote:
>> Then, what does the E820 look like?
>
> See attached serial console log of the latest crash

Can you boot with "debug ignore_loglevel" so we can see following print out
for vmemmap?

[    0.352486]  [ffffea0000000000-ffffea0001ffffff] PMD ->
[ffff88007de00000-ffff88007fdfffff] on node 0
[    0.358758]  [ffffea0004000000-ffffea0005ffffff] PMD ->
[ffff88017d600000-ffff88017f5fffff] on node 1

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Tony Luck]

[-- Attachment #1: Type: text/plain, Size: 763 bytes --]

On Mon, Aug 24, 2015 at 2:25 PM, Yinghai Lu <yinghai@kernel.org> wrote:

> Can you boot with "debug ignore_loglevel" so we can see following print out
> for vmemmap?

See attached. There are a few extra messages from my own debug printk()
calls. It seems that we successfully deal with node 0 from topology_init()
but die walking node 1. I see that the NODE_DATA limits for memory
on node 1 were from 1d70000 to 3a00000. But when we get into
register_mem_sect_under_node() we have rounded the start pfn down to
1d00000 ... and we panic processing that range (which is in a hole in e820).

We seem to die here:

        for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
                int page_nid;

                page_nid = get_nid_for_pfn(pfn);

-Tony

[-- Attachment #2: dmesg2 --]
[-- Type: application/octet-stream, Size: 58798 bytes --]

[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Initializing cgroup subsys cpuacct
[    0.000000] Linux version 4.2.0-rc8 (aegl@brk-bdx-01.sc.intel.com) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-4) (GCC) ) #2 SMP Mon Aug 24 14:22:56 PDT 2015
[    0.000000] Command line: BOOT_IMAGE=/vmlinuz-4.2.0-rc8 root=/dev/mapper/rhel_brk--bdx--01-root ro rd.lvm.lv=rhel_brk-bdx-01/root rd.lvm.lv=rhel_brk-bdx-01/swap debug ignore_loglevel console=tty1 console=ttyS0,115200 LANG=en_US.UTF-8
[    0.000000] x86/fpu: xstate_offset[2]: 0240, xstate_sizes[2]: 0100
[    0.000000] x86/fpu: Supporting XSAVE feature 0x01: 'x87 floating point registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x02: 'SSE registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x04: 'AVX registers'
[    0.000000] x86/fpu: Enabled xstate features 0x7, context size is 0x340 bytes, using 'standard' format.
[    0.000000] x86/fpu: Using 'eager' FPU context switches.
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000008dfff] usable
[    0.000000] BIOS-e820: [mem 0x000000000008e000-0x000000000008ffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000090000-0x000000000009ffff] usable
[    0.000000] BIOS-e820: [mem 0x00000000000a0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000005cc0afff] usable
[    0.000000] BIOS-e820: [mem 0x000000005cc0b000-0x000000005e108fff] reserved
[    0.000000] BIOS-e820: [mem 0x000000005e109000-0x000000006035cfff] ACPI NVS
[    0.000000] BIOS-e820: [mem 0x000000006035d000-0x00000000604fcfff] ACPI data
[    0.000000] BIOS-e820: [mem 0x00000000604fd000-0x000000007bafffff] usable
[    0.000000] BIOS-e820: [mem 0x000000007bb00000-0x000000008fffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fed1c000-0x00000000fed1ffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000100000000-0x000000128fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000001300000000-0x0000001cffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000001d70000000-0x0000001ec7ffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000001f00000000-0x0000002bffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000002c18000000-0x0000002d6fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000002e00000000-0x00000039ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000003ac0000000-0x0000003c17ffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000003d00000000-0x00000048ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000004968000000-0x0000004abfffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000004b00000000-0x00000057ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x0000005810000000-0x0000005967ffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000005a00000000-0x00000065ffffffff] usable
[    0.000000] BIOS-e820: [mem 0x00000066b8000000-0x000000680fffefff] usable
[    0.000000] BIOS-e820: [mem 0x0000006900000000-0x000000755fffffff] usable
[    0.000000] debug: ignoring loglevel setting.
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] e820: update [mem 0x5b7a7018-0x5b7ee457] usable ==> usable
[    0.000000] e820: update [mem 0x5b782018-0x5b7a6e57] usable ==> usable
[    0.000000] e820: update [mem 0x5b75d018-0x5b781e57] usable ==> usable
[    0.000000] e820: update [mem 0x5b754018-0x5b75c057] usable ==> usable
[    0.000000] extended physical RAM map:
[    0.000000] reserve setup_data: [mem 0x0000000000000000-0x000000000008dfff] usable
[    0.000000] reserve setup_data: [mem 0x000000000008e000-0x000000000008ffff] reserved
[    0.000000] reserve setup_data: [mem 0x0000000000090000-0x000000000009ffff] usable
[    0.000000] reserve setup_data: [mem 0x00000000000a0000-0x00000000000fffff] reserved
[    0.000000] reserve setup_data: [mem 0x0000000000100000-0x000000005b754017] usable
[    0.000000] reserve setup_data: [mem 0x000000005b754018-0x000000005b75c057] usable
[    0.000000] reserve setup_data: [mem 0x000000005b75c058-0x000000005b75d017] usable
[    0.000000] reserve setup_data: [mem 0x000000005b75d018-0x000000005b781e57] usable
[    0.000000] reserve setup_data: [mem 0x000000005b781e58-0x000000005b782017] usable
[    0.000000] reserve setup_data: [mem 0x000000005b782018-0x000000005b7a6e57] usable
[    0.000000] reserve setup_data: [mem 0x000000005b7a6e58-0x000000005b7a7017] usable
[    0.000000] reserve setup_data: [mem 0x000000005b7a7018-0x000000005b7ee457] usable
[    0.000000] reserve setup_data: [mem 0x000000005b7ee458-0x000000005cc0afff] usable
[    0.000000] reserve setup_data: [mem 0x000000005cc0b000-0x000000005e108fff] reserved
[    0.000000] reserve setup_data: [mem 0x000000005e109000-0x000000006035cfff] ACPI NVS
[    0.000000] reserve setup_data: [mem 0x000000006035d000-0x00000000604fcfff] ACPI data
[    0.000000] reserve setup_data: [mem 0x00000000604fd000-0x000000007bafffff] usable
[    0.000000] reserve setup_data: [mem 0x000000007bb00000-0x000000008fffffff] reserved
[    0.000000] reserve setup_data: [mem 0x00000000fed1c000-0x00000000fed1ffff] reserved
[    0.000000] reserve setup_data: [mem 0x0000000100000000-0x000000128fffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000001300000000-0x0000001cffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000001d70000000-0x0000001ec7ffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000001f00000000-0x0000002bffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000002c18000000-0x0000002d6fffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000002e00000000-0x00000039ffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000003ac0000000-0x0000003c17ffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000003d00000000-0x00000048ffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000004968000000-0x0000004abfffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000004b00000000-0x00000057ffffffff] usable
[    0.000000] reserve setup_data: [mem 0x0000005810000000-0x0000005967ffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000005a00000000-0x00000065ffffffff] usable
[    0.000000] reserve setup_data: [mem 0x00000066b8000000-0x000000680fffefff] usable
[    0.000000] reserve setup_data: [mem 0x0000006900000000-0x000000755fffffff] usable
[    0.000000] efi: EFI v2.40 by EDK II
[    0.000000]                              BIOS ID: BRBDXSD1.86B.0326.D03.1508171454
[    0.000000] efi:  ACPI=0x604fc000  ACPI 2.0=0x604fc014  SMBIOS=0x5d007000 
[    0.000000] SMBIOS 3.0 present.
[    0.000000] DMI: Intel Corporation BRICKLAND/BRICKLAND, BIOS BRBDXSD1.86B.0326.D03.1508171454 08/17/2015
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x7560000 max_arch_pfn = 0x400000000
[    0.000000] MTRR default type: write-back
[    0.000000] MTRR fixed ranges enabled:
[    0.000000]   00000-9FFFF write-back
[    0.000000]   A0000-BFFFF uncachable
[    0.000000]   C0000-DFFFF write-protect
[    0.000000]   E0000-FFFFF uncachable
[    0.000000] MTRR variable ranges enabled:
[    0.000000]   0 base 000080000000 mask 3FFF80000000 uncachable
[    0.000000]   1 base 380000000000 mask 3F8000000000 uncachable
[    0.000000]   2 base 00007C000000 mask 3FFFFC000000 uncachable
[    0.000000]   3 base 00007FC00000 mask 3FFFFFC00000 uncachable
[    0.000000]   4 disabled
[    0.000000]   5 disabled
[    0.000000]   6 disabled
[    0.000000]   7 disabled
[    0.000000]   8 disabled
[    0.000000]   9 disabled
[    0.000000] x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WC  UC- WT  
[    0.000000] e820: last_pfn = 0x7bb00 max_arch_pfn = 0x400000000
[    0.000000] efi: Memory: 45441M/452863M mirrored memory
[    0.000000] Base memory trampoline at [ffff880000099000] 99000 size 24576
[    0.000000] Using GB pages for direct mapping
[    0.000000] init_memory_mapping: [mem 0x00000000-0x000fffff]
[    0.000000]  [mem 0x00000000-0x000fffff] page 4k
[    0.000000] BRK [0x01fda000, 0x01fdafff] PGTABLE
[    0.000000] BRK [0x01fdb000, 0x01fdbfff] PGTABLE
[    0.000000] BRK [0x01fdc000, 0x01fdcfff] PGTABLE
[    0.000000] init_memory_mapping: [mem 0x680fc00000-0x680fdfffff]
[    0.000000]  [mem 0x680fc00000-0x680fdfffff] page 2M
[    0.000000] BRK [0x01fdd000, 0x01fddfff] PGTABLE
[    0.000000] init_memory_mapping: [mem 0x6800000000-0x680fbfffff]
[    0.000000]  [mem 0x6800000000-0x680fbfffff] page 2M
[    0.000000] init_memory_mapping: [mem 0x67e0000000-0x67ffffffff]
[    0.000000]  [mem 0x67e0000000-0x67ffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x6000000000-0x65ffffffff]
[    0.000000]  [mem 0x6000000000-0x65ffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x66b8000000-0x67dfffffff]
[    0.000000]  [mem 0x66b8000000-0x66bfffffff] page 2M
[    0.000000]  [mem 0x66c0000000-0x67dfffffff] page 1G
[    0.000000] BRK [0x01fde000, 0x01fdefff] PGTABLE
[    0.000000] init_memory_mapping: [mem 0x4000000000-0x48ffffffff]
[    0.000000]  [mem 0x4000000000-0x48ffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x4968000000-0x4abfffefff]
[    0.000000]  [mem 0x4968000000-0x497fffffff] page 2M
[    0.000000]  [mem 0x4980000000-0x4a7fffffff] page 1G
[    0.000000]  [mem 0x4a80000000-0x4abfdfffff] page 2M
[    0.000000]  [mem 0x4abfe00000-0x4abfffefff] page 4k
[    0.000000] BRK [0x01fdf000, 0x01fdffff] PGTABLE
[    0.000000] init_memory_mapping: [mem 0x4b00000000-0x57ffffffff]
[    0.000000]  [mem 0x4b00000000-0x57ffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x5810000000-0x5967ffefff]
[    0.000000]  [mem 0x5810000000-0x583fffffff] page 2M
[    0.000000]  [mem 0x5840000000-0x593fffffff] page 1G
[    0.000000]  [mem 0x5940000000-0x5967dfffff] page 2M
[    0.000000]  [mem 0x5967e00000-0x5967ffefff] page 4k
[    0.000000] init_memory_mapping: [mem 0x5a00000000-0x5fffffffff]
[    0.000000]  [mem 0x5a00000000-0x5fffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x00100000-0x5cc0afff]
[    0.000000]  [mem 0x00100000-0x001fffff] page 4k
[    0.000000]  [mem 0x00200000-0x5cbfffff] page 2M
[    0.000000]  [mem 0x5cc00000-0x5cc0afff] page 4k
[    0.000000] init_memory_mapping: [mem 0x604fd000-0x7bafffff]
[    0.000000]  [mem 0x604fd000-0x605fffff] page 4k
[    0.000000]  [mem 0x60600000-0x7b9fffff] page 2M
[    0.000000]  [mem 0x7ba00000-0x7bafffff] page 4k
[    0.000000] init_memory_mapping: [mem 0x100000000-0xfffffffff]
[    0.000000]  [mem 0x100000000-0xfffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x1000000000-0x128fffefff]
[    0.000000]  [mem 0x1000000000-0x127fffffff] page 1G
[    0.000000]  [mem 0x1280000000-0x128fdfffff] page 2M
[    0.000000]  [mem 0x128fe00000-0x128fffefff] page 4k
[    0.000000] init_memory_mapping: [mem 0x1300000000-0x1cffffffff]
[    0.000000]  [mem 0x1300000000-0x1cffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x1d70000000-0x1ec7ffefff]
[    0.000000]  [mem 0x1d70000000-0x1d7fffffff] page 2M
[    0.000000]  [mem 0x1d80000000-0x1ebfffffff] page 1G
[    0.000000]  [mem 0x1ec0000000-0x1ec7dfffff] page 2M
[    0.000000]  [mem 0x1ec7e00000-0x1ec7ffefff] page 4k
[    0.000000] init_memory_mapping: [mem 0x1f00000000-0x2bffffffff]
[    0.000000]  [mem 0x1f00000000-0x2bffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x2c18000000-0x2d6fffefff]
[    0.000000]  [mem 0x2c18000000-0x2c3fffffff] page 2M
[    0.000000]  [mem 0x2c40000000-0x2d3fffffff] page 1G
[    0.000000]  [mem 0x2d40000000-0x2d6fdfffff] page 2M
[    0.000000]  [mem 0x2d6fe00000-0x2d6fffefff] page 4k
[    0.000000] init_memory_mapping: [mem 0x2e00000000-0x39ffffffff]
[    0.000000]  [mem 0x2e00000000-0x39ffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x3ac0000000-0x3c17ffefff]
[    0.000000]  [mem 0x3ac0000000-0x3bffffffff] page 1G
[    0.000000]  [mem 0x3c00000000-0x3c17dfffff] page 2M
[    0.000000]  [mem 0x3c17e00000-0x3c17ffefff] page 4k
[    0.000000] init_memory_mapping: [mem 0x3d00000000-0x3fffffffff]
[    0.000000]  [mem 0x3d00000000-0x3fffffffff] page 1G
[    0.000000] init_memory_mapping: [mem 0x680fe00000-0x680fffefff]
[    0.000000]  [mem 0x680fe00000-0x680fffefff] page 4k
[    0.000000] init_memory_mapping: [mem 0x6900000000-0x755fffffff]
[    0.000000]  [mem 0x6900000000-0x753fffffff] page 1G
[    0.000000]  [mem 0x7540000000-0x755fffffff] page 2M
[    0.000000] RAMDISK: [mem 0x2bbb0000-0x2ecf0fff]
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000604FC014 000024 (v02 INTEL )
[    0.000000] ACPI: XSDT 0x00000000604FB0E8 0000BC (v01 INTEL  TIANO    00000000      01000013)
[    0.000000] ACPI: FACP 0x00000000604F8000 0000F4 (v04 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: DSDT 0x00000000604C6000 02503B (v02 INTEL  TIANO    00000003 MSFT 01000013)
[    0.000000] ACPI: FACS 0x000000005EABC000 000040
[    0.000000] ACPI: TCPA 0x00000000604FA000 000064 (v02 INTEL  EDK2     00000002      01000013)
[    0.000000] ACPI: BDAT 0x00000000604F9000 000030 (v01 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: HPET 0x00000000604F7000 000038 (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: APIC 0x00000000604F6000 000AFC (v03 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: MCFG 0x00000000604F5000 00003C (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: MSCT 0x00000000604F4000 000090 (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: PCCT 0x00000000604F3000 0000AC (v01 INTEL  TIANO    00000002 MSFT 01000013)
[    0.000000] ACPI: PMCT 0x00000000604F2000 000060 (v01 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: RASF 0x00000000604F1000 000030 (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: SLIT 0x00000000604F0000 00006C (v01 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: SRAT 0x00000000604EE000 001130 (v03 INTEL  TIANO    00000001 MSFT 01000013)
[    0.000000] ACPI: SVOS 0x00000000604ED000 000032 (v01 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: WDDT 0x00000000604EC000 000040 (v01 INTEL  TIANO    00000000 MSFT 01000013)
[    0.000000] ACPI: SSDT 0x00000000603B6000 10F622 (v02 INTEL  SSDT  PM 00004000 INTL 20090521)
[    0.000000] ACPI: SSDT 0x00000000603B3000 00246F (v02 INTEL  SpsNm    00000002 INTL 20090521)
[    0.000000] ACPI: SSDT 0x00000000603B2000 000040 (v02 INTEL  SpsNvs   00000002 INTL 20090521)
[    0.000000] ACPI: PRAD 0x00000000603B1000 000125 (v02 INTEL  SpsPrAgg 00000002 INTL 20090521)
[    0.000000] ACPI: SPCR 0x00000000603B0000 000050 (v01                 00000000      00000000)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] SRAT: PXM 0 -> APIC 0x00 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x02 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x04 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x06 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x08 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x10 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x12 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x14 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x16 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x20 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x22 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x24 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x26 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x28 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x30 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x32 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x34 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x36 -> Node 0
[    0.000000] SRAT: PXM 1 -> APIC 0x40 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x42 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x44 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x46 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x48 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x50 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x52 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x54 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x56 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x60 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x62 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x64 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x66 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x68 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x70 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x72 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x74 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x76 -> Node 1
[    0.000000] SRAT: PXM 2 -> APIC 0x80 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x82 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x84 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x86 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x88 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x90 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x92 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x94 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x96 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa0 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa2 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa4 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa6 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa8 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb0 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb2 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb4 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb6 -> Node 2
[    0.000000] SRAT: PXM 3 -> APIC 0xc0 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc2 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc4 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc6 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc8 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd0 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd2 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd4 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd6 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe0 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe2 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe4 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe6 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe8 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf0 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf2 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf4 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf6 -> Node 3
[    0.000000] SRAT: PXM 0 -> APIC 0x01 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x03 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x05 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x07 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x09 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x11 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x13 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x15 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x17 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x21 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x23 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x25 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x27 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x29 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x31 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x33 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x35 -> Node 0
[    0.000000] SRAT: PXM 0 -> APIC 0x37 -> Node 0
[    0.000000] SRAT: PXM 1 -> APIC 0x41 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x43 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x45 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x47 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x49 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x51 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x53 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x55 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x57 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x61 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x63 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x65 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x67 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x69 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x71 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x73 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x75 -> Node 1
[    0.000000] SRAT: PXM 1 -> APIC 0x77 -> Node 1
[    0.000000] SRAT: PXM 2 -> APIC 0x81 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x83 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x85 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x87 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x89 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x91 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x93 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x95 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0x97 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa1 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa3 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa5 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa7 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xa9 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb1 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb3 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb5 -> Node 2
[    0.000000] SRAT: PXM 2 -> APIC 0xb7 -> Node 2
[    0.000000] SRAT: PXM 3 -> APIC 0xc1 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc3 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc5 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc7 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xc9 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd1 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd3 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd5 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xd7 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe1 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe3 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe5 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe7 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xe9 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf1 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf3 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf5 -> Node 3
[    0.000000] SRAT: PXM 3 -> APIC 0xf7 -> Node 3
[    0.000000] SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
[    0.000000] SRAT: Node 0 PXM 0 [mem 0x100000000-0xfffffffff]
[    0.000000] SRAT: Node 0 PXM 0 [mem 0x1000000000-0x1d6fffffff]
[    0.000000] SRAT: Node 1 PXM 1 [mem 0x1d70000000-0x2c17ffffff]
[    0.000000] SRAT: Node 1 PXM 1 [mem 0x2c18000000-0x3abfffffff]
[    0.000000] SRAT: Node 2 PXM 2 [mem 0x3ac0000000-0x4967ffffff]
[    0.000000] SRAT: Node 2 PXM 2 [mem 0x4968000000-0x580fffffff]
[    0.000000] SRAT: Node 3 PXM 3 [mem 0x5810000000-0x66b7ffffff]
[    0.000000] SRAT: Node 3 PXM 3 [mem 0x66b8000000-0x755fffffff]
[    0.000000] NUMA: Initialized distance table, cnt=4
[    0.000000] NUMA: Node 0 [mem 0x00000000-0x7fffffff] + [mem 0x100000000-0xfffffffff] -> [mem 0x00000000-0xfffffffff]
[    0.000000] NUMA: Node 0 [mem 0x00000000-0xfffffffff] + [mem 0x1000000000-0x1d6fffffff] -> [mem 0x00000000-0x1d6fffffff]
[    0.000000] NUMA: Node 1 [mem 0x1d70000000-0x2c17ffffff] + [mem 0x2c18000000-0x3abfffffff] -> [mem 0x1d70000000-0x3abfffffff]
[    0.000000] NUMA: Node 2 [mem 0x3ac0000000-0x4967ffffff] + [mem 0x4968000000-0x580fffffff] -> [mem 0x3ac0000000-0x580fffffff]
[    0.000000] NUMA: Node 3 [mem 0x5810000000-0x66b7ffffff] + [mem 0x66b8000000-0x755fffffff] -> [mem 0x5810000000-0x755fffffff]
[    0.000000] NODE_DATA(0) allocated [mem 0x128ffd8000-0x128fffefff]
[    0.000000] NODE_DATA(1) allocated [mem 0x2d6ffd8000-0x2d6fffefff]
[    0.000000] NODE_DATA(2) allocated [mem 0x4abffd8000-0x4abfffefff]
[    0.000000] NODE_DATA(3) allocated [mem 0x680ffd6000-0x680fffcfff]
[    0.000000]  [ffffea0000000000-ffffea0073ffffff] PMD -> [ffff88121fa00000-ffff88128fdfffff] on node 0
[    0.000000]  [ffffea0075c00000-ffffea00e7ffffff] PMD -> [ffff882d01200000-ffff882d6fdfffff] on node 1
[    0.000000]  [ffffea00eb000000-ffffea015fffffff] PMD -> [ffff884a51200000-ffff884abfdfffff] on node 2
[    0.000000]  [ffffea0160400000-ffffea01d57fffff] PMD -> [ffff8867a3000000-ffff88680f3fffff] on node 3
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.000000]   DMA32    [mem 0x0000000001000000-0x00000000ffffffff]
[    0.000000]   Normal   [mem 0x0000000100000000-0x000000755fffffff]
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x0000000000001000-0x000000000008dfff]
[    0.000000]   node   0: [mem 0x0000000000090000-0x000000000009ffff]
[    0.000000]   node   0: [mem 0x0000000000100000-0x000000005cc0afff]
[    0.000000]   node   0: [mem 0x00000000604fd000-0x000000007bafffff]
[    0.000000]   node   0: [mem 0x0000000100000000-0x0000000fffffffff]
[    0.000000]   node   0: [mem 0x0000001000000000-0x000000128fffefff]
[    0.000000]   node   0: [mem 0x0000001300000000-0x0000001cffffffff]
[    0.000000]   node   1: [mem 0x0000001d70000000-0x0000001ec7ffefff]
[    0.000000]   node   1: [mem 0x0000001f00000000-0x0000002bffffffff]
[    0.000000]   node   1: [mem 0x0000002c18000000-0x0000002d6fffefff]
[    0.000000]   node   1: [mem 0x0000002e00000000-0x00000039ffffffff]
[    0.000000]   node   2: [mem 0x0000003ac0000000-0x0000003c17ffefff]
[    0.000000]   node   2: [mem 0x0000003d00000000-0x00000048ffffffff]
[    0.000000]   node   2: [mem 0x0000004968000000-0x0000004abfffefff]
[    0.000000]   node   2: [mem 0x0000004b00000000-0x00000057ffffffff]
[    0.000000]   node   3: [mem 0x0000005810000000-0x0000005967ffefff]
[    0.000000]   node   3: [mem 0x0000005a00000000-0x00000065ffffffff]
[    0.000000]   node   3: [mem 0x00000066b8000000-0x000000680fffefff]
[    0.000000]   node   3: [mem 0x0000006900000000-0x000000755fffffff]
[    0.000000] Initmem setup node 0 [mem 0x0000000000001000-0x0000001cffffffff]
[    0.000000] On node 0 totalpages: 29393322
[    0.000000]   DMA zone: 64 pages used for memmap
[    0.000000]   DMA zone: 284 pages reserved
[    0.000000]   DMA zone: 3997 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 7625 pages used for memmap
[    0.000000]   DMA32 zone: 487950 pages, LIFO batch:31
[    0.000000]   Normal zone: 458752 pages used for memmap
[    0.000000]   Normal zone: 28901375 pages, LIFO batch:31
[    0.000000] Initmem setup node 1 [mem 0x0000001d70000000-0x00000039ffffffff]
[    0.000000] On node 1 totalpages: 29032446
[    0.000000]   Normal zone: 467968 pages used for memmap
[    0.000000]   Normal zone: 29032446 pages, LIFO batch:31
[    0.000000] Initmem setup node 2 [mem 0x0000003ac0000000-0x00000057ffffffff]
[    0.000000] On node 2 totalpages: 29032446
[    0.000000]   Normal zone: 479232 pages used for memmap
[    0.000000]   Normal zone: 29032446 pages, LIFO batch:31
[    0.000000] Initmem setup node 3 [mem 0x0000005810000000-0x000000755fffffff]
[    0.000000] On node 3 totalpages: 28377086
[    0.000000]   Normal zone: 443392 pages used for memmap
[    0.000000]   Normal zone: 28377086 pages, LIFO batch:31
[    0.000000] tboot: non-0 tboot_addr but it is not of type E820_RESERVED
[    0.000000] ACPI: PM-Timer IO Port: 0x408
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x00] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x01] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x02] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x03] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x04] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x05] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x06] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x07] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x08] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x09] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x0f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x10] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x11] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x12] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x13] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x14] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x15] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x16] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x17] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x18] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x19] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x1f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x20] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x21] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x22] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x23] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x24] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x25] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x26] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x27] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x28] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x29] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x2f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x30] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x31] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x32] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x33] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x34] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x35] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x36] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x37] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x38] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x39] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x3f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x40] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x41] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x42] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x43] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x44] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x45] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x46] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x47] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x48] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x49] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x4f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x50] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x51] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x52] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x53] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x54] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x55] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x56] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x57] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x58] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x59] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x5f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x60] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x61] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x62] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x63] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x64] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x65] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x66] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x67] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x68] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x69] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x6f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x70] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x71] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x72] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x73] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x74] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x75] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x76] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x77] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x78] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x79] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x7f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x80] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x81] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x82] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x83] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x84] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x85] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x86] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x87] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x88] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x89] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x8f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x90] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x91] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x92] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x93] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x94] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x95] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x96] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x97] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x98] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x99] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9a] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9b] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9c] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9d] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9e] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0x9f] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa0] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa1] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa2] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa3] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa4] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa5] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa6] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa7] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa8] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xa9] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xaa] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xab] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xac] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xad] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xae] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xaf] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb0] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb1] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb2] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb3] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb4] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb5] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb6] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb7] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb8] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xb9] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xba] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbb] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbc] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbd] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbe] high level lint[0x1])
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xbf] high level lint[0x1])
[    0.000000] IOAPIC[0]: apic_id 8, version 32, address 0xfec00000, GSI 0-23
[    0.000000] IOAPIC[1]: apic_id 9, version 32, address 0xfec01000, GSI 24-47
[    0.000000] IOAPIC[2]: apic_id 10, version 32, address 0xfec40000, GSI 48-71
[    0.000000] IOAPIC[3]: apic_id 11, version 32, address 0xfec80000, GSI 72-95
[    0.000000] IOAPIC[4]: apic_id 12, version 32, address 0xfecc0000, GSI 96-119
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a301 base: 0xfed00000
[    0.000000] smpboot: Allowing 192 CPUs, 48 hotplug CPUs
[    0.000000] PM: Registered nosave memory: [mem 0x00000000-0x00000fff]
[    0.000000] PM: Registered nosave memory: [mem 0x0008e000-0x0008ffff]
[    0.000000] PM: Registered nosave memory: [mem 0x000a0000-0x000fffff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b754000-0x5b754fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b75c000-0x5b75cfff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b75d000-0x5b75dfff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b781000-0x5b781fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b782000-0x5b782fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b7a6000-0x5b7a6fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b7a7000-0x5b7a7fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5b7ee000-0x5b7eefff]
[    0.000000] PM: Registered nosave memory: [mem 0x5cc0b000-0x5e108fff]
[    0.000000] PM: Registered nosave memory: [mem 0x5e109000-0x6035cfff]
[    0.000000] PM: Registered nosave memory: [mem 0x6035d000-0x604fcfff]
[    0.000000] PM: Registered nosave memory: [mem 0x7bb00000-0x8fffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x90000000-0xfed1bfff]
[    0.000000] PM: Registered nosave memory: [mem 0xfed1c000-0xfed1ffff]
[    0.000000] PM: Registered nosave memory: [mem 0xfed20000-0xffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x128ffff000-0x12ffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x1d00000000-0x1d6fffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x1ec7fff000-0x1effffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x2c00000000-0x2c17ffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x2d6ffff000-0x2dffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x3a00000000-0x3abfffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x3c17fff000-0x3cffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x4900000000-0x4967ffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x4abffff000-0x4affffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x5800000000-0x580fffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x5967fff000-0x59ffffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x6600000000-0x66b7ffffff]
[    0.000000] PM: Registered nosave memory: [mem 0x680ffff000-0x68ffffffff]
[    0.000000] e820: [mem 0x90000000-0xfed1bfff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on bare hardware
[    0.000000] clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 1910969940391419 ns
[    0.000000] setup_percpu: NR_CPUS:8192 nr_cpumask_bits:192 nr_cpu_ids:192 nr_node_ids:4
[    0.000000] PERCPU: Embedded 33 pages/cpu @ffff88121ee00000 s96984 r8192 d29992 u262144
[    0.000000] pcpu-alloc: s96984 r8192 d29992 u262144 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 000 001 002 003 004 005 006 007 
[    0.000000] pcpu-alloc: [0] 008 009 010 011 012 013 014 015 
[    0.000000] pcpu-alloc: [0] 016 017 072 073 074 075 076 077 
[    0.000000] pcpu-alloc: [0] 078 079 080 081 082 083 084 085 
[    0.000000] pcpu-alloc: [0] 086 087 088 089 144 148 152 156 
[    0.000000] pcpu-alloc: [0] 160 164 168 172 176 180 184 188 
[    0.000000] pcpu-alloc: [1] 018 019 020 021 022 023 024 025 
[    0.000000] pcpu-alloc: [1] 026 027 028 029 030 031 032 033 
[    0.000000] pcpu-alloc: [1] 034 035 090 091 092 093 094 095 
[    0.000000] pcpu-alloc: [1] 096 097 098 099 100 101 102 103 
[    0.000000] pcpu-alloc: [1] 104 105 106 107 145 149 153 157 
[    0.000000] pcpu-alloc: [1] 161 165 169 173 177 181 185 189 
[    0.000000] pcpu-alloc: [2] 036 037 038 039 040 041 042 043 
[    0.000000] pcpu-alloc: [2] 044 045 046 047 048 049 050 051 
[    0.000000] pcpu-alloc: [2] 052 053 108 109 110 111 112 113 
[    0.000000] pcpu-alloc: [2] 114 115 116 117 118 119 120 121 
[    0.000000] pcpu-alloc: [2] 122 123 124 125 146 150 154 158 
[    0.000000] pcpu-alloc: [2] 162 166 170 174 178 182 186 190 
[    0.000000] pcpu-alloc: [3] 054 055 056 057 058 059 060 061 
[    0.000000] pcpu-alloc: [3] 062 063 064 065 066 067 068 069 
[    0.000000] pcpu-alloc: [3] 070 071 126 127 128 129 130 131 
[    0.000000] pcpu-alloc: [3] 132 133 134 135 136 137 138 139 
[    0.000000] pcpu-alloc: [3] 140 141 142 143 147 151 155 159 
[    0.000000] pcpu-alloc: [3] 163 167 171 175 179 183 187 191 
[    0.000000] Built 4 zonelists in Node order, mobility grouping on.  Total pages: 113977983
[    0.000000] Policy zone: Normal
[    0.000000] Kernel command line: BOOT_IMAGE=/vmlinuz-4.2.0-rc8 root=/dev/mapper/rhel_brk--bdx--01-root ro rd.lvm.lv=rhel_brk-bdx-01/root rd.lvm.lv=rhel_brk-bdx-01/swap debug ignore_loglevel console=tty1 console=ttyS0,115200 LANG=en_US.UTF-8
[    0.000000] log_buf_len individual max cpu contribution: 4096 bytes
[    0.000000] log_buf_len total cpu_extra contributions: 782336 bytes
[    0.000000] log_buf_len min size: 1048576 bytes
[    0.000000] log_buf_len: 2097152 bytes
[    0.000000] early log buf free: 1002072(95%)
[    0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    0.000000] Memory: 455606604K/463341200K available (6685K kernel code, 1324K rwdata, 3220K rodata, 1744K init, 3204K bss, 7734596K reserved, 0K cma-reserved)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=192, Nodes=4
[    0.000000] Hierarchical RCU implementation.
[    0.000000] 	Build-time adjustment of leaf fanout to 64.
[    0.000000] 	RCU restricting CPUs from NR_CPUS=8192 to nr_cpu_ids=192.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=192
[    0.000000] NR_IRQS:524544 nr_irqs:3592 16
[    0.000000] 	Offload RCU callbacks from all CPUs
[    0.000000] 	Offload RCU callbacks from CPUs: 0-191.
[    0.000000] Console: colour dummy device 80x25
[    0.000000] console [tty1] enabled
[    0.000000] console [ttyS0] enabled
[    0.000000] mempolicy: Enabling automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl
[    0.000000] clocksource: hpet: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 133484882848 ns
[    0.000000] hpet clockevent registered
[    0.000000] tsc: Fast TSC calibration using PIT
[    0.000000] tsc: Detected 1695.771 MHz processor
[    0.000106] Calibrating delay loop (skipped), value calculated using timer frequency.. 3391.54 BogoMIPS (lpj=1695771)
[    0.011984] pid_max: default: 196608 minimum: 1536
[    0.017406] ACPI: Core revision 20150619
[    0.183067] ACPI: All ACPI Tables successfully acquired
[    0.191544] Security Framework initialized
[    0.196130] SELinux:  Initializing.
[    0.200044] SELinux:  Starting in permissive mode
[    0.258019] Dentry cache hash table entries: 67108864 (order: 17, 536870912 bytes)
[    0.464398] Inode-cache hash table entries: 33554432 (order: 16, 268435456 bytes)
[    0.559497] Mount-cache hash table entries: 1048576 (order: 11, 8388608 bytes)
[    0.568437] Mountpoint-cache hash table entries: 1048576 (order: 11, 8388608 bytes)
[    0.583061] Initializing cgroup subsys blkio
[    0.587844] Initializing cgroup subsys memory
[    0.592789] Initializing cgroup subsys devices
[    0.597760] Initializing cgroup subsys freezer
[    0.602731] Initializing cgroup subsys net_cls
[    0.607695] Initializing cgroup subsys perf_event
[    0.613012] Initializing cgroup subsys hugetlb
[    0.618121] CPU: Physical Processor ID: 0
[    0.622597] CPU: Processor Core ID: 0
[    0.627602] mce: CPU supports 22 MCE banks
[    0.632236] CPU0: Thermal monitoring enabled (TM1)
[    0.637636] process: using mwait in idle threads
[    0.642803] Last level iTLB entries: 4KB 64, 2MB 8, 4MB 8
[    0.648838] Last level dTLB entries: 4KB 64, 2MB 0, 4MB 0, 1GB 4
[    0.656088] Freeing SMP alternatives memory: 24K (ffffffff81cb0000 - ffffffff81cb6000)
[    0.680915] ftrace: allocating 26061 entries in 102 pages
[    0.703091] x2apic: IRQ remapping doesn't support X2APIC mode
[    0.709656] Switched APIC routing to physical flat.
[    0.716008] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.732725] TSC deadline timer enabled
[    0.736947] smpboot: CPU0: Genuine Intel(R) CPU 0000 @ 1.70GHz (fam: 06, model: 4f, stepping: 00)
[    0.746948] Performance Events: PEBS fmt2+, 16-deep LBR, Broadwell events, full-width counters, Intel PMU driver.
[    0.758501] ... version:                3
[    0.762977] ... bit width:              48
[    0.767551] ... generic registers:      4
[    0.772028] ... value mask:             0000ffffffffffff
[    0.777956] ... max period:             0000ffffffffffff
[    0.783889] ... fixed-purpose events:   3
[    0.788365] ... event mask:             000000070000000f
[    0.798859] x86: Booting SMP configuration:
[    0.803535] .... node  #0, CPUs:          #1
[    0.829604] NMI watchdog: enabled on all CPUs, permanently consumes one hw-PMU counter.
[    0.838935]    #2   #3   #4   #5   #6   #7   #8   #9  #10  #11  #12  #13  #14  #15  #16  #17
[    0.950769] .... node  #1, CPUs:    #18  #19  #20  #21  #22  #23  #24  #25  #26  #27  #28  #29  #30  #31  #32  #33  #34  #35
[    1.155260] .... node  #2, CPUs:    #36  #37  #38  #39  #40  #41  #42  #43  #44  #45  #46  #47  #48  #49  #50  #51  #52  #53
[    1.359413] .... node  #3, CPUs:    #54  #55  #56  #57  #58  #59  #60  #61  #62  #63  #64  #65  #66  #67  #68  #69  #70  #71
[    1.563582] .... node  #0, CPUs:    #72  #73  #74  #75  #76  #77  #78  #79  #80  #81  #82  #83  #84  #85  #86  #87  #88  #89
[    1.689721] .... node  #1, CPUs:    #90  #91  #92  #93  #94  #95  #96  #97  #98  #99 #100 #101 #102 #103 #104 #105 #106 #107
[    1.816325] .... node  #2, CPUs:   #108 #109 #110 #111 #112 #113 #114 #115 #116 #117 #118 #119 #120 #121 #122 #123 #124 #125
[    1.941977] .... node  #3, CPUs:   #126 #127 #128 #129 #130 #131 #132 #133 #134 #135 #136 #137 #138 #139 #140 #141 #142 #143
[    2.067174] x86: Booted up 4 nodes, 144 CPUs
[    2.072159] smpboot: Total of 144 processors activated (489047.61 BogoMIPS)
[    3.039713] devtmpfs: initialized
[    3.043506] Using 2GB memory block size for large-memory system
[    3.054955] evm: security.selinux
[    3.058650] evm: security.ima
[    3.061966] evm: security.capability
[    3.066104] PM: Registering ACPI NVS region [mem 0x5e109000-0x6035cfff] (35995648 bytes)
[    3.076398] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 1911260446275000 ns
[    3.088366] atomic64_test: passed for x86-64 platform with CX8 and with SSE
[    3.096146] pinctrl core: initialized pinctrl subsystem
[    3.103442] NET: Registered protocol family 16
[    3.112357] cpuidle: using governor menu
[    3.117390] ACPI FADT declares the system doesn't support PCIe ASPM, so disable it
[    3.125850] ACPI: bus type PCI registered
[    3.130330] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[    3.137794] PCI: MMCONFIG for domain 0000 [bus 00-ff] at [mem 0x80000000-0x8fffffff] (base 0x80000000)
[    3.148195] PCI: MMCONFIG at [mem 0x80000000-0x8fffffff] reserved in E820
[    3.155798] PCI: Using configuration type 1 for base access
[    3.162134] aegl: register_one_node(nid=0)
[    3.166742] aegl: link_mem_sections start=1 end = 1d00000
[    3.172771] register_mem_sect_under_node: s=0 e=7ffff
[    3.178422] register_mem_sect_under_node: s=100000 e=17ffff
[    3.184657] register_mem_sect_under_node: s=180000 e=1fffff
[    3.190892] register_mem_sect_under_node: s=200000 e=27ffff
[    3.197123] register_mem_sect_under_node: s=280000 e=2fffff
[    3.203356] register_mem_sect_under_node: s=300000 e=37ffff
[    3.209589] register_mem_sect_under_node: s=380000 e=3fffff
[    3.215822] register_mem_sect_under_node: s=400000 e=47ffff
[    3.222054] register_mem_sect_under_node: s=480000 e=4fffff
[    3.228282] register_mem_sect_under_node: s=500000 e=57ffff
[    3.234510] register_mem_sect_under_node: s=580000 e=5fffff
[    3.240738] register_mem_sect_under_node: s=600000 e=67ffff
[    3.246964] register_mem_sect_under_node: s=680000 e=6fffff
[    3.253197] register_mem_sect_under_node: s=700000 e=77ffff
[    3.259425] register_mem_sect_under_node: s=780000 e=7fffff
[    3.265652] register_mem_sect_under_node: s=800000 e=87ffff
[    3.271885] register_mem_sect_under_node: s=880000 e=8fffff
[    3.278116] register_mem_sect_under_node: s=900000 e=97ffff
[    3.284349] register_mem_sect_under_node: s=980000 e=9fffff
[    3.290575] register_mem_sect_under_node: s=a00000 e=a7ffff
[    3.296800] register_mem_sect_under_node: s=a80000 e=afffff
[    3.303032] register_mem_sect_under_node: s=b00000 e=b7ffff
[    3.309254] register_mem_sect_under_node: s=b80000 e=bfffff
[    3.315472] register_mem_sect_under_node: s=c00000 e=c7ffff
[    3.321699] register_mem_sect_under_node: s=c80000 e=cfffff
[    3.327929] register_mem_sect_under_node: s=d00000 e=d7ffff
[    3.334168] register_mem_sect_under_node: s=d80000 e=dfffff
[    3.340401] register_mem_sect_under_node: s=e00000 e=e7ffff
[    3.346627] register_mem_sect_under_node: s=e80000 e=efffff
[    3.352858] register_mem_sect_under_node: s=f00000 e=f7ffff
[    3.359093] register_mem_sect_under_node: s=f80000 e=ffffff
[    3.365324] register_mem_sect_under_node: s=1000000 e=107ffff
[    3.371744] register_mem_sect_under_node: s=1080000 e=10fffff
[    3.378166] register_mem_sect_under_node: s=1100000 e=117ffff
[    3.384588] register_mem_sect_under_node: s=1180000 e=11fffff
[    3.391010] register_mem_sect_under_node: s=1200000 e=127ffff
[    3.397430] register_mem_sect_under_node: s=1280000 e=12fffff
[    3.403850] register_mem_sect_under_node: s=1300000 e=137ffff
[    3.410273] register_mem_sect_under_node: s=1380000 e=13fffff
[    3.416695] register_mem_sect_under_node: s=1400000 e=147ffff
[    3.423116] register_mem_sect_under_node: s=1480000 e=14fffff
[    3.429537] register_mem_sect_under_node: s=1500000 e=157ffff
[    3.435958] register_mem_sect_under_node: s=1580000 e=15fffff
[    3.442384] register_mem_sect_under_node: s=1600000 e=167ffff
[    3.448804] register_mem_sect_under_node: s=1680000 e=16fffff
[    3.455229] register_mem_sect_under_node: s=1700000 e=177ffff
[    3.461652] register_mem_sect_under_node: s=1780000 e=17fffff
[    3.468074] register_mem_sect_under_node: s=1800000 e=187ffff
[    3.474497] register_mem_sect_under_node: s=1880000 e=18fffff
[    3.480919] register_mem_sect_under_node: s=1900000 e=197ffff
[    3.487344] register_mem_sect_under_node: s=1980000 e=19fffff
[    3.493765] register_mem_sect_under_node: s=1a00000 e=1a7ffff
[    3.500185] register_mem_sect_under_node: s=1a80000 e=1afffff
[    3.506607] register_mem_sect_under_node: s=1b00000 e=1b7ffff
[    3.513029] register_mem_sect_under_node: s=1b80000 e=1bfffff
[    3.519455] register_mem_sect_under_node: s=1c00000 e=1c7ffff
[    3.525875] register_mem_sect_under_node: s=1c80000 e=1cfffff
[    3.532294] aegl: register_one_node(nid=1)
[    3.536901] aegl: link_mem_sections start=1d70000 end = 3a00000
[    3.543521] register_mem_sect_under_node: s=1d00000 e=1d7ffff
[    3.549951] BUG: unable to handle kernel paging request at ffffea0074000020
[    3.557747] IP: [<ffffffff81670557>] get_nid_for_pfn+0x17/0x40
[    3.564283] PGD 128ffcb067 PUD 128ffc9067 PMD 0 
[    3.569474] Oops: 0000 [#1] SMP 
[    3.573101] Modules linked in:
[    3.576525] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.2.0-rc8 #2
[    3.583431] Hardware name: Intel Corporation BRICKLAND/BRICKLAND, BIOS BRBDXSD1.86B.0326.D03.1508171454 08/17/2015
[    3.594989] task: ffff8839f3990000 ti: ffff881cf35e4000 task.ti: ffff881cf35e4000
[    3.603347] RIP: 0010:[<ffffffff81670557>]  [<ffffffff81670557>] get_nid_for_pfn+0x17/0x40
[    3.612587] RSP: 0000:ffff881cf35e7d38  EFLAGS: 00010286
[    3.618519] RAX: ffffea0000000000 RBX: ffff88755254d800 RCX: 00000000ffffffff
[    3.626479] RDX: 0000000000000000 RSI: 0000000000000246 RDI: ffffea0074000000
[    3.634450] RBP: ffff881cf35e7d78 R08: 0000000000000000 R09: ffff881cffc1af00
[    3.642408] R10: ffffffff816f6d40 R11: 000000000000034f R12: 0000000000000001
[    3.650378] R13: 0000000000000001 R14: 0000000001d00000 R15: 0000000001d7ffff
[    3.658337] FS:  0000000000000000(0000) GS:ffff88121ee00000(0000) knlGS:0000000000000000
[    3.667373] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[    3.673789] CR2: ffffea0074000020 CR3: 00000000019bc000 CR4: 00000000003406f0
[    3.681758] Stack:
[    3.683997]  ffff881cf35e7d78 ffffffff81453b79 0000000000000000 0000000001d70000
[    3.692299]  0000000000000001 0000000000000000 ffff88755254d800 0000000003a00000
[    3.700601]  ffff881cf35e7dc8 ffffffff81453f05 ffff881cf35e7de8 0000000000000001
[    3.708904] Call Trace:
[    3.711642]  [<ffffffff81453b79>] ? register_mem_sect_under_node+0x89/0xe0
[    3.719319]  [<ffffffff81453f05>] register_one_node+0x195/0x270
[    3.725932]  [<ffffffff81b1f1ed>] ? pci_iommu_alloc+0x6e/0x6e
[    3.732349]  [<ffffffff81b1f229>] topology_init+0x3c/0x95
[    3.738377]  [<ffffffff8100213d>] do_one_initcall+0xcd/0x1f0
[    3.744704]  [<ffffffff8109f500>] ? parse_args+0x200/0x470
[    3.750836]  [<ffffffff810c3fa8>] ? __wake_up+0x48/0x60
[    3.756676]  [<ffffffff81b15342>] kernel_init_freeable+0x1a5/0x249
[    3.763580]  [<ffffffff81b149dd>] ? initcall_blacklist+0xb6/0xb6
[    3.770295]  [<ffffffff8166c640>] ? rest_init+0x80/0x80
[    3.776128]  [<ffffffff8166c64e>] kernel_init+0xe/0xe0
[    3.781875]  [<ffffffff81682a9f>] ret_from_fork+0x3f/0x70
[    3.787895]  [<ffffffff8166c640>] ? rest_init+0x80/0x80
[    3.793727] Code: 5c 41 5d 41 5e 41 5f 5d c3 41 be ed ff ff ff eb e6 0f 1f 00 0f 1f 44 00 00 48 c1 e7 06 48 b8 00 00 00 00 00 ea ff ff 55 48 01 c7 <48> 83 7f 20 00 48 89 e5 74 09 48 8b 07 5d 48 c1 e8 36 c3 b8 ff 
[    3.815439] RIP  [<ffffffff81670557>] get_nid_for_pfn+0x17/0x40
[    3.822062]  RSP <ffff881cf35e7d38>
[    3.825955] CR2: ffffea0074000020
[    3.829659] ---[ end trace c3e458d8ffdfb77d ]---
[    3.834815] Kernel panic - not syncing: Fatal exception
[    3.840692] ---[ end Kernel panic - not syncing: Fatal exception

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

On Mon, Aug 24, 2015 at 3:39 PM, Tony Luck <tony.luck@gmail.com> wrote:
> On Mon, Aug 24, 2015 at 2:25 PM, Yinghai Lu <yinghai@kernel.org> wrote:
>
>> Can you boot with "debug ignore_loglevel" so we can see following print out
>> for vmemmap?
>
> See attached. There are a few extra messages from my own debug printk()
> calls. It seems that we successfully deal with node 0 from topology_init()
> but die walking node 1. I see that the NODE_DATA limits for memory
> on node 1 were from 1d70000 to 3a00000. But when we get into
> register_mem_sect_under_node() we have rounded the start pfn down to
> 1d00000 ... and we panic processing that range (which is in a hole in e820).
>
> We seem to die here:
>
>         for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
>                 int page_nid;
>
>                 page_nid = get_nid_for_pfn(pfn);

oh, no.
register_mem_sect_under_node() is assuming:
first section in the block is present and first page in that section is present.

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

[-- Attachment #1: Type: text/plain, Size: 1124 bytes --]

On Mon, Aug 24, 2015 at 4:41 PM, Yinghai Lu <yinghai@kernel.org> wrote:
> On Mon, Aug 24, 2015 at 3:39 PM, Tony Luck <tony.luck@gmail.com> wrote:
>> On Mon, Aug 24, 2015 at 2:25 PM, Yinghai Lu <yinghai@kernel.org> wrote:
>>
>>> Can you boot with "debug ignore_loglevel" so we can see following print out
>>> for vmemmap?
>>
>> See attached. There are a few extra messages from my own debug printk()
>> calls. It seems that we successfully deal with node 0 from topology_init()
>> but die walking node 1. I see that the NODE_DATA limits for memory
>> on node 1 were from 1d70000 to 3a00000. But when we get into
>> register_mem_sect_under_node() we have rounded the start pfn down to
>> 1d00000 ... and we panic processing that range (which is in a hole in e820).
>>
>> We seem to die here:
>>
>>         for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
>>                 int page_nid;
>>
>>                 page_nid = get_nid_for_pfn(pfn);
>
> oh, no.
> register_mem_sect_under_node() is assuming:
> first section in the block is present and first page in that section is present.

attached should fix the problem:

[-- Attachment #2: check_section_for_memory_block_register.patch --]
[-- Type: text/x-patch, Size: 1404 bytes --]

diff --git a/drivers/base/node.c b/drivers/base/node.c
index 31df474d..cc910ad 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -390,8 +390,14 @@ int register_mem_sect_under_node(struct memory_block *mem_blk, int nid)
 	sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr);
 	sect_end_pfn += PAGES_PER_SECTION - 1;
 	for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
-		int page_nid;
+		int page_nid, scn_nr;
 
+		scn_nr = pfn_to_section_nr(pfn);
+		if (!present_section_nr(scn_nr)) {
+			pfn = round_down(pfn + PAGES_PER_SECTION,
+					 PAGES_PER_SECTION) - 1;
+			continue;
+		}
 		page_nid = get_nid_for_pfn(pfn);
 		if (page_nid < 0)
 			continue;
@@ -426,10 +432,18 @@ int unregister_mem_sect_under_nodes(struct memory_block *mem_blk,
 		return -ENOMEM;
 	nodes_clear(*unlinked_nodes);
 
-	sect_start_pfn = section_nr_to_pfn(phys_index);
-	sect_end_pfn = sect_start_pfn + PAGES_PER_SECTION - 1;
+	sect_start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
+	sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr);
+	sect_end_pfn += PAGES_PER_SECTION - 1;
 	for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
-		int nid;
+		int nid, scn_nr;
+
+		scn_nr = pfn_to_section_nr(pfn);
+		if (!present_section_nr(scn_nr)) {
+			pfn = round_down(pfn + PAGES_PER_SECTION,
+					 PAGES_PER_SECTION) - 1;
+			continue;
+		}
 
 		nid = get_nid_for_pfn(pfn);
 		if (nid < 0)

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

On Tue, Aug 25, 2015 at 10:03 AM, Tony Luck <tony.luck@gmail.com> wrote:
> On Mon, Aug 24, 2015 at 4:59 PM, Yinghai Lu <yinghai@kernel.org> wrote:
>> attached should fix the problem:
>
> It does ... but this (attached) is simpler.  Your patch and mine both
> allow the system to boot ...

The version that fix with section_nr present checking may save couple thousands
calling to get_nid_for_pfn(). section size / page_size = 128M/4k = 32k

> but it is not happy. See all the chatter from systemd in the attached dmesg.

because of you have "debug ignore_loglevel" ?

>
> x86 doesn't allow me to set CONFIG_HOLES_IN_ZONE ... but now I'm
> worried about all the other places use pfn_valid_within()
>
> Still trying to get an answer from the BIOS folks on whether these
> holes are normal when setting up mirrored areas of memory.

The problem only happens when memory block size is 512M and section
size is 128M.
when you have them both at 128M, the system works. so current kernel
should only has
problem with hole size  > 128M to leave some section not present.

Thanks

Yinghai

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Tony Luck]

On Tue, Aug 25, 2015 at 12:01 PM, Yinghai Lu <yinghai@kernel.org> wrote:
>> It does ... but this (attached) is simpler.  Your patch and mine both
>> allow the system to boot ...
>
> The version that fix with section_nr present checking may save couple thousands
> calling to get_nid_for_pfn(). section size / page_size = 128M/4k = 32k

Actually saves about 1.2 million calls. Your patch wins :-)

Reported-and-tested-by: Tony Luck <tony.luck@intel.com>

-Tony

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Ingo Molnar]

* Yinghai Lu <yinghai@kernel.org> wrote:

> --- a/drivers/base/node.c
> +++ b/drivers/base/node.c
> @@ -390,8 +390,14 @@ int register_mem_sect_under_node(struct memory_block *mem_blk, int nid)
>  	sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr);
>  	sect_end_pfn += PAGES_PER_SECTION - 1;
>  	for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
> -		int page_nid;
> +		int page_nid, scn_nr;
>  
> +		scn_nr = pfn_to_section_nr(pfn);
> +		if (!present_section_nr(scn_nr)) {
> +			pfn = round_down(pfn + PAGES_PER_SECTION,
> +					 PAGES_PER_SECTION) - 1;
> +			continue;
> +		}
>  		page_nid = get_nid_for_pfn(pfn);
>  		if (page_nid < 0)
>  			continue;
> @@ -426,10 +432,18 @@ int unregister_mem_sect_under_nodes(struct memory_block *mem_blk,
>  		return -ENOMEM;
>  	nodes_clear(*unlinked_nodes);
>  
> -	sect_start_pfn = section_nr_to_pfn(phys_index);
> -	sect_end_pfn = sect_start_pfn + PAGES_PER_SECTION - 1;
> +	sect_start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
> +	sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr);
> +	sect_end_pfn += PAGES_PER_SECTION - 1;
>  	for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
> -		int nid;
> +		int nid, scn_nr;
> +
> +		scn_nr = pfn_to_section_nr(pfn);
> +		if (!present_section_nr(scn_nr)) {
> +			pfn = round_down(pfn + PAGES_PER_SECTION,
> +					 PAGES_PER_SECTION) - 1;
> +			continue;
> +		}

NAK due to lack of cleanliness: the two loops look almost identical - this sure 
can be factored out...

Thanks,

	Ingo

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

On Tue, Aug 25, 2015 at 9:17 PM, Ingo Molnar <mingo@kernel.org> wrote:
> NAK due to lack of cleanliness: the two loops look almost identical - this sure
> can be factored out...

Please check complete version at

https://patchwork.kernel.org/patch/7074341/

Andrew,
Ingo NAKed raw version of this patch, so you may need to remove it
from -mm tree.

Thanks

Yinghai

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Andrew Morton]

On Tue, 25 Aug 2015 22:42:05 -0700 Yinghai Lu <yinghai@kernel.org> wrote:

> On Tue, Aug 25, 2015 at 9:17 PM, Ingo Molnar <mingo@kernel.org> wrote:
> > NAK due to lack of cleanliness: the two loops look almost identical - this sure
> > can be factored out...
> 
> Please check complete version at
> 
> https://patchwork.kernel.org/patch/7074341/

That doesn't do what Ingo suggested: "can be factored out...".

Please review this?

--- a/drivers/base/node.c~mm-check-if-section-present-during-memory-block-unregistering-v2-fix
+++ a/drivers/base/node.c
@@ -375,6 +375,22 @@ static int __init_refok get_nid_for_pfn(
 	return pfn_to_nid(pfn);
 }
 
+/*
+ * A memory block can have several absent sections.  A helper function for
+ * skipping over these holes.
+ *
+ * If an absent section is detected, skip_absent_section() will advance *pfn
+ * to the final page in that section and will return true.
+ */
+static bool skip_absent_section(unsigned long *pfn)
+{
+	if (present_section_nr(pfn_to_section_nr(*pfn)))
+		return false;
+
+	*pfn = round_down(*pfn + PAGES_PER_SECTION, PAGES_PER_SECTION) - 1;
+	return true;
+}
+
 /* register memory section under specified node if it spans that node */
 int register_mem_sect_under_node(struct memory_block *mem_blk, int nid)
 {
@@ -390,18 +406,10 @@ int register_mem_sect_under_node(struct
 	sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr);
 	sect_end_pfn += PAGES_PER_SECTION - 1;
 	for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
-		int page_nid, scn_nr;
+		int page_nid;
 
-		/*
-		 * memory block could have several absent sections from start.
-		 * skip pfn range from absent section
-		 */
-		scn_nr = pfn_to_section_nr(pfn);
-		if (!present_section_nr(scn_nr)) {
-			pfn = round_down(pfn + PAGES_PER_SECTION,
-					 PAGES_PER_SECTION) - 1;
+		if (skip_absent_section(&pfn))
 			continue;
-		}
 
 		page_nid = get_nid_for_pfn(pfn);
 		if (page_nid < 0)
@@ -441,18 +449,10 @@ int unregister_mem_sect_under_nodes(stru
 	sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr);
 	sect_end_pfn += PAGES_PER_SECTION - 1;
 	for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
-		int nid, scn_nr;
+		int nid;
 
-		/*
-		 * memory block could have several absent sections from start.
-		 * skip pfn range from absent section
-		 */
-		scn_nr = pfn_to_section_nr(pfn);
-		if (!present_section_nr(scn_nr)) {
-			pfn = round_down(pfn + PAGES_PER_SECTION,
-					 PAGES_PER_SECTION) - 1;
+		if (skip_absent_section(&pfn))
 			continue;
-		}
 
 		nid = get_nid_for_pfn(pfn);
 		if (nid < 0)
_


> Andrew,
> Ingo NAKed raw version of this patch, so you may need to remove it
> from -mm tree.

I don't know what that means.  We have multiple patches under at least
two different Subject:s.  Please be very careful and very specific when
identifying patches.  Otherwise mistakes will be made.


I presently have three patches:

mm-check-if-section-present-during-memory-block-unregistering.patch
mm-check-if-section-present-during-memory-block-unregistering-v2.patch
mm-check-if-section-present-during-memory-block-unregistering-v2-fix.patch

When these are consolidated together, this is the result:


From: Yinghai Lu <yinghai@kernel.org>
Subject: mm: check if section present during memory block (un)registering

Tony Luck found on his setup, if memory block size 512M will cause crash
during booting.

 BUG: unable to handle kernel paging request at ffffea0074000020
 IP: [<ffffffff81670527>] get_nid_for_pfn+0x17/0x40
 PGD 128ffcb067 PUD 128ffc9067 PMD 0
 Oops: 0000 [#1] SMP
 Modules linked in:
 CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.2.0-rc8 #1
...
 Call Trace:
  [<ffffffff81453b56>] ? register_mem_sect_under_node+0x66/0xe0
  [<ffffffff81453eeb>] register_one_node+0x17b/0x240
  [<ffffffff81b1f1ed>] ? pci_iommu_alloc+0x6e/0x6e
  [<ffffffff81b1f229>] topology_init+0x3c/0x95
  [<ffffffff8100213d>] do_one_initcall+0xcd/0x1f0

The system has non continuous RAM address:
 BIOS-e820: [mem 0x0000001300000000-0x0000001cffffffff] usable
 BIOS-e820: [mem 0x0000001d70000000-0x0000001ec7ffefff] usable
 BIOS-e820: [mem 0x0000001f00000000-0x0000002bffffffff] usable
 BIOS-e820: [mem 0x0000002c18000000-0x0000002d6fffefff] usable
 BIOS-e820: [mem 0x0000002e00000000-0x00000039ffffffff] usable

So there are start sections in memory block not present.
For example:
memory block : [0x2c18000000, 0x2c20000000) 512M
first three sections are not present.

Current register_mem_sect_under_node() assume first section is present,
but memory block section number range [start_section_nr, end_section_nr]
would include not present section.

For arch that support vmemmap, we don't setup memmap for struct page area
within not present sections area.

So skip the pfn range that belong to not present section.

Also fixes unregister_mem_sect_under_nodes().

Fixes: bdee237c0343 ("x86: mm: Use 2GB memory block size on large memory x86-64 systems")
Fixes: 982792c782ef ("x86, mm: probe memory block size for generic x86 64bit")
[akpm@linux-foundation.org: factor out common code]
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Reported-by: Tony Luck <tony.luck@intel.com>
Tested-by: Tony Luck <tony.luck@intel.com>
Cc: Greg KH <greg@kroah.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: <stable@vger.kernel.org>	[3.15+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---

 drivers/base/node.c |   27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff -puN drivers/base/node.c~mm-check-if-section-present-during-memory-block-unregistering drivers/base/node.c
--- a/drivers/base/node.c~mm-check-if-section-present-during-memory-block-unregistering
+++ a/drivers/base/node.c
@@ -375,6 +375,22 @@ static int __init_refok get_nid_for_pfn(
 	return pfn_to_nid(pfn);
 }
 
+/*
+ * A memory block can have several absent sections.  A helper function for
+ * skipping over these holes.
+ *
+ * If an absent section is detected, skip_absent_section() will advance *pfn
+ * to the final page in that section and will return true.
+ */
+static bool skip_absent_section(unsigned long *pfn)
+{
+	if (present_section_nr(pfn_to_section_nr(*pfn)))
+		return false;
+
+	*pfn = round_down(*pfn + PAGES_PER_SECTION, PAGES_PER_SECTION) - 1;
+	return true;
+}
+
 /* register memory section under specified node if it spans that node */
 int register_mem_sect_under_node(struct memory_block *mem_blk, int nid)
 {
@@ -392,6 +408,9 @@ int register_mem_sect_under_node(struct
 	for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
 		int page_nid;
 
+		if (skip_absent_section(&pfn))
+			continue;
+
 		page_nid = get_nid_for_pfn(pfn);
 		if (page_nid < 0)
 			continue;
@@ -426,11 +445,15 @@ int unregister_mem_sect_under_nodes(stru
 		return -ENOMEM;
 	nodes_clear(*unlinked_nodes);
 
-	sect_start_pfn = section_nr_to_pfn(phys_index);
-	sect_end_pfn = sect_start_pfn + PAGES_PER_SECTION - 1;
+	sect_start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
+	sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr);
+	sect_end_pfn += PAGES_PER_SECTION - 1;
 	for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
 		int nid;
 
+		if (skip_absent_section(&pfn))
+			continue;
+
 		nid = get_nid_for_pfn(pfn);
 		if (nid < 0)
 			continue;
_

Re: [PATCH v4 4/4] Use 2GB memory block size on large-memory x86-64 systems

[Yinghai Lu]

On Wed, Aug 26, 2015 at 1:49 PM, Andrew Morton
<akpm@linux-foundation.org> wrote:
> On Tue, 25 Aug 2015 22:42:05 -0700 Yinghai Lu <yinghai@kernel.org> wrote:
> I don't know what that means.  We have multiple patches under at least
> two different Subject:s.  Please be very careful and very specific when
> identifying patches.  Otherwise mistakes will be made.
>
>
> I presently have three patches:
>
> mm-check-if-section-present-during-memory-block-unregistering.patch
> mm-check-if-section-present-during-memory-block-unregistering-v2.patch
> mm-check-if-section-present-during-memory-block-unregistering-v2-fix.patch
>
> When these are consolidated together, this is the result:

Please drop all three, and apply v3 directly from

https://patchwork.kernel.org/patch/7080111/

we should not touch unregiser path, as unregister_memory_section()
already check if the section is present before.

Thanks

Yinghai

[tip:x86/platform] x86: numachip: Fix 16-bit APIC ID truncation

[tip-bot for Daniel J Blueman]

Commit-ID:  00e7977dd1bbd46e336d7ef907d0fb6b6a4c294f
Gitweb:     http://git.kernel.org/tip/00e7977dd1bbd46e336d7ef907d0fb6b6a4c294f
Author:     Daniel J Blueman <daniel@numascale.com>
AuthorDate: Tue, 4 Nov 2014 16:29:41 +0800
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 4 Nov 2014 18:17:27 +0100

x86: numachip: Fix 16-bit APIC ID truncation

Prevent 16-bit APIC IDs being truncated by using correct mask. This fixes
booting large systems, where the wrong core would receive the startup and
init IPIs, causing hanging.

Signed-off-by: Daniel J Blueman <daniel@numascale.com>
Cc: Steffen Persvold <sp@numascale.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Link: http://lkml.kernel.org/r/1415089784-28779-1-git-send-email-daniel@numascale.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/apic/apic_numachip.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 4128b5f..2aaee79 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -40,7 +40,7 @@ static unsigned int get_apic_id(unsigned long x)
 	unsigned int id;
 
 	rdmsrl(MSR_FAM10H_NODE_ID, value);
-	id = ((x >> 24) & 0xffU) | ((value << 2) & 0x3f00U);
+	id = ((x >> 24) & 0xffU) | ((value << 2) & 0xff00U);
 
 	return id;
 }