From: Hinko Kocevar <hinko.kocevar@ess.eu>
To: Keith Busch <kbusch@kernel.org>
Cc: "Kelley, Sean V" <sean.v.kelley@intel.com>,
Linux PCI <linux-pci@vger.kernel.org>,
Bjorn Helgaas <helgaas@kernel.org>
Subject: Re: [PATCHv2 0/5] aer handling fixups
Date: Mon, 11 Jan 2021 21:02:39 +0100 [thread overview]
Message-ID: <6783d09d-1431-15fd-961e-3820b14e001e@ess.eu> (raw)
In-Reply-To: <20210111163708.GA1458209@dhcp-10-100-145-180.wdc.com>
[-- Attachment #1: Type: text/plain, Size: 3443 bytes --]
On 1/11/21 5:37 PM, Keith Busch wrote:
> On Mon, Jan 11, 2021 at 02:39:20PM +0100, Hinko Kocevar wrote:
>> Testing this patch a bit more (without the 5/5) resulted in the same CPU
>> lockup:
>>
>> watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [irq/122-aerdrv:128]
>>
>> as I initially reported with the 5/5 of this patch included.
>>
>> It seems more infrequent, though. For example, after reboot this is not
>> observed and the recovery process is successful, whereas when 5/5 is also
>> used every recovery resulted in CPU lockup.
>
> I am assuming this soft lockup is still when restoring the downstream
> port's virtual channel capability. Your initial sighting indicates that
> it doesn't appear to be a deadlock, but the stack trace never existed
> pci_restore_vc_state() either. I did not find any obvious issues here
> just from code inspection, so if you could try applying the following
> patch and send the kernel messages output, that would help.
>
> ---
> diff --git a/drivers/pci/vc.c b/drivers/pci/vc.c
> index 5fc59ac31145..4834af7eb582 100644
> --- a/drivers/pci/vc.c
> +++ b/drivers/pci/vc.c
> @@ -28,6 +28,7 @@ static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos,
> {
> int i;
>
> + pci_warn(dev, "%s: pos:%d dwords:%d\n", __func__, pos, dwords);
> for (i = 0; i < dwords; i++, buf++) {
> if (save)
> pci_read_config_dword(dev, pos + (i * 4), buf);
> @@ -110,6 +111,8 @@ static void pci_vc_enable(struct pci_dev *dev, int pos, int res)
> if (!pci_is_pcie(dev) || !pcie_downstream_port(dev))
> return;
>
> + pci_warn(dev, "%s: pos:%d res:%d\n", __func__, pos, res);
> +
> ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
> status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
>
> @@ -165,6 +168,8 @@ static void pci_vc_enable(struct pci_dev *dev, int pos, int res)
> if (link && !pci_wait_for_pending(link, status_pos2,
> PCI_VC_RES_STATUS_NEGO))
> pci_err(link, "VC%d negotiation stuck pending\n", id);
> +
> + pci_warn(dev, "%s: pos:%d res:%d return\n", __func__, pos, res);
> }
>
> /**
> @@ -190,6 +195,7 @@ static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
> int i, len = 0;
> u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL;
>
> + pci_warn(dev, "%s: buf:%d pos:%d\n", __func__, buf != NULL, pos);
> /* Sanity check buffer size for save/restore */
> if (buf && save_state->cap.size !=
> pci_vc_do_save_buffer(dev, pos, NULL, save)) {
> @@ -278,6 +284,8 @@ static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
> pci_read_config_dword(dev, pos + PCI_VC_RES_CAP +
> (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap);
> parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16;
> + pci_warn(dev, "%s: i:%d evcc:%d parb_offset:%d\n", __func__, i,
> + evcc, parb_offset);
> if (parb_offset) {
> int size, parb_phases = 0;
>
> @@ -332,6 +340,7 @@ static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
> len += 4;
> }
>
> + pci_warn(dev, "%s: len:%d\n", __func__, len);
> return buf ? 0 : len;
> }
>
> @@ -399,6 +408,7 @@ void pci_restore_vc_state(struct pci_dev *dev)
> if (!save_state || !pos)
> continue;
>
> + pci_warn(dev, "%s: i:%d pos:%d\n", __func__, i, pos);
> pci_vc_do_save_buffer(dev, pos, save_state, false);
> }
> }
> --
>
Attached are the messages.
Thanks!
[-- Attachment #2: cpu-lockups2 --]
[-- Type: text/plain, Size: 35612 bytes --]
...
[ 64.649399] xdma: loading out-of-tree module taints kernel.
[ 64.656255] xdma:xdma_mod_init: Xilinx XDMA Reference Driver xdma v2020.1.8
[ 64.663324] xdma:xdma_mod_init: desc_blen_max: 0xfffffff/268435455, timeout: h2c 10 c2h 10 sec.
[ 64.672189] xdma:xdma_device_open: xdma device 0000:08:00.0, 0x00000000591acbe0.
[ 64.680346] xdma:map_single_bar: BAR0 at 0xde000000 mapped at 0x000000000fdb12a5, length=16777216(/16777216)
[ 64.690290] xdma:map_single_bar: BAR1 at 0xdf000000 mapped at 0x000000005fe214c9, length=65536(/65536)
[ 64.699702] xdma:map_bars: config bar 1, pos 1.
[ 64.704339] xdma:identify_bars: 2 BARs: config 1, user 0, bypass -1.
[ 64.711245] xdma:pci_keep_intx_enabled: 0000:08:00.0: clear INTX_DISABLE, 0x407 -> 0x7.
[ 64.719369] xdma:irq_msix_channel_setup: engine 0-H2C0-MM, irq#169.
[ 64.725751] xdma:irq_msix_channel_setup: engine 0-C2H0-MM, irq#170.
[ 64.732134] xdma:irq_msix_user_setup: 0-USR-0, IRQ#171 with 0x0000000016487e14
[ 64.739479] xdma:irq_msix_user_setup: 0-USR-1, IRQ#172 with 0x0000000017b744b7
[ 64.746813] xdma:irq_msix_user_setup: 0-USR-2, IRQ#173 with 0x000000007491af44
[ 64.754149] xdma:irq_msix_user_setup: 0-USR-3, IRQ#174 with 0x000000009e92f5ff
[ 64.761484] xdma:irq_msix_user_setup: 0-USR-4, IRQ#175 with 0x0000000056e09b58
[ 64.768819] xdma:irq_msix_user_setup: 0-USR-5, IRQ#176 with 0x00000000af6d01f5
[ 64.776159] xdma:irq_msix_user_setup: 0-USR-6, IRQ#177 with 0x000000002ef15b69
[ 64.783493] xdma:irq_msix_user_setup: 0-USR-7, IRQ#178 with 0x0000000099fd5ab6
[ 64.790832] xdma:irq_msix_user_setup: 0-USR-8, IRQ#179 with 0x0000000014654474
[ 64.798167] xdma:irq_msix_user_setup: 0-USR-9, IRQ#180 with 0x00000000963ca534
[ 64.805504] xdma:irq_msix_user_setup: 0-USR-10, IRQ#181 with 0x0000000086853299
[ 64.812928] xdma:irq_msix_user_setup: 0-USR-11, IRQ#182 with 0x00000000d0d53381
[ 64.820354] xdma:irq_msix_user_setup: 0-USR-12, IRQ#183 with 0x000000008ce02f44
[ 64.827776] xdma:irq_msix_user_setup: 0-USR-13, IRQ#184 with 0x00000000bc4b16b8
[ 64.835200] xdma:irq_msix_user_setup: 0-USR-14, IRQ#185 with 0x0000000088677e88
[ 64.842623] xdma:irq_msix_user_setup: 0-USR-15, IRQ#186 with 0x000000000d9849f6
[ 64.850044] xdma:probe_one: 0000:08:00.0 xdma0, pdev 0x00000000591acbe0, xdev 0x00000000c77a5002, 0x00000000947398f0, usr 16, ch 1,1.
[ 64.864139] xdma:cdev_xvc_init: xcdev 0x000000004eda4704, bar 0, offset 0x40000.
[ 68.523443] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 68.541024] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 68.547489] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 68.554208] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 68.560929] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.568603] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.574632] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.582305] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.588338] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 68.597139] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 68.605946] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 68.615568] pcieport 0000:00:01.1: PME# disabled
[ 68.739042] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 68.747225] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 68.755222] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 68.763310] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 68.771310] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 68.778974] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 68.786628] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 68.794629] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 68.802379] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 68.810298] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 68.818315] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 68.825980] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 68.833651] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 68.841312] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 68.849061] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 68.856719] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 68.864817] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 68.871433] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 68.878049] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.885620] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.891555] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.899128] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.905178] pcieport 0000:00:01.1: PME# enabled
[ 68.921051] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 68.927905] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 68.946078] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 68.952545] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 68.959265] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 68.965984] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.973658] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.979682] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.987353] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.993381] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 69.002178] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 69.010980] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 69.020624] pcieport 0000:00:01.1: PME# disabled
[ 69.084588] pcieport 0000:02:02.0: PME# disabled
[ 69.101016] pcieport 0000:02:02.0: pci_restore_vc_state: i:1 pos:328
[ 69.107376] pcieport 0000:02:02.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 69.113998] pcieport 0000:02:02.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 69.120646] pcieport 0000:02:02.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:0
[ 69.128225] pcieport 0000:02:02.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.135795] pcieport 0000:02:02.0: pci_vc_do_save_buffer: len:12
[ 69.141818] pcieport 0000:02:02.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:0
[ 69.149407] pcieport 0000:02:02.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.149972] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 69.156981] pcieport 0000:02:02.0: pci_vc_do_save_buffer: len:12
[ 69.157058] pcieport 0000:02:02.0: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 69.157063] pcieport 0000:02:02.0: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 69.157067] pcieport 0000:02:02.0: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 69.157261] pcieport 0000:02:02.0: PME# disabled
[ 69.165517] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 69.171639] pci_bus 0000:0a: busn_res: [bus 0a] is released
[ 69.180336] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 69.188880] pcieport 0000:02:08.0: PME# disabled
[ 69.203089] pcieport 0000:02:08.0: pci_restore_vc_state: i:1 pos:328
[ 69.203091] pcieport 0000:02:08.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 69.203093] pcieport 0000:02:08.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 69.203097] pcieport 0000:02:08.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 69.203100] pcieport 0000:02:08.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.203101] pcieport 0000:02:08.0: pci_vc_do_save_buffer: len:76
[ 69.203108] pcieport 0000:02:08.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 69.211361] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 69.216693] pcieport 0000:02:08.0: pci_vc_save_restore_dwords: pos:376 dwords:16
[ 69.216758] pcieport 0000:02:08.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.216763] pcieport 0000:02:08.0: pci_vc_do_save_buffer: len:76
[ 69.216804] pcieport 0000:02:08.0: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 69.225139] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 69.229513] pcieport 0000:02:08.0: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 69.229520] pcieport 0000:02:08.0: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 69.229614] pcieport 0000:02:08.0: PME# disabled
[ 69.229829] pci_bus 0000:0b: busn_res: [bus 0b] is released
[ 69.236231] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 69.242730] pcieport 0000:02:09.0: PME# disabled
[ 69.249579] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 69.257048] pci_bus 0000:0c: dev 00, dec refcount to 0
[ 69.257060] pci_bus 0000:0c: dev 00, released physical slot 9
[ 69.257223] pci_bus 0000:0c: busn_res: [bus 0c] is released
[ 69.264928] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 69.270998] pcieport 0000:02:0a.0: PME# disabled
[ 69.278705] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 69.302025] pcieport 0000:02:0a.0: pci_restore_vc_state: i:1 pos:328
[ 69.307986] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 69.316417] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 69.316419] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 69.316428] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:0
[ 69.316432] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.316433] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: len:12
[ 69.316446] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:0
[ 69.316453] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.316460] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: len:12
[ 69.316514] pcieport 0000:02:0a.0: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 69.316521] pcieport 0000:02:0a.0: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 69.316525] pcieport 0000:02:0a.0: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 69.316660] pcieport 0000:02:0a.0: PME# disabled
[ 69.324439] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 69.333201] pci_bus 0000:0d: busn_res: [bus 0d] is released
[ 69.342639] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 69.347130] plx_dma 0000:01:00.1: PME# disabled
[ 69.352951] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 69.546955] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 69.554803] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 69.562563] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 69.570767] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 69.577487] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 69.584208] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 69.591884] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 69.597910] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 69.605583] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 69.611806] pcieport 0000:00:01.1: PME# enabled
[ 69.623179] plx_dma 0000:01:00.2: PME# disabled
[ 69.628424] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 69.640379] plx_dma 0000:01:00.3: PME# disabled
[ 69.656336] plx_dma 0000:01:00.4: PME# disabled
[ 72.886830] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 72.904973] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 72.911333] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 72.917963] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 72.924583] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 72.932152] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 72.938089] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 72.945665] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 72.951596] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 72.960292] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 72.968989] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 72.978623] pcieport 0000:00:01.1: PME# disabled
[ 73.099568] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 73.107774] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 73.115786] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 73.123878] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 73.131892] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 73.139559] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 73.147222] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 73.155220] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 73.162968] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 73.170892] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 73.178913] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 73.186579] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 73.194234] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 73.201890] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 73.209637] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 73.217296] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 73.225411] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 73.232025] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 73.238644] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.246217] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.252153] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.259736] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.265707] pcieport 0000:00:01.1: PME# enabled
[ 73.282253] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 73.289128] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 73.306977] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 73.313335] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 73.319962] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 73.326583] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.334159] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.340092] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.347671] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.353617] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 73.362320] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 73.371018] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 73.380445] pcieport 0000:00:01.1: PME# disabled
[ 73.430387] pcieport 0000:00:01.0: aer_inject: Injecting errors 00000000/00004000 into device 0000:00:01.0
[ 73.440086] pcieport 0000:00:01.0: AER: Uncorrected (Fatal) error received: 0000:00:01.0
[ 73.448186] pcieport 0000:00:01.0: PCIe Bus Error: severity=Uncorrected (Fatal), type=Transaction Layer, (Requester ID)
[ 73.458969] pcieport 0000:00:01.0: device [8086:1901] error status/mask=00004000/00000000
[ 73.467328] pcieport 0000:00:01.0: [14] CmpltTO
[ 73.473434] xdma_error_detected: >>>
[ 73.477012] xdma:xdma_error_detected: dev 0x00000000591acbe0,0x00000000c77a5002, frozen state error, reset controller
[ 73.487626] xdma:xdma_device_offline: pdev 0x00000000591acbe0, xdev 0x00000000947398f0.
[ 73.495738] xdma:xdma_device_offline: xdev 0x00000000947398f0, done.
[ 73.509969] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 73.518157] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 73.526165] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 73.534267] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 73.542269] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 73.549931] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 73.557591] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 73.565605] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 73.573359] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 73.581278] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 73.589286] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 73.596945] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 73.604609] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 73.612272] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 73.620024] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 73.627691] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 73.635804] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 73.642419] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 73.649037] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.656602] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.662525] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.670102] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.676118] pcieport 0000:00:01.1: PME# enabled
[ 73.692066] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 74.509998] pcieport 0000:00:01.0: AER: Root Port link has been reset (0)
[ 74.516820] pcieport 0000:01:00.0: pci_restore_vc_state: i:1 pos:328
[ 74.523169] pcieport 0000:01:00.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 74.529783] pcieport 0000:01:00.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 74.536405] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 74.544059] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 74.551630] pcieport 0000:01:00.0: pci_vc_do_save_buffer: len:76
[ 74.557663] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 74.565327] pcieport 0000:01:00.0: pci_vc_save_restore_dwords: pos:376 dwords:16
[ 74.572741] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 74.580318] pcieport 0000:01:00.0: pci_vc_do_save_buffer: len:76
[ 74.586354] pcieport 0000:01:00.0: restoring config space at offset 0x3c (was 0x100, writing 0x12010b)
[ 74.595676] pcieport 0000:01:00.0: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 74.604377] pcieport 0000:01:00.0: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 74.613076] pcieport 0000:01:00.0: restoring config space at offset 0x24 (was 0x1fff1, writing 0x91119001)
[ 74.622730] pcieport 0000:01:00.0: restoring config space at offset 0x20 (was 0xfff0, writing 0xdf10de00)
[ 74.632300] pcieport 0000:01:00.0: restoring config space at offset 0x1c (was 0x1f1, writing 0x8121)
[ 74.641427] pcieport 0000:01:00.0: restoring config space at offset 0x18 (was 0x0, writing 0xd0201)
[ 74.650479] pcieport 0000:01:00.0: restoring config space at offset 0x10 (was 0x0, writing 0xdf200000)
[ 74.659787] pcieport 0000:01:00.0: restoring config space at offset 0xc (was 0x810000, writing 0x810010)
[ 74.669275] pcieport 0000:01:00.0: restoring config space at offset 0x4 (was 0x100000, writing 0x100407)
[ 74.678805] pcieport 0000:01:00.0: saving config space at offset 0x0 (reading 0x872510b5)
[ 74.686984] pcieport 0000:01:00.0: saving config space at offset 0x4 (reading 0x100407)
[ 74.694996] pcieport 0000:01:00.0: saving config space at offset 0x8 (reading 0x60400ca)
[ 74.703094] pcieport 0000:01:00.0: saving config space at offset 0xc (reading 0x810010)
[ 74.711114] pcieport 0000:01:00.0: saving config space at offset 0x10 (reading 0xdf200000)
[ 74.719381] pcieport 0000:01:00.0: saving config space at offset 0x14 (reading 0x0)
[ 74.727031] pcieport 0000:01:00.0: saving config space at offset 0x18 (reading 0xd0201)
[ 74.735027] pcieport 0000:01:00.0: saving config space at offset 0x1c (reading 0x8121)
[ 74.742939] pcieport 0000:01:00.0: saving config space at offset 0x20 (reading 0xdf10de00)
[ 74.751201] pcieport 0000:01:00.0: saving config space at offset 0x24 (reading 0x91119001)
[ 74.759472] pcieport 0000:01:00.0: saving config space at offset 0x28 (reading 0x0)
[ 74.767134] pcieport 0000:01:00.0: saving config space at offset 0x2c (reading 0x0)
[ 74.774788] pcieport 0000:01:00.0: saving config space at offset 0x30 (reading 0x0)
[ 74.782441] pcieport 0000:01:00.0: saving config space at offset 0x34 (reading 0x40)
[ 74.790188] pcieport 0000:01:00.0: saving config space at offset 0x38 (reading 0x0)
[ 74.797844] pcieport 0000:01:00.0: saving config space at offset 0x3c (reading 0x12010b)
[ 74.806027] pcieport 0000:01:00.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 74.812641] pcieport 0000:01:00.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 74.819255] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 74.826911] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 74.834483] pcieport 0000:01:00.0: pci_vc_do_save_buffer: len:76
[ 74.840505] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 74.848160] pcieport 0000:01:00.0: pci_vc_save_restore_dwords: pos:376 dwords:16
[ 74.855580] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 74.863150] pcieport 0000:01:00.0: pci_vc_do_save_buffer: len:76
[ 76.390998] sched: RT throttling activated
[ 78.616988] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 78.636106] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 78.642483] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 78.649139] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 78.655790] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 78.663376] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 78.669325] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 78.676919] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 78.682969] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 78.691694] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 78.700420] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 78.710200] pcieport 0000:00:01.1: PME# disabled
[ 78.838975] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 78.847182] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 78.855215] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 78.863331] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 78.871363] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 78.879043] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 78.886728] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 78.894762] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 78.902529] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 78.910469] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 78.918498] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 78.926176] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 78.933857] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 78.941537] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 78.949299] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 78.956985] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 78.965256] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 78.971890] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 78.978537] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 78.986126] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 78.992093] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 78.999683] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 79.005664] pcieport 0000:00:01.1: PME# enabled
[ 79.023036] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 101.457576] watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [irq/122-aerdrv:129]
[ 101.465153] Modules linked in: xdma(O) aer_inject xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6tai
[ 101.465206] drm_kms_helper syscopyarea sysfillrect igb sysimgblt ptp fb_sys_fops libahci crc32c_intel pps_core serio_raw dca i2c_algo_bit libata drm video
[ 101.566072] CPU: 2 PID: 129 Comm: irq/122-aerdrv Tainted: G S O 5.10.0-1.keithv3.el7.x86_64 #1
[ 101.575814] Hardware name: AMI AM G6x/msd/AM G6x/msd, BIOS 4.08.01 02/19/2019
[ 101.582950] RIP: 0010:pci_mmcfg_read+0xb7/0xe0
[ 101.587403] Code: 63 c5 48 01 d8 66 8b 00 0f b7 c0 41 89 04 24 eb dc 49 63 c5 48 01 d8 8a 00 0f b6 c0 41 89 04 24 eb cb 49 63 c5 48 01 d8 8b 00 <41> 89 04 24 eb bd e8 be f4 78 ff 5b 41 c7 04 24 ff ff ff ff b8 ea
[ 101.606158] RSP: 0018:ffffc90000303b68 EFLAGS: 00000286
[ 101.611391] RAX: 00000000b701000d RBX: ffffc90010208000 RCX: 0000000000000f24
[ 101.618531] RDX: 00000000000000ff RSI: 0000000000000002 RDI: 0000000000000000
[ 101.625673] RBP: ffffc90000303b90 R08: 0000000000000004 R09: ffffc90000303bc4
[ 101.632811] R10: 0000000000000008 R11: ffffc900003038e0 R12: ffffc90000303bc4
[ 101.639944] R13: 0000000000000f24 R14: 0000000000008000 R15: 0000000000000004
[ 101.647082] FS: 0000000000000000(0000) GS:ffff88845dc80000(0000) knlGS:0000000000000000
[ 101.655174] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 101.660920] CR2: 00007f9369637b80 CR3: 000000000240a005 CR4: 00000000003706e0
[ 101.668057] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 101.675190] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 101.682323] Call Trace:
[ 101.684780] raw_pci_read+0x35/0x40
[ 101.688277] pci_read+0x2c/0x30
[ 101.691428] pci_bus_read_config_dword+0x4a/0x70
[ 101.696051] pci_read_config_dword+0x23/0x40
[ 101.700331] pci_find_next_ext_capability.part.24+0x6b/0xe0
[ 101.705912] pci_find_ext_capability.part.25+0x12/0x20
[ 101.711058] pci_find_ext_capability+0x1a/0x20
[ 101.715509] pci_restore_vc_state+0x32/0xb0
[ 101.719695] ? pcie_capability_write_word+0x32/0x90
[ 101.724579] pci_restore_state.part.44+0x177/0x420
[ 101.729377] pci_restore_state+0x18/0x20
[ 101.733310] pcie_portdrv_slot_reset+0x12/0x30
[ 101.737765] report_slot_reset+0x46/0x80
[ 101.741695] ? merge_result.part.4+0x40/0x40
[ 101.745971] pci_walk_bus+0x48/0x90
[ 101.749466] pci_walk_bridge+0x1d/0x30
[ 101.753223] pcie_do_recovery+0x1bd/0x270
[ 101.757241] ? aer_dev_correctable_show+0xd0/0xd0
[ 101.761948] ? aer_dev_correctable_show+0xd0/0xd0
[ 101.766657] aer_process_err_devices+0xd0/0xe9
[ 101.771104] aer_isr.cold.25+0x54/0xa3
[ 101.774862] ? irq_finalize_oneshot.part.48+0xf0/0xf0
[ 101.779924] irq_thread_fn+0x24/0x60
[ 101.783516] irq_thread+0xea/0x170
[ 101.786926] ? irq_forced_thread_fn+0x80/0x80
[ 101.791292] ? irq_thread_check_affinity+0xf0/0xf0
[ 101.796088] kthread+0x126/0x140
[ 101.799330] ? kthread_park+0x90/0x90
[ 101.803004] ret_from_fork+0x22/0x30
[ 111.539063] pcieport 0000:02:01.0: pci_restore_vc_state: i:1 pos:328
[ 111.545503] pcieport 0000:02:01.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 129.266656] pcieport 0000:02:01.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 134.720858] pcieport 0000:02:01.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 134.728712] pcieport 0000:02:01.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 136.397775] rcu: INFO: rcu_sched self-detected stall on CPU
[ 136.410927] rcu: 2-....: (1 GPs behind) idle=7b2/1/0x4000000000000000 softirq=5916/5917 fqs=14413
[ 136.419972] (t=60000 jiffies g=12137 q=11828)
[ 136.424422] NMI backtrace for cpu 2
[ 136.427915] CPU: 2 PID: 129 Comm: irq/122-aerdrv Tainted: G S O L 5.10.0-1.keithv3.el7.x86_64 #1
[ 136.437651] Hardware name: AMI AM G6x/msd/AM G6x/msd, BIOS 4.08.01 02/19/2019
[ 136.444785] Call Trace:
[ 136.447237] <IRQ>
[ 136.449263] dump_stack+0x74/0x92
[ 136.452595] nmi_cpu_backtrace.cold.8+0x32/0x68
[ 136.457134] ? lapic_can_unplug_cpu+0x80/0x80
[ 136.461494] nmi_trigger_cpumask_backtrace+0xe6/0xf0
[ 136.466467] arch_trigger_cpumask_backtrace+0x19/0x20
[ 136.471526] rcu_dump_cpu_stacks+0xb6/0xe6
[ 136.475635] rcu_sched_clock_irq.cold.92+0x124/0x3a9
[ 136.480610] ? account_system_index_time+0x94/0xa0
[ 136.485417] ? tick_sched_do_timer+0x60/0x60
[ 136.489696] update_process_times+0x93/0xd0
[ 136.493892] tick_sched_handle.isra.23+0x25/0x60
[ 136.498517] tick_sched_timer+0x6b/0x80
[ 136.502360] __hrtimer_run_queues+0x105/0x240
[ 136.506726] hrtimer_interrupt+0x101/0x220
[ 136.510836] __sysvec_apic_timer_interrupt+0x62/0xe0
[ 136.515815] asm_call_irq_on_stack+0x12/0x20
[ 136.520092] </IRQ>
[ 136.522202] sysvec_apic_timer_interrupt+0x7d/0x90
[ 136.527003] asm_sysvec_apic_timer_interrupt+0x12/0x20
[ 136.532152] RIP: 0010:vprintk_emit+0x10b/0x250
[ 136.536601] Code: 1f 44 00 00 e8 e6 0b 00 00 8b 7d d4 48 8b 55 c8 89 de 4d 89 f8 4c 89 f1 e8 62 fa ff ff 89 c3 e8 eb 0b 00 00 48 8b 7d c0 57 9d <0f> 1f 44 00 00 45 84 e4 74 1f 80 3d 95 0a 6f 01 00 74 05 e8 3d d6
[ 136.555356] RSP: 0018:ffffc90000303968 EFLAGS: 00000246
[ 136.560583] RAX: 0000000000000046 RBX: 0000000000000046 RCX: 0000000000001d91
[ 136.567718] RDX: ffffffff825d9628 RSI: 80000000ffffc9db RDI: 0000000000000246
[ 136.574852] RBP: ffffc900003039b0 R08: ffffffff8246a940 R09: 0000000000000000
[ 136.581992] R10: ffffffff82b21518 R11: ffffffff82b21516 R12: 0000000000000000
[ 136.589126] R13: ffffffff82176221 R14: ffffffff8215574f R15: ffffc90000303a60
[ 136.596267] dev_vprintk_emit+0x13b/0x15f
[ 136.600286] dev_printk_emit+0x4e/0x65
[ 136.604048] ? pci_mmcfg_read+0xb7/0xe0
[ 136.607890] __dev_printk+0x58/0x6b
[ 136.611383] _dev_warn+0x6c/0x83
[ 136.614622] ? pci_bus_read_config_dword+0x4a/0x70
[ 136.619418] pci_vc_do_save_buffer+0x251/0x5ab
[ 136.623869] pci_vc_do_save_buffer+0x87/0x5ab
[ 136.628238] ? _dev_warn+0x6c/0x83
[ 136.631642] pci_restore_vc_state+0x7e/0xb0
[ 136.635839] pci_restore_state.part.44+0x177/0x420
[ 136.640636] pci_restore_state+0x18/0x20
[ 136.644569] pcie_portdrv_slot_reset+0x12/0x30
[ 136.649017] report_slot_reset+0x46/0x80
[ 136.652945] ? merge_result.part.4+0x40/0x40
[ 136.657221] pci_walk_bus+0x48/0x90
[ 136.660715] pci_walk_bridge+0x1d/0x30
[ 136.664471] pcie_do_recovery+0x1bd/0x270
[ 136.668483] ? aer_dev_correctable_show+0xd0/0xd0
[ 136.673192] ? aer_dev_correctable_show+0xd0/0xd0
[ 136.677901] aer_process_err_devices+0xd0/0xe9
[ 136.682351] aer_isr.cold.25+0x54/0xa3
[ 136.686108] ? irq_finalize_oneshot.part.48+0xf0/0xf0
[ 136.691167] irq_thread_fn+0x24/0x60
[ 136.694754] irq_thread+0xea/0x170
[ 136.698169] ? irq_forced_thread_fn+0x80/0x80
[ 136.702533] ? irq_thread_check_affinity+0xf0/0xf0
[ 136.707330] kthread+0x126/0x140
[ 136.710572] ? kthread_park+0x90/0x90
[ 136.714247] ret_from_fork+0x22/0x30
[ 136.717927] pcieport 0000:02:01.0: pci_vc_do_save_buffer: len:76
[ 161.459414] watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [irq/122-aerdrv:129]
[ 161.459417] Modules linked in: xdma(O) aer_inject xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6tai
[ 161.459460] drm_kms_helper syscopyarea sysfillrect igb sysimgblt ptp fb_sys_fops libahci crc32c_intel pps_core serio_raw dca i2c_algo_bit libata drm video
[ 161.459468] CPU: 2 PID: 129 Comm: irq/122-aerdrv Tainted: G S O L 5.10.0-1.keithv3.el7.x86_64 #1
[ 161.459470] Hardware name: AMI AM G6x/msd/AM G6x/msd, BIOS 4.08.01 02/19/2019
[ 161.459471] RIP: 0010:console_unlock+0x374/0x510
[ 161.459475] Code: ff e8 10 f9 ff ff 85 c0 0f 85 4f ff ff ff e8 c3 f8 ff ff 85 c0 0f 85 18 fd ff ff e9 3d ff ff ff e8 71 2b 00 00 4c 89 ff 57 9d <0f> 1f 44 00 00 8b 85 44 ff ff ff 85 c0 0f 84 3a fd ff ff e8 74 03
[ 161.459476] RSP: 0018:ffffc90000303898 EFLAGS: 00000247
[ 161.459478] RAX: 0000000000000000 RBX: ffffffff82af03ac RCX: ffffffff815dba40
[ 161.459479] RDX: 0000000000000000 RSI: ffffffff82af2420 RDI: 0000000000000247
[ 161.459480] RBP: ffffc90000303958 R08: 0000019b07f91685 R09: 0000000000000002
[ 161.459481] R10: 32303a3030303020 R11: 74726f7065696370 R12: 0000000000000000
[ 161.459482] R13: 0000000000000000 R14: ffffffff827ecfa0 R15: 0000000000000247
[ 161.459483] FS: 0000000000000000(0000) GS:ffff88845dc80000(0000) knlGS:0000000000000000
[ 161.459485] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 161.459486] CR2: 00007f9369637b80 CR3: 000000000240a005 CR4: 00000000003706e0
[ 161.459487] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 161.459488] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 161.459489] Call Trace:
[ 161.459491] vprintk_emit+0x14b/0x250
[ 161.459493] dev_vprintk_emit+0x13b/0x15f
[ 161.459497] dev_printk_emit+0x4e/0x65
[ 161.459499] ? pci_mmcfg_read+0xb7/0xe0
[ 161.459503] __dev_printk+0x58/0x6b
[ 161.459505] _dev_warn+0x6c/0x83
[ 161.459508] ? pci_bus_read_config_dword+0x4a/0x70
[ 161.459510] pci_vc_do_save_buffer+0x576/0x5ab
...
next prev parent reply other threads:[~2021-01-11 20:03 UTC|newest]
Thread overview: 36+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-04 23:02 [PATCHv2 0/5] aer handling fixups Keith Busch
2021-01-04 23:02 ` [PATCHv2 1/5] PCI/ERR: Clear status of the reporting device Keith Busch
2021-01-04 23:02 ` [PATCHv2 2/5] PCI/AER: Actually get the root port Keith Busch
2021-01-04 23:02 ` [PATCHv2 3/5] PCI/ERR: Retain status from error notification Keith Busch
2021-03-03 5:34 ` Williams, Dan J
2021-03-03 5:46 ` Kuppuswamy, Sathyanarayanan
2021-03-04 20:01 ` Keith Busch
2021-03-04 22:11 ` Dan Williams
[not found] ` <23551edc-965c-21dc-0da8-a492c27c362d@intel.com>
2021-03-04 22:59 ` Dan Williams
2021-03-04 23:19 ` Kuppuswamy, Sathyanarayanan
2021-03-05 0:23 ` Dan Williams
2021-03-05 0:54 ` Keith Busch
2021-01-04 23:02 ` [PATCHv2 4/5] PCI/AER: Specify the type of port that was reset Keith Busch
2021-01-04 23:03 ` [PATCHv2 5/5] PCI/portdrv: Report reset for frozen channel Keith Busch
2021-01-05 14:21 ` [PATCHv2 0/5] aer handling fixups Hinko Kocevar
2021-01-05 15:06 ` Hinko Kocevar
2021-01-05 18:33 ` Keith Busch
2021-01-05 23:07 ` Kelley, Sean V
2021-01-07 21:42 ` Keith Busch
2021-01-08 9:38 ` Hinko Kocevar
2021-01-11 13:39 ` Hinko Kocevar
2021-01-11 16:37 ` Keith Busch
2021-01-11 20:02 ` Hinko Kocevar [this message]
2021-01-11 22:09 ` Keith Busch
[not found] ` <ed8256dd-d70d-b8dc-fdc0-a78b9aa3bbd9@ess.eu>
2021-01-12 19:27 ` Keith Busch
2021-01-12 22:19 ` Hinko Kocevar
2021-01-12 23:17 ` Keith Busch
2021-01-18 8:00 ` Hinko Kocevar
2021-01-19 18:28 ` Keith Busch
2021-02-03 0:03 ` Keith Busch
2021-02-04 8:35 ` Hinko Kocevar
2021-02-08 12:55 ` Hedi Berriche
2021-02-09 23:06 ` Bjorn Helgaas
2021-02-10 4:05 ` Keith Busch
2021-02-10 21:38 ` Bjorn Helgaas
2021-02-10 9:36 ` Yicong Yang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=6783d09d-1431-15fd-961e-3820b14e001e@ess.eu \
--to=hinko.kocevar@ess.eu \
--cc=helgaas@kernel.org \
--cc=kbusch@kernel.org \
--cc=linux-pci@vger.kernel.org \
--cc=sean.v.kelley@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.