[  532.046617] cxgb4 0000:03:00.4: Port 0 link down, reason: Link Down

[  532.047027] cxgb4 0000:03:00.4 ens1f4: link down

[  545.865331] cxgb4 0000:03:00.4 ens1f4: link up, 100Gbps, full-duplex, Tx/Rx PAUSE

[  547.498413] ------------[ cut here ]------------

[  547.499049] refcount_t: underflow; use-after-free.

[  547.499757] WARNING: CPU: 5 PID: 801 at lib/refcount.c:28 refcount_warn_saturate+0x9b/0x100

[  547.500464] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  547.500533]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  547.512682] CPU: 5 PID: 801 Comm: kworker/5:1H Kdump: loaded Not tainted 5.6.0-rc1+ #18

[  547.513954] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  547.515337] Workqueue: ib-comp-wq ib_cq_poll_work [ib_core]

[  547.516662] RIP: 0010:refcount_warn_saturate+0x9b/0x100

[  547.517941] Code: e9 1e 01 01 e8 d6 73 bf ff 0f 0b c3 80 3d 98 e9 1e 01 00 75 a9 48 c7 c7 b0 f9 34 97 31 c0 c6 05 86 e9 1e 01 01 e8 b5 73 bf ff <0f> 0b c3 80 3d 7a e9 1e 01 00 75 88 48 c7 c7 58 f9 34 97 31 c0 c6

[  547.520824] RSP: 0018:ffffa2d14065fd50 EFLAGS: 00010282

[  547.522500] RAX: 0000000000000000 RBX: ffff8d8610a22c90 RCX: 0000000000000000

[  547.523976] RDX: ffff8d87dc0268c0 RSI: ffff8d87dbc1a0f8 RDI: ffff8d87dbc1a0f8

[  547.525567] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000001

[  547.526989] R10: 0000000000000000 R11: ffff8d8583af86f0 R12: 0000000000000002

[  547.528551] R13: 0000000000000000 R14: 0000000000000010 R15: ffff8d87a488bc40

[  547.530003] FS:  0000000000000000(0000) GS:ffff8d87dbc00000(0000) knlGS:0000000000000000

[  547.531637] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  547.533166] CR2: 00007f52e7df0140 CR3: 00000002fd810002 CR4: 00000000003606e0

[  547.534786] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  547.536466] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  547.538018] Call Trace:

[  547.539726]  transport_generic_free_cmd+0x155/0x160 [target_core_mod]

[  547.541466]  __ib_process_cq+0xc0/0x280 [ib_core]

[  547.543121]  ib_cq_poll_work+0x26/0x80 [ib_core]

[  547.544848]  process_one_work+0x202/0x580

[  547.546620]  ? process_one_work+0x17c/0x580

[  547.548361]  worker_thread+0x4c/0x3f0

[  547.550065]  kthread+0x103/0x140

[  547.551877]  ? process_one_work+0x580/0x580

[  547.553700]  ? kthread_bind+0x10/0x10

[  547.555523]  ret_from_fork+0x3a/0x50

[  547.557310] irq event stamp: 730

[  547.559101] hardirqs last  enabled at (729): [<ffffffff96b858b4>] _raw_spin_unlock_irq+0x24/0x40

[  547.560943] hardirqs last disabled at (730): [<ffffffff96b7e44a>] __schedule+0xda/0x990

[  547.562821] softirqs last  enabled at (726): [<ffffffff969a3a24>] flush_backlog+0x134/0x150

[  547.564703] softirqs last disabled at (722): [<ffffffff969a38f5>] flush_backlog+0x5/0x150

[  547.566545] ---[ end trace e964c55ac82fd95e ]---

[  563.114653] iSCSI Login timeout on Network Portal 10.0.2.2:3260

[  572.936188] watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  572.936293] watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [iscsi_trx:12748]

[  572.937403] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  572.937467]  ipmi_msghandler

[  572.938545] Modules linked in:

[  572.947394]  snd_timer

[  572.948499]  target_core_user

[  572.949546]  wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  572.950618]  uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm ipmi_msghandler snd_timer wmi snd soundcore

[  572.950643]  lpc_ich

[  572.951651] irq event stamp: 0

[  572.951655] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  572.954755]  i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  572.963700] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  572.963702] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  572.964863] irq event stamp: 0

[  572.964865] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  572.966050] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  572.966053] CPU: 6 PID: 12756 Comm: iscsi_trx Kdump: loaded Tainted: G        W         5.6.0-rc1+ #18

[  572.967213] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  572.967216] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  572.970899] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  572.970903] RIP: 0010:lock_acquire+0x3a/0x1c0

[  572.972170] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  572.972172] CPU: 13 PID: 12748 Comm: iscsi_trx Kdump: loaded Tainted: G        W         5.6.0-rc1+ #18

[  572.973451] Code: 40 8e 01 00 55 53 48 83 ec 30 8b 82 bc 0c 00 00 48 89 7c 24 20 85 c0 0f 85 ad 00 00 00 89 f5 41 89 cd 45 89 c6 4d 89 cf 9c 58 <0f> 1f 44 00 00 48 89 44 24 28 fa 66 0f 1f 44 00 00 c7 82 bc 0c 00

[  572.974678] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  572.974682] RIP: 0010:__local_bh_enable_ip+0x87/0xf0

[  572.975916] RSP: 0018:ffffa2d1430cfd40 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  572.977228] Code: 74 57 65 01 1d 32 08 d7 69 65 8b 05 2b 08 d7 69 a9 00 ff 1f 00 74 55 65 ff 0d 1d 08 d7 69 e8 20 e8 12 00 fb 66 0f 1f 44 00 00 <5b> 5d c3 8b 80 88 0c 00 00 85 c0 75 a7 80 3d 88 ea 5e 01 00 75 9e

[  572.978557] RAX: 0000000000000246 RBX: ffff8d8610a23aa0 RCX: 0000000000000000

[  572.978558] RDX: ffff8d878eb1c180 RSI: 0000000000000000 RDI: ffff8d8610a23ab8

[  572.979827] RSP: 0018:ffffa2d14308fdd0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  572.981123] RBP: 0000000000000000 R08: 0000000000000001 R09: 0000000000000000

[  572.981124] R10: 0000000000000000 R11: ffffa2d1430cfdb0 R12: 0000000000000000

[  572.982452] RAX: 0000000000000000 RBX: 00000000fffffe00 RCX: 0000000000000000

[  572.982453] RDX: ffff8d8786724180 RSI: 0000000000000201 RDI: ffffffff962a85f0

[  572.983742] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000

[  572.983743] FS:  0000000000000000(0000) GS:ffff8d87dc200000(0000) knlGS:0000000000000000

[  572.985017] RBP: ffffffffc0db70fe R08: 0000000000000000 R09: 0000000000000000

[  572.985018] R10: 0000000000000000 R11: ffffa2d14308fdb0 R12: 0000000000000001

[  572.986355] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033



Message from syslogd@mehrangarh at Feb 10 15:06:42 ...

 kernel:watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  572.986356] CR2: 00007f52e7df0140 CR3: 00000002fd810001 CR4: 00000000003606e0

[  572.989015] R13: ffff8d808737f800 R14: ffff8d808737fde0 R15: ffff8d86111af160

[  572.989016] FS:  0000000000000000(0000) GS:ffff8d87dec00000(0000) knlGS:0000000000000000

[  572.990426] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  572.991772] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  572.991773] CR2: 00007f4df5029248 CR3: 00000002fd810003 CR4: 00000000003606e0

[  572.993133] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  572.993134] Call Trace:

[  572.995871] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  572.997343]  ? iscsit_stop_dataout_timer+0x19/0x80 [iscsi_target_mod]

[  572.998648] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  572.998649] Call Trace:

[  572.999941]  _raw_spin_lock_bh+0x3d/0x50

[  573.001326]  __iscsit_free_cmd+0x28e/0x2c0 [iscsi_target_mod]

[  573.002585]  ? iscsit_free_r2ts_from_list+0x24/0x70 [iscsi_target_mod]

[  573.003794]  iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  573.005047]  iscsit_free_r2ts_from_list+0x24/0x70 [iscsi_target_mod]

[  573.006314]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  573.007570]  __iscsit_free_cmd+0x251/0x2c0 [iscsi_target_mod]

[  573.008807]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  573.010048]  iscsit_free_cmd+0x48/0xa0 [iscsi_target_mod]

[  573.011296]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]



Message from syslogd@mehrangarh at Feb 10 15:06:42 ...

 kernel:watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [iscsi_trx:12748]

[  573.012519]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  573.013714]  kthread+0x103/0x140

[  573.014900]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  573.016085]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  573.017215]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  573.018340]  ? kthread_bind+0x10/0x10

[  573.019435]  kthread+0x103/0x140

[  573.020532]  ret_from_fork+0x3a/0x50

[  573.021599]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  573.045424]  ? kthread_bind+0x10/0x10

[  573.046408]  ret_from_fork+0x3a/0x50

[  600.936308] watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  600.936414] watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [iscsi_trx:12748]

[  600.937243] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  600.937275]  ipmi_msghandler

[  600.938049] Modules linked in:

[  600.944983]  snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  600.945964]  target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm ipmi_msghandler snd_timer

[  600.945985]  wmi

[  600.946934] irq event stamp: 0

[  600.946937] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  600.949897]  snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  600.958443] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  600.958445] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  600.959594] irq event stamp: 0

[  600.959596] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  600.960731] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  600.960733] CPU: 6 PID: 12756 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  600.961867] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  600.961869] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  600.965365] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  600.965369] RIP: 0010:_raw_spin_unlock_irqrestore+0x32/0x52

[  600.966604] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  600.966606] CPU: 13 PID: 12748 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  600.967804] Code: fd 48 8d 7f 18 53 48 89 f3 48 8b 74 24 10 e8 55 ef 78 ff 48 89 ef e8 9d 3b 79 ff f6 c7 02 74 19 e8 b3 11 85 ff 48 89 df 57 9d <0f> 1f 44 00 00 65 ff 0d 9a 31 49 69 5b 5d c3 48 89 df 57 9d 0f 1f

[  600.968983] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  600.968986] RIP: 0010:lock_release+0xf/0x2a0

[  600.970199] RSP: 0018:ffffa2d1430cfd90 EFLAGS: 00000296 ORIG_RAX: ffffffffffffff13

[  600.971426] Code: c6 d0 23 30 97 48 c7 c7 a2 65 2e 97 31 c0 e8 08 b0 f8 ff 0f 0b e9 e8 fe ff ff 90 41 57 65 48 8b 14 25 40 8e 01 00 41 56 41 55 <41> 54 55 53 48 83 ec 10 44 8b 82 bc 0c 00 00 65 48 8b 04 25 28 00

[  600.972637] RAX: 0000000000000001 RBX: 0000000000000296 RCX: 0000000000000000

[  600.972638] RDX: 0000000000000006 RSI: ffffffffc0d53e4a RDI: 0000000000000296



Message from syslogd@mehrangarh at Feb 10 15:07:10 ...

 kernel:watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  600.973824] RSP: 0018:ffffa2d14308fdb8 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  600.975009] RBP: ffff8d8610a23cf8 R08: 0000000000000000 R09: 0000000000000001

[  600.975010] R10: 0000000000000000 R11: ffffa2d1430cfdb0 R12: 0000000000000001

[  600.976228] RAX: 0000000000000000 RBX: ffff8d808737fe50 RCX: 0000000000000000

[  600.976230] RDX: ffff8d8786724180 RSI: ffffffffc0db70f1 RDI: ffff8d808737fe68

[  600.977458] R13: 0000000000000001 R14: ffff8d8610a23be0 R15: ffff8d8610a238e0

[  600.977460] FS:  0000000000000000(0000) GS:ffff8d87dc200000(0000) knlGS:0000000000000000

[  600.978662] RBP: ffff8d86111af470 R08: 0000000000000001 R09: 0000000000000000

[  600.978663] R10: 0000000000000000 R11: ffffa2d14308fdb0 R12: 0000000000000001

[  600.979845] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  600.979846] CR2: 00007f52e7df0140 CR3: 00000002fd810001 CR4: 00000000003606e0

[  600.982292] R13: ffff8d808737f800 R14: ffff8d808737fe50 R15: ffff8d86111af160

[  600.982294] FS:  0000000000000000(0000) GS:ffff8d87dec00000(0000) knlGS:0000000000000000

[  600.983568] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  600.984810] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  600.984811] CR2: 00007f4df5029248 CR3: 00000002fd810003 CR4: 00000000003606e0

[  600.986052] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400



Message from syslogd@mehrangarh at Feb 10 15:07:10 ...

 kernel:watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [iscsi_trx:12748]

[  600.986053] Call Trace:

[  600.988635] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  600.989941]  transport_generic_free_cmd+0x12a/0x160 [target_core_mod]

[  600.991230] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  600.991231] Call Trace:

[  600.992544]  iscsit_free_cmd+0x58/0xa0 [iscsi_target_mod]

[  600.993790]  _raw_spin_unlock_bh+0x17/0x30

[  600.995013]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  600.996240]  __iscsit_free_cmd+0x281/0x2c0 [iscsi_target_mod]

[  600.997468]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  600.998638]  iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  600.999801]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  601.000947]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  601.002088]  kthread+0x103/0x140

[  601.003223]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  601.004345]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  601.005483]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  601.006603]  ? kthread_bind+0x10/0x10

[  601.006606]  ret_from_fork+0x3a/0x50

[  601.007701]  kthread+0x103/0x140

[  601.030430]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  601.031328]  ? kthread_bind+0x10/0x10

[  601.032240]  ret_from_fork+0x3a/0x50

[  605.130023] cxgb4 0000:03:00.4: Port 0 link down, reason: Link Down

[  605.131611] cxgb4 0000:03:00.4 ens1f4: link down

[  612.501078] rcu: INFO: rcu_sched self-detected stall on CPU

[  612.501922] rcu:     13-....: (64820 ticks this GP) idle=bd6/1/0x4000000000000002 softirq=27999/27999 fqs=15343

[  612.503598]  (t=65002 jiffies g=69481 q=31952)

[  612.504590] NMI backtrace for cpu 13

[  612.505775] CPU: 13 PID: 12748 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  612.507577] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  612.509359] Call Trace:

[  612.511146]  <IRQ>

[  612.512865]  dump_stack+0x82/0xb3

[  612.514613]  nmi_cpu_backtrace+0x90/0xa0

[  612.516343]  ? lapic_can_unplug_cpu+0xa0/0xa0

[  612.518050]  nmi_trigger_cpumask_backtrace+0x73/0x130

[  612.519773]  rcu_dump_cpu_stacks+0x97/0xc5

[  612.521482]  rcu_sched_clock_irq+0x6ec/0x980

[  612.523172]  ? tick_sched_do_timer+0x70/0x70

[  612.524829]  update_process_times+0x24/0x50

[  612.526495]  tick_sched_handle+0x25/0x60

[  612.528128]  tick_sched_timer+0x37/0x70

[  612.529720]  __hrtimer_run_queues+0xe8/0x450

[  612.531315]  hrtimer_interrupt+0x122/0x270

[  612.532882]  ? trace_hardirqs_off_thunk+0x1a/0x1c

[  612.534466]  smp_apic_timer_interrupt+0x86/0x240

[  612.536019]  apic_timer_interrupt+0xf/0x20

[  612.537576]  </IRQ>

[  612.539071] RIP: 0010:do_raw_spin_lock+0x3e/0xa0

[  612.540595] Code: 8b 43 10 65 48 8b 14 25 40 8e 01 00 48 39 c2 74 66 8b 53 08 65 8b 05 29 8c cf 69 39 c2 74 47 31 c0 ba 01 00 00 00 f0 0f b1 13 <0f> 94 c2 84 d2 74 27 65 8b 05 0c 8c cf 69 89 43 08 65 48 8b 04 25

[  612.543756] RSP: 0018:ffffa2d14308fdd8 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  612.545368] RAX: 0000000000000000 RBX: ffff8d808737fde0 RCX: 0000000000000000

[  612.547012] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff8d808737fde0

[  612.548630] RBP: ffff8d86111af470 R08: 0000000000000001 R09: 0000000000000000

[  612.550237] R10: 0000000000000000 R11: ffffa2d14308fdb0 R12: 0000000000000001

[  612.551819] R13: ffff8d808737f800 R14: ffff8d808737fde0 R15: ffff8d86111af160

[  612.553458]  ? __iscsit_free_cmd+0xa9/0x2c0 [iscsi_target_mod]

[  612.555043]  __iscsit_free_cmd+0xa9/0x2c0 [iscsi_target_mod]

[  612.556645]  iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  612.558229]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  612.559802]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  612.561401]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  612.562982]  kthread+0x103/0x140

[  612.564547]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  612.566140]  ? kthread_bind+0x10/0x10

[  612.567724]  ret_from_fork+0x3a/0x50

[  618.978720] cxgb4 0000:03:00.4 ens1f4: link up, 100Gbps, full-duplex, Tx/Rx PAUSE

[  628.936291] watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  628.937158] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  628.937191]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  628.947559] irq event stamp: 0

[  628.948639] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  628.949684] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80



Message from syslogd@mehrangarh at Feb 10 15:07:38 ...

 kernel:watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  628.950729] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  628.951764] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  628.952774] CPU: 6 PID: 12756 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  628.953800] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  628.954831] RIP: 0010:__local_bh_enable_ip+0x87/0xf0

[  628.955845] Code: 74 57 65 01 1d 32 08 d7 69 65 8b 05 2b 08 d7 69 a9 00 ff 1f 00 74 55 65 ff 0d 1d 08 d7 69 e8 20 e8 12 00 fb 66 0f 1f 44 00 00 <5b> 5d c3 8b 80 88 0c 00 00 85 c0 75 a7 80 3d 88 ea 5e 01 00 75 9e

[  628.957947] RSP: 0018:ffffa2d1430cfdd0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  628.959010] RAX: 0000000000000000 RBX: 00000000fffffe00 RCX: 0000000000000000

[  628.960117] RDX: ffff8d878eb1c180 RSI: 0000000000000201 RDI: ffffffff962a85f0

[  628.961208] RBP: ffffffffc0db70f1 R08: 0000000000000000 R09: 0000000000000000

[  628.962312] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001

[  628.963440] R13: ffff8d8562ea2800 R14: ffff8d8562ea2e50 R15: ffff8d8610a238e0

[  628.964550] FS:  0000000000000000(0000) GS:ffff8d87dc200000(0000) knlGS:0000000000000000

[  628.965630] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  628.966710] CR2: 00007f52e7df0140 CR3: 00000002fd810001 CR4: 00000000003606e0

[  628.967789] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  628.968863] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  628.969907] Call Trace:

[  628.970964]  __iscsit_free_cmd+0x281/0x2c0 [iscsi_target_mod]

[  628.972026]  iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  628.973100]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  628.974135]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  628.975201]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  628.976260]  kthread+0x103/0x140

[  628.977319]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  628.978418]  ? kthread_bind+0x10/0x10

[  628.979486]  ret_from_fork+0x3a/0x50

[  636.936393] watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [iscsi_trx:12748]

[  636.937563] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  636.937605]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]



Message from syslogd@mehrangarh at Feb 10 15:07:46 ...

 kernel:watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [iscsi_trx:12748]

[  636.950692] irq event stamp: 0

[  636.951792] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  636.952889] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  636.953960] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  636.955021] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  636.956046] CPU: 13 PID: 12748 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  636.957104] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  636.958169] RIP: 0010:cpumask_next+0x0/0x20

[  636.959224] Code: 45 ff ff ff c6 00 00 eb be 48 83 c0 01 48 89 02 8d 41 ff 48 01 f8 80 38 22 75 9d c6 00 00 eb 98 cc cc cc cc cc cc cc cc cc cc <48> 89 f0 8b 35 bb 93 d6 00 83 c7 01 48 63 d7 48 89 c7 e8 39 66 b4

[  636.961414] RSP: 0018:ffffa2d14308fd20 EFLAGS: 00000206 ORIG_RAX: ffffffffffffff13

[  636.962540] RAX: 0000000000000003 RBX: ffffffffc0d8d660 RCX: ffff8d87db42d000

[  636.963646] RDX: ffff8d87db000000 RSI: ffffffff978c6920 RDI: 0000000000000003

[  636.964711] RBP: 0000000000000000 R08: 0000000000000000 R09: 000000000000fff8

[  636.965780] R10: 0000000000000000 R11: ffffa2d14308fdb0 R12: 000000000042d000

[  636.966837] R13: 0000000000000000 R14: ffff8d86111af460 R15: ffff8d86111af160

[  636.967885] FS:  0000000000000000(0000) GS:ffff8d87dec00000(0000) knlGS:0000000000000000

[  636.968940] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  636.969990] CR2: 00007f4df5029248 CR3: 00000002fd810003 CR4: 00000000003606e0

[  636.971038] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  636.972065] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  636.973064] Call Trace:

[  636.974057]  __is_kernel_percpu_address+0x39/0xa0

[  636.975056]  static_obj+0x25/0x50

[  636.976050]  lockdep_init_map+0x79/0x1f0

[  636.977041]  __init_waitqueue_head+0x36/0x50

[  636.978074]  transport_generic_free_cmd+0x40/0x160 [target_core_mod]

[  636.979096]  iscsit_free_cmd+0x58/0xa0 [iscsi_target_mod]

[  636.980072]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  636.981071]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  636.982070]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  636.983063]  kthread+0x103/0x140

[  636.984045]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  636.985056]  ? kthread_bind+0x10/0x10

[  636.986050]  ret_from_fork+0x3a/0x50

[  656.936314] watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  656.937474] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  656.937513]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  656.949583] irq event stamp: 0

[  656.950654] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  656.951731] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  656.952818] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  656.953887] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  656.954916] CPU: 6 PID: 12756 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  656.955949] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  656.956981] RIP: 0010:do_raw_spin_lock+0x3e/0xa0

[  656.957988] Code: 8b 43 10 65 48 8b 14 25 40 8e 01 00 48 39 c2 74 66 8b 53 08 65 8b 05 29 8c cf 69 39 c2 74 47 31 c0 ba 01 00 00 00 f0 0f b1 13 <0f> 94 c2 84 d2 74 27 65 8b 05 0c 8c cf 69 89 43 08 65 48 8b 04 25

[  656.959999] RSP: 0018:ffffa2d1430cfdc0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  656.961029] RAX: 0000000000000000 RBX: ffff8d8610a239f8 RCX: 0000000000000000

[  656.962055] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff8d8610a239f8

[  656.963070] RBP: ffff8d8610a239f8 R08: 0000000000000001 R09: 0000000000000000

[  656.964089] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001

[  656.965167] R13: ffff8d8562ea2800 R14: ffff8d8610a23be0 R15: ffff8d8610a238e0

[  656.966184] FS:  0000000000000000(0000) GS:ffff8d87dc200000(0000) knlGS:0000000000000000

[  656.967207] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  656.968231] CR2: 00007f52e7df0140 CR3: 00000002fd810001 CR4: 00000000003606e0

[  656.969271] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  656.970314] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  656.971391] Call Trace:

[  656.972500]  iscsit_stop_dataout_timer+0x19/0x80 [iscsi_target_mod]

[  656.973582]  __iscsit_free_cmd+0x249/0x2c0 [iscsi_target_mod]

[  656.974664]  iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  656.975741]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  656.976817]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  656.977892]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  656.978940]  kthread+0x103/0x140

[  656.979992]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  656.981046]  ? kthread_bind+0x10/0x10

[  656.982083]  ret_from_fork+0x3a/0x50

[  664.936435] watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [iscsi_trx:12748]

[  664.937608] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  664.937646]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  664.950547] irq event stamp: 0

[  664.951626] hardirqs last  enabled at (0): [<0000000000000000>] 0x0



Message from syslogd@mehrangarh at Feb 10 15:08:14 ...

 kernel:watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [iscsi_trx:12748]

[  664.952702] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  664.953783] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  664.954846] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  664.955900] CPU: 13 PID: 12748 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  664.956970] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  664.958041] RIP: 0010:__raw_spin_lock_init+0xa/0x50

[  664.959103] Code: c7 c7 08 c0 36 97 31 c0 e8 e7 f7 00 00 5b 48 c7 c7 b8 b4 39 97 31 c0 e9 d8 f7 00 00 5b c3 cc cc 0f 1f 44 00 00 41 54 49 89 d4 <55> 48 89 f5 be 38 00 00 00 53 48 89 fb e8 d4 82 ff ff 48 8d 7b 18

[  664.961224] RSP: 0018:ffffa2d14308fd78 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  664.962309] RAX: 0000000000000000 RBX: ffffa2d14308fdc0 RCX: 0000000000000000

[  664.963388] RDX: ffffffff9831f530 RSI: ffffffff97300412 RDI: ffffa2d14308fdc0

[  664.964490] RBP: ffffffffc0d67e09 R08: 0000000000000000 R09: 0000000000000000

[  664.965580] R10: 0000000000000000 R11: ffffa2d14308fdb0 R12: ffffffff9831f530

[  664.966667] R13: 0000000000000001 R14: ffff8d86111af460 R15: ffff8d86111af160

[  664.967745] FS:  0000000000000000(0000) GS:ffff8d87dec00000(0000) knlGS:0000000000000000

[  664.968810] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  664.969884] CR2: 00007f4df5029248 CR3: 00000002fd810003 CR4: 00000000003606e0

[  664.970957] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  664.972011] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  664.973058] Call Trace:

[  664.974097]  __init_waitqueue_head+0x25/0x50

[  664.975143]  transport_generic_free_cmd+0x40/0x160 [target_core_mod]

[  664.976168]  ? __local_bh_enable_ip+0x80/0xf0

[  664.977188]  ? __iscsit_free_cmd+0x281/0x2c0 [iscsi_target_mod]

[  664.978236]  iscsit_free_cmd+0x58/0xa0 [iscsi_target_mod]

[  664.979289]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  664.980348]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  664.981479]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  664.982563]  kthread+0x103/0x140

[  664.983632]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  664.984703]  ? kthread_bind+0x10/0x10

[  664.985762]  ret_from_fork+0x3a/0x50

[  673.196146] rcu: INFO: rcu_sched detected expedited stalls on CPUs/tasks: { 6-... 13-... } 68051 jiffies s: 409 root: 0x2040/.

[  673.199958] rcu: blocking rcu_node structures:

[  673.201895] Task dump for CPU 6:

[  673.203851] iscsi_trx       R  running task        0 12756      2 0x8000408c

[  673.205900] Call Trace:

[  673.207825]  ? trace_hardirqs_on_thunk+0x1a/0x1c

[  673.209691]  ? retint_kernel+0x10/0x10

[  673.211611]  ? trace_hardirqs_on_thunk+0x1a/0x1c

[  673.213445]  ? trace_hardirqs_on_thunk+0x1a/0x1c

[  673.215228]  ? __is_kernel_percpu_address+0x39/0xa0

[  673.216826]  ? __iscsit_free_cmd+0x28e/0x2c0 [iscsi_target_mod]

[  673.218524]  ? __iscsit_free_cmd+0x251/0x2c0 [iscsi_target_mod]

[  673.220046]  ? iscsit_stop_dataout_timer+0x19/0x80 [iscsi_target_mod]

[  673.221765]  ? iscsit_free_cmd+0x48/0xa0 [iscsi_target_mod]

[  673.223383]  ? iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  673.224960]  ? iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  673.226588]  ? iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  673.228207]  ? kthread+0x103/0x140

[  673.229733]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  673.231326]  ? kthread_bind+0x10/0x10

[  673.232884]  ? ret_from_fork+0x3a/0x50

[  673.234527] Task dump for CPU 13:

[  673.236003] iscsi_trx       R  running task        0 12748      2 0x8000408c

[  673.237607] Call Trace:

[  673.239116]  ? trace_hardirqs_on_thunk+0x1a/0x1c

[  673.240644]  ? run_timer_softirq+0x52d/0x570

[  673.242090]  ? run_timer_softirq+0x52d/0x570

[  673.243596]  ? retint_kernel+0x10/0x10

[  673.245020]  ? trace_hardirqs_on_thunk+0x1a/0x1c

[  673.246562]  ? iscsit_free_all_datain_reqs+0x24/0xa0 [iscsi_target_mod]

[  673.247962]  ? __iscsit_free_cmd+0x121/0x2c0 [iscsi_target_mod]

[  673.249398]  ? __iscsit_free_cmd+0x281/0x2c0 [iscsi_target_mod]

[  673.250740]  ? iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  673.252186]  ? iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  673.253560]  ? iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  673.255025]  ? iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  673.256578]  ? kthread+0x103/0x140

[  673.257991]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  673.259536]  ? kthread_bind+0x10/0x10

[  673.260961]  ? ret_from_fork+0x3a/0x50

[  684.936368] watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  684.937180] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  684.937210]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  684.946894] irq event stamp: 0

[  684.947821] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  684.948772] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  684.949743] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  684.950709] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  684.951673] CPU: 6 PID: 12756 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  684.952663] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  684.953657] RIP: 0010:lock_acquire+0xdf/0x1c0

[  684.954629] Code: 83 e1 01 e8 e3 e7 ff ff 65 48 8b 04 25 40 8e 01 00 48 89 df c7 80 bc 0c 00 00 00 00 00 00 57 9d 0f 1f 44 00 00 48 83 c4 30 5b <5d> 41 5c 41 5d 41 5e 41 5f c3 65 8b 05 38 d4 cf 69 89 c0 48 0f a3

[  684.956615] RSP: 0018:ffffa2d1430cfd88 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13

[  684.957650] RAX: ffff8d878eb1c180 RBX: ffff8d8610a239f8 RCX: 0000000000000000

[  684.958685] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000246

[  684.959714] RBP: 0000000000000000 R08: 0000000000000001 R09: 0000000000000000

[  684.960732] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000

[  684.961746] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000

[  684.962756] FS:  0000000000000000(0000) GS:ffff8d87dc200000(0000) knlGS:0000000000000000

[  684.963776] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  684.964769] CR2: 00007f52e7df0140 CR3: 00000002fd810001 CR4: 00000000003606e0

[  684.965769] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  684.966790] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  684.967829] Call Trace:

[  684.968828]  _raw_spin_lock_bh+0x3d/0x50

[  684.969833]  ? iscsit_stop_dataout_timer+0x19/0x80 [iscsi_target_mod]

[  684.970863]  iscsit_stop_dataout_timer+0x19/0x80 [iscsi_target_mod]

[  684.971890]  __iscsit_free_cmd+0x249/0x2c0 [iscsi_target_mod]

[  684.972977]  iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  684.974006]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  684.975041]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  684.976070]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  684.977092]  kthread+0x103/0x140

[  684.978104]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  684.979120]  ? kthread_bind+0x10/0x10

[  684.980119]  ret_from_fork+0x3a/0x50

[  692.936490] watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [iscsi_trx:12748]

[  692.937632] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  692.937671]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]



Message from syslogd@mehrangarh at Feb 10 15:08:42 ...

 kernel:watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [iscsi_trx:12748]

[  692.949728] irq event stamp: 0

[  692.950765] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  692.951807] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  692.952857] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  692.953898] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  692.954934] CPU: 13 PID: 12748 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  692.955975] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  692.957004] RIP: 0010:lock_release+0x0/0x2a0

[  692.958019] Code: 83 3d b3 84 5b 01 00 0f 85 04 ff ff ff 48 c7 c6 d0 23 30 97 48 c7 c7 a2 65 2e 97 31 c0 e8 08 b0 f8 ff 0f 0b e9 e8 fe ff ff 90 <41> 57 65 48 8b 14 25 40 8e 01 00 41 56 41 55 41 54 55 53 48 83 ec

[  692.960047] RSP: 0018:ffffa2d14308fdd0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  692.961073] RAX: 0000000000000000 RBX: ffff8d808737fe50 RCX: 0000000000000000

[  692.962097] RDX: 0000000000000001 RSI: ffffffffc0db70f1 RDI: ffff8d808737fe68

[  692.963115] RBP: ffff8d86111af470 R08: 0000000000000001 R09: 0000000000000000

[  692.964139] R10: 0000000000000000 R11: ffffa2d14308fdb0 R12: 0000000000000001

[  692.965165] R13: ffff8d808737f800 R14: ffff8d808737fe50 R15: ffff8d86111af160

[  692.966206] FS:  0000000000000000(0000) GS:ffff8d87dec00000(0000) knlGS:0000000000000000

[  692.967258] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  692.968301] CR2: 00007f4df5029248 CR3: 00000002fd810003 CR4: 00000000003606e0

[  692.969351] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  692.970413] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  692.971509] Call Trace:

[  692.972586]  _raw_spin_unlock_bh+0x17/0x30

[  692.973666]  __iscsit_free_cmd+0x281/0x2c0 [iscsi_target_mod]

[  692.974745]  iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  692.975812]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  692.976886]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  692.977961]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  692.979021]  kthread+0x103/0x140

[  692.980049]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  692.981103]  ? kthread_bind+0x10/0x10

[  692.982141]  ret_from_fork+0x3a/0x50

[  712.936369] watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [iscsi_trx:12756]

[  712.937563] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  712.937602]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  712.950360] irq event stamp: 0

[  712.951416] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  712.952502] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  712.953594] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  712.954669] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  712.955735] CPU: 6 PID: 12756 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  712.956820] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  712.957901] RIP: 0010:_raw_spin_unlock_irqrestore+0x3e/0x52

[  712.958967] Code: 24 10 e8 55 ef 78 ff 48 89 ef e8 9d 3b 79 ff f6 c7 02 74 19 e8 b3 11 85 ff 48 89 df 57 9d 0f 1f 44 00 00 65 ff 0d 9a 31 49 69 <5b> 5d c3 48 89 df 57 9d 0f 1f 44 00 00 e8 b0 10 85 ff eb e5 cc cc

[  712.961098] RSP: 0018:ffffa2d1430cfd90 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  712.962194] RAX: 0000000000000001 RBX: 0000000000000296 RCX: 0000000000000000

[  712.963247] RDX: 0000000000000006 RSI: ffffffffc0d53e4a RDI: 0000000000000296

[  712.964289] RBP: ffff8d8610a23cf8 R08: 0000000000000000 R09: 0000000000000001

[  712.965329] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001

[  712.966364] R13: 0000000000000001 R14: ffff8d8610a23be0 R15: ffff8d8610a238e0

[  712.967403] FS:  0000000000000000(0000) GS:ffff8d87dc200000(0000) knlGS:0000000000000000

[  712.968519] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  712.969577] CR2: 00007f52e7df0140 CR3: 00000002fd810001 CR4: 00000000003606e0

[  712.970632] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  712.971700] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  712.972754] Call Trace:

[  712.973786]  transport_generic_free_cmd+0x12a/0x160 [target_core_mod]

[  712.974854]  iscsit_free_cmd+0x58/0xa0 [iscsi_target_mod]

[  712.975906]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  712.976946]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  712.977989]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  712.979022]  kthread+0x103/0x140

[  712.980064]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  712.981111]  ? kthread_bind+0x10/0x10

[  712.982154]  ret_from_fork+0x3a/0x50

[  720.936424] watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [iscsi_trx:12748]

[  720.937584] Modules linked in: target_core_user uio target_core_pscsi target_core_file target_core_iblock mlx5_ib mlx5_core mlxfw pci_hyperv_intf nvmet_tcp nvmet_rdma nvmet rdma_ucm iw_cxgb4 ib_uverbs cxgb4 libcxgb netconsole xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rfkill ib_isert iscsi_target_mod target_core_mod sunrpc intel_rapl_msr intel_rapl_common sb_edac ib_umad joydev x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm snd_hda_codec_realtek snd_hda_codec_generic rdma_cm iw_cm ledtrig_audio snd_hda_intel irqbypass snd_intel_dspcfg ib_cm snd_hda_codec ib_core crct10dif_pclmul iTCO_wdt iTCO_vendor_support ipmi_ssif crc32_pclmul snd_hda_core ghash_clmulni_intel aesni_intel snd_hwdep crypto_simd snd_seq snd_seq_device cryptd ipmi_si glue_helper sg pcspkr ipmi_devintf snd_pcm

[  720.937622]  ipmi_msghandler snd_timer wmi snd soundcore lpc_ich i2c_i801 dm_multipath sch_fq_codel ip_tables ext4 mbcache jbd2 sd_mod t10_pi ast drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm igb ahci libahci crc32c_intel libata dca i2c_algo_bit i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ib_uverbs]

[  720.950527] irq event stamp: 0

[  720.951618] hardirqs last  enabled at (0): [<0000000000000000>] 0x0

[  720.952701] hardirqs last disabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  720.953765] softirqs last  enabled at (0): [<ffffffff9629d07f>] copy_process+0x76f/0x1f80

[  720.954818] softirqs last disabled at (0): [<0000000000000000>] 0x0

[  720.955860] CPU: 13 PID: 12748 Comm: iscsi_trx Kdump: loaded Tainted: G        W    L    5.6.0-rc1+ #18

[  720.956914] Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0a 06/23/2016

[  720.957958] RIP: 0010:lock_release+0x142/0x2a0

[  720.959010] Code: 11 44 2b 3c 24 44 39 bb b8 0c 00 00 0f 85 34 01 00 00 65 48 8b 04 25 40 8e 01 00 4c 89 ef c7 80 bc 0c 00 00 00 00 00 00 57 9d <0f> 1f 44 00 00 48 8b 44 24 08 65 48 33 04 25 28 00 00 00 0f 85 e9

[  720.961159] RSP: 0018:ffffa2d14308fd90 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13

[  720.962243] RAX: ffff8d8786724180 RBX: ffff8d8786724180 RCX: 0000000000000000

[  720.963316] RDX: 0000000000000000 RSI: ffffffffc0db70f1 RDI: 0000000000000246

[  720.964373] RBP: ffff8d808737fe68 R08: 0000000000000000 R09: 0000000000000000

[  720.965462] R10: 0000000000000000 R11: ffffa2d14308fdb0 R12: ffffffffc0db70f1

[  720.966542] R13: 0000000000000246 R14: ffff8d808737fe50 R15: ffff8d86111af160

[  720.967600] FS:  0000000000000000(0000) GS:ffff8d87dec00000(0000) knlGS:0000000000000000

[  720.968680] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[  720.969754] CR2: 00007f4df5029248 CR3: 00000002fd810003 CR4: 00000000003606e0

[  720.970831] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  720.971899] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400

[  720.972954] Call Trace:

[  720.973993]  _raw_spin_unlock_bh+0x17/0x30

[  720.975037]  __iscsit_free_cmd+0x281/0x2c0 [iscsi_target_mod]

[  720.976089]  iscsit_free_cmd+0x76/0xa0 [iscsi_target_mod]

[  720.977128]  iscsit_close_connection+0x570/0x920 [iscsi_target_mod]

[  720.978149]  iscsit_take_action_for_connection_exit+0x7e/0x100 [iscsi_target_mod]

[  720.979194]  iscsi_target_rx_thread+0xdf/0xf0 [iscsi_target_mod]

[  720.980237]  kthread+0x103/0x140

[  720.981271]  ? iscsi_target_tx_thread+0x210/0x210 [iscsi_target_mod]

[  720.982317]  ? kthread_bind+0x10/0x10

[  720.983414]  ret_from_fork+0x3a/0x50