[PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Hannes Reinecke]

When scheduling workqueue elements the callback function might be called
directly, so holding the event lock is potentially dangerous as it might
lead to a deadlock:

[  989.542827] INFO: task systemd-udevd:459 blocked for more than 480 seconds.
[  989.609721]       Not tainted 4.10.0-rc4+ #546
[  989.648545] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this 
message.
[  989.716429] systemd-udevd   D13368   459      1 0x00000004
[  989.716435] Call Trace:
[  989.716444]  __schedule+0x2f2/0xb10
[  989.716447]  schedule+0x3d/0x90
[  989.716449]  schedule_timeout+0x2fc/0x600
[  989.716451]  ? wait_for_completion+0xac/0x110
[  989.716456]  ? mark_held_locks+0x66/0x90
[  989.716458]  ? _raw_spin_unlock_irq+0x2c/0x40
[  989.716460]  ? trace_hardirqs_on_caller+0x111/0x1e0
[  989.716461]  wait_for_completion+0xb4/0x110
[  989.716464]  ? wake_up_q+0x80/0x80
[  989.716469]  flush_work+0x1ea/0x2a0
[  989.716470]  ? flush_work+0x24e/0x2a0
[  989.716472]  ? destroy_worker+0xd0/0xd0
[  989.716474]  __cancel_work_timer+0x11a/0x1e0
[  989.716476]  ? trace_hardirqs_on_caller+0x111/0x1e0
[  989.716477]  cancel_delayed_work_sync+0x13/0x20
[  989.716482]  disk_block_events+0x82/0x90
[  989.716487]  __blkdev_get+0x58/0x450
[  989.716488]  blkdev_get+0x1ce/0x340
[  989.716490]  ? _raw_spin_unlock+0x27/0x40
[  989.716492]  blkdev_open+0x5b/0x70
[  989.716501]  do_dentry_open+0x213/0x310
[  989.716505]  ? blkdev_get_by_dev+0x50/0x50
[  989.716507]  vfs_open+0x4f/0x80
[  989.716518]  ? may_open+0x9b/0x100
[  989.716521]  path_openat+0x48a/0xdc0
[  989.716527]  ? _crng_backtrack_protect+0x30/0x80
[  989.716530]  do_filp_open+0x7e/0xd0
[  989.716533]  ? _raw_spin_unlock+0x27/0x40
[  989.716537]  ? __alloc_fd+0xf7/0x210
[  989.716539]  do_sys_open+0x115/0x1f0
[  989.716542]  SyS_open+0x1e/0x20
[  989.716546]  entry_SYSCALL_64_fastpath+0x23/0xc6

Signed-off-by: Hannes Reinecke <hare@suse.com>

diff --git a/block/genhd.c b/block/genhd.c
index fcd6d4f..ae46caa 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1426,8 +1426,7 @@ struct disk_events {
 	struct gendisk		*disk;		/* the associated disk */
 	spinlock_t		lock;
 
-	struct mutex		block_mutex;	/* protects blocking */
-	int			block;		/* event blocking depth */
+	atomic_t		block;		/* event blocking depth */
 	unsigned int		pending;	/* events already sent out */
 	unsigned int		clearing;	/* events being cleared */
 
@@ -1488,26 +1487,13 @@ static unsigned long disk_events_poll_jiffies(struct gendisk *disk)
 void disk_block_events(struct gendisk *disk)
 {
 	struct disk_events *ev = disk->ev;
-	unsigned long flags;
-	bool cancel;
 
 	if (!ev)
 		return;
 
-	/*
-	 * Outer mutex ensures that the first blocker completes canceling
-	 * the event work before further blockers are allowed to finish.
-	 */
-	mutex_lock(&ev->block_mutex);
-
-	spin_lock_irqsave(&ev->lock, flags);
-	cancel = !ev->block++;
-	spin_unlock_irqrestore(&ev->lock, flags);
-
-	if (cancel)
+	if (atomic_inc_return(&ev->block) == 1)
 		cancel_delayed_work_sync(&disk->ev->dwork);
 
-	mutex_unlock(&ev->block_mutex);
 }
 
 static void __disk_unblock_events(struct gendisk *disk, bool check_now)
@@ -1516,23 +1502,18 @@ static void __disk_unblock_events(struct gendisk *disk, bool check_now)
 	unsigned long intv;
 	unsigned long flags;
 
-	spin_lock_irqsave(&ev->lock, flags);
-
-	if (WARN_ON_ONCE(ev->block <= 0))
-		goto out_unlock;
-
-	if (--ev->block)
-		goto out_unlock;
+	if (atomic_dec_return(&ev->block) > 0)
+		return;
 
+	spin_lock_irqsave(&ev->lock, flags);
 	intv = disk_events_poll_jiffies(disk);
+	spin_unlock_irqrestore(&ev->lock, flags);
 	if (check_now)
 		queue_delayed_work(system_freezable_power_efficient_wq,
 				&ev->dwork, 0);
 	else if (intv)
 		queue_delayed_work(system_freezable_power_efficient_wq,
 				&ev->dwork, intv);
-out_unlock:
-	spin_unlock_irqrestore(&ev->lock, flags);
 }
 
 /**
@@ -1572,10 +1553,10 @@ void disk_flush_events(struct gendisk *disk, unsigned int mask)
 
 	spin_lock_irq(&ev->lock);
 	ev->clearing |= mask;
-	if (!ev->block)
+	spin_unlock_irq(&ev->lock);
+	if (!atomic_read(&ev->block))
 		mod_delayed_work(system_freezable_power_efficient_wq,
 				&ev->dwork, 0);
-	spin_unlock_irq(&ev->lock);
 }
 
 /**
@@ -1666,12 +1647,11 @@ static void disk_check_events(struct disk_events *ev,
 	*clearing_ptr &= ~clearing;
 
 	intv = disk_events_poll_jiffies(disk);
-	if (!ev->block && intv)
+	spin_unlock_irq(&ev->lock);
+	if (!atomic_read(&ev->block) && intv)
 		queue_delayed_work(system_freezable_power_efficient_wq,
 				&ev->dwork, intv);
 
-	spin_unlock_irq(&ev->lock);
-
 	/*
 	 * Tell userland about new events.  Only the events listed in
 	 * @disk->events are reported.  Unlisted events are processed the
@@ -1824,8 +1804,7 @@ static void disk_alloc_events(struct gendisk *disk)
 	INIT_LIST_HEAD(&ev->node);
 	ev->disk = disk;
 	spin_lock_init(&ev->lock);
-	mutex_init(&ev->block_mutex);
-	ev->block = 1;
+	atomic_set(&ev->block, 1);
 	ev->poll_msecs = -1;
 	INIT_DELAYED_WORK(&ev->dwork, disk_events_workfn);
 
@@ -1870,6 +1849,6 @@ static void disk_del_events(struct gendisk *disk)
 static void disk_release_events(struct gendisk *disk)
 {
 	/* the block count should be 1 from disk_del_events() */
-	WARN_ON_ONCE(disk->ev && disk->ev->block != 1);
+	WARN_ON_ONCE(disk->ev && atomic_read(&disk->ev->block) != 1);
 	kfree(disk->ev);
 }

Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Bart Van Assche]

On Wed, 2017-01-18 at 10:48 +0100, Hannes Reinecke wrote:
> @@ -1488,26 +1487,13 @@ static unsigned long disk_events_poll_jiffies(str=
uct gendisk *disk)
> =A0void disk_block_events(struct gendisk *disk)
> =A0{
> =A0=A0=A0=A0=A0=A0=A0=A0struct disk_events *ev =3D disk->ev;
> -=A0=A0=A0=A0=A0=A0=A0unsigned long flags;
> -=A0=A0=A0=A0=A0=A0=A0bool cancel;
> =A0
> =A0=A0=A0=A0=A0=A0=A0=A0if (!ev)
> =A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0return;
> =A0
> -=A0=A0=A0=A0=A0=A0=A0/*
> -=A0=A0=A0=A0=A0=A0=A0 * Outer mutex ensures that the first blocker compl=
etes canceling
> -=A0=A0=A0=A0=A0=A0=A0 * the event work before further blockers are allow=
ed to finish.
> -=A0=A0=A0=A0=A0=A0=A0 */
> -=A0=A0=A0=A0=A0=A0=A0mutex_lock(&ev->block_mutex);
> -
> -=A0=A0=A0=A0=A0=A0=A0spin_lock_irqsave(&ev->lock, flags);
> -=A0=A0=A0=A0=A0=A0=A0cancel =3D !ev->block++;
> -=A0=A0=A0=A0=A0=A0=A0spin_unlock_irqrestore(&ev->lock, flags);
> -
> -=A0=A0=A0=A0=A0=A0=A0if (cancel)
> +=A0=A0=A0=A0=A0=A0=A0if (atomic_inc_return(&ev->block) =3D=3D 1)
> =A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0cancel_delayed_work_sync(=
&disk->ev->dwork);
> =A0
> -=A0=A0=A0=A0=A0=A0=A0mutex_unlock(&ev->block_mutex);
> =A0}

Hello Hannes,

I have already encountered a few times a deadlock that was caused by the
event checking code so I agree with you that it would be a big step forward
if such deadlocks wouldn't occur anymore. However, this patch realizes a
change that has not been described in the patch description, namely that
disk_block_events() calls are no longer serialized. Are you sure it is safe
to drop the serialization of disk_block_events() calls?

Thanks,

Bart.=

Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Bart Van Assche]

On Wed, 2017-01-18 at 10:48 +0100, Hannes Reinecke wrote:
> @@ -1488,26 +1487,13 @@ static unsigned long disk_events_poll_jiffies(struct gendisk *disk)
>  void disk_block_events(struct gendisk *disk)
>  {
>         struct disk_events *ev = disk->ev;
> -       unsigned long flags;
> -       bool cancel;
>  
>         if (!ev)
>                 return;
>  
> -       /*
> -        * Outer mutex ensures that the first blocker completes canceling
> -        * the event work before further blockers are allowed to finish.
> -        */
> -       mutex_lock(&ev->block_mutex);
> -
> -       spin_lock_irqsave(&ev->lock, flags);
> -       cancel = !ev->block++;
> -       spin_unlock_irqrestore(&ev->lock, flags);
> -
> -       if (cancel)
> +       if (atomic_inc_return(&ev->block) == 1)
>                 cancel_delayed_work_sync(&disk->ev->dwork);
>  
> -       mutex_unlock(&ev->block_mutex);
>  }

Hello Hannes,

I have already encountered a few times a deadlock that was caused by the
event checking code so I agree with you that it would be a big step forward
if such deadlocks wouldn't occur anymore. However, this patch realizes a
change that has not been described in the patch description, namely that
disk_block_events() calls are no longer serialized. Are you sure it is safe
to drop the serialization of disk_block_events() calls?

Thanks,

Bart.

Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Hannes Reinecke]

On 01/31/2017 01:31 AM, Bart Van Assche wrote:
> On Wed, 2017-01-18 at 10:48 +0100, Hannes Reinecke wrote:
>> @@ -1488,26 +1487,13 @@ static unsigned long disk_events_poll_jiffies(struct gendisk *disk)
>>  void disk_block_events(struct gendisk *disk)
>>  {
>>         struct disk_events *ev = disk->ev;
>> -       unsigned long flags;
>> -       bool cancel;
>>  
>>         if (!ev)
>>                 return;
>>  
>> -       /*
>> -        * Outer mutex ensures that the first blocker completes canceling
>> -        * the event work before further blockers are allowed to finish.
>> -        */
>> -       mutex_lock(&ev->block_mutex);
>> -
>> -       spin_lock_irqsave(&ev->lock, flags);
>> -       cancel = !ev->block++;
>> -       spin_unlock_irqrestore(&ev->lock, flags);
>> -
>> -       if (cancel)
>> +       if (atomic_inc_return(&ev->block) == 1)
>>                 cancel_delayed_work_sync(&disk->ev->dwork);
>>  
>> -       mutex_unlock(&ev->block_mutex);
>>  }
> 
> Hello Hannes,
> 
> I have already encountered a few times a deadlock that was caused by the
> event checking code so I agree with you that it would be a big step forward
> if such deadlocks wouldn't occur anymore. However, this patch realizes a
> change that has not been described in the patch description, namely that
> disk_block_events() calls are no longer serialized. Are you sure it is safe
> to drop the serialization of disk_block_events() calls?
> 
Well, this whole synchronization stuff it a bit weird; I so totally fail
to see the rationale for it.
But anyway, once we've converted ev->block to atomics I _think_ the
mutex_lock can remain; will be checking.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		               zSeries & Storage
hare@suse.com			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N�rnberg
GF: F. Imend�rffer, J. Smithard, D. Upmanyu, G. Norton
HRB 21284 (AG N�rnberg)

Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Hannes Reinecke]

On 01/31/2017 01:31 AM, Bart Van Assche wrote:
> On Wed, 2017-01-18 at 10:48 +0100, Hannes Reinecke wrote:
>> @@ -1488,26 +1487,13 @@ static unsigned long disk_events_poll_jiffies(struct gendisk *disk)
>>  void disk_block_events(struct gendisk *disk)
>>  {
>>         struct disk_events *ev = disk->ev;
>> -       unsigned long flags;
>> -       bool cancel;
>>  
>>         if (!ev)
>>                 return;
>>  
>> -       /*
>> -        * Outer mutex ensures that the first blocker completes canceling
>> -        * the event work before further blockers are allowed to finish.
>> -        */
>> -       mutex_lock(&ev->block_mutex);
>> -
>> -       spin_lock_irqsave(&ev->lock, flags);
>> -       cancel = !ev->block++;
>> -       spin_unlock_irqrestore(&ev->lock, flags);
>> -
>> -       if (cancel)
>> +       if (atomic_inc_return(&ev->block) == 1)
>>                 cancel_delayed_work_sync(&disk->ev->dwork);
>>  
>> -       mutex_unlock(&ev->block_mutex);
>>  }
> 
> Hello Hannes,
> 
> I have already encountered a few times a deadlock that was caused by the
> event checking code so I agree with you that it would be a big step forward
> if such deadlocks wouldn't occur anymore. However, this patch realizes a
> change that has not been described in the patch description, namely that
> disk_block_events() calls are no longer serialized. Are you sure it is safe
> to drop the serialization of disk_block_events() calls?
> 
Well, this whole synchronization stuff it a bit weird; I so totally fail
to see the rationale for it.
But anyway, once we've converted ev->block to atomics I _think_ the
mutex_lock can remain; will be checking.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		               zSeries & Storage
hare@suse.com			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: F. Imendörffer, J. Smithard, D. Upmanyu, G. Norton
HRB 21284 (AG Nürnberg)

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

> From: linux-kernel-owner@vger.kernel.org [mailto:linux-kernel-
> owner@vger.kernel.org] On Behalf Of Hannes Reinecke
> Sent: Wednesday, February 1, 2017 00:15
> To: Bart Van Assche <Bart.VanAssche@sandisk.com>; hare@suse.de;
> axboe@kernel.dk
> Cc: hch@lst.de; linux-kernel@vger.kernel.org; linux-block@vger.kernel.org=
;
> jth@kernel.org
> Subject: Re: [PATCH] genhd: Do not hold event lock when scheduling workqu=
eue
> elements
>=20
> On 01/31/2017 01:31 AM, Bart Van Assche wrote:
> > On Wed, 2017-01-18 at 10:48 +0100, Hannes Reinecke wrote:
> >> @@ -1488,26 +1487,13 @@ static unsigned long
> disk_events_poll_jiffies(struct gendisk *disk)
> >>  void disk_block_events(struct gendisk *disk)
> >>  {
> >>         struct disk_events *ev =3D disk->ev;
> >> -       unsigned long flags;
> >> -       bool cancel;
> >>
> >>         if (!ev)
> >>                 return;
> >>
> >> -       /*
> >> -        * Outer mutex ensures that the first blocker completes cancel=
ing
> >> -        * the event work before further blockers are allowed to finis=
h.
> >> -        */
> >> -       mutex_lock(&ev->block_mutex);
> >> -
> >> -       spin_lock_irqsave(&ev->lock, flags);
> >> -       cancel =3D !ev->block++;
> >> -       spin_unlock_irqrestore(&ev->lock, flags);
> >> -
> >> -       if (cancel)
> >> +       if (atomic_inc_return(&ev->block) =3D=3D 1)
> >>                 cancel_delayed_work_sync(&disk->ev->dwork);
> >>
> >> -       mutex_unlock(&ev->block_mutex);
> >>  }
> >
> > Hello Hannes,
> >
> > I have already encountered a few times a deadlock that was caused by th=
e
> > event checking code so I agree with you that it would be a big step for=
ward
> > if such deadlocks wouldn't occur anymore. However, this patch realizes =
a
> > change that has not been described in the patch description, namely tha=
t
> > disk_block_events() calls are no longer serialized. Are you sure it is =
safe
> > to drop the serialization of disk_block_events() calls?
> >
> Well, this whole synchronization stuff it a bit weird; I so totally fail
> to see the rationale for it.
> But anyway, once we've converted ev->block to atomics I _think_ the
> mutex_lock can remain; will be checking.
>=20
> Cheers,
>=20
> Hannes
> --

Hi, I think I got the same calltrace with today's linux-next (next-20170203=
).

The issue happened every time when my Linux virtual machine booted and
Hannes's patch could NOT help.

The calltrace is pasted below.

Thanks,
-- Dexuan

[    9.718802] scsi host2: storvsc_host_t
[    9.723854] scsi 2:0:0:0: Direct-Access     Msft     Virtual Disk     1.=
0  PQ: 0 ANSI: 5
[    9.753161] sd 2:0:0:0: Attached scsi generic sg1 type 0
[    9.766383] scsi host3: storvsc_host_t
[    9.771759] scsi 3:0:0:0: Direct-Access     Msft     Virtual Disk     1.=
0  PQ: 0 ANSI: 5
[    9.781836] hv_utils: VSS IC version 5.0
[    9.822511] sd 3:0:0:0: Attached scsi generic sg2 type 0
[    9.829039] sd 3:0:0:0: [sdb] 266338304 512-byte logical blocks: (136 GB=
/127 GiB)
[    9.838525] sd 3:0:0:0: [sdb] 4096-byte physical blocks
[    9.845350] sd 3:0:0:0: [sdb] Write Protect is off
[    9.851077] sd 3:0:0:0: [sdb] Mode Sense: 0f 00 00 00
[    9.859765] sd 3:0:0:0: [sdb] Write cache: enabled, read cache: enabled,=
 doesn't support DPO or FUA
[    9.872728]  sdb: sdb1
[    9.877279] sd 3:0:0:0: [sdb] Attached SCSI disk
[    9.964093] psmouse serio1: trackpoint: failed to get extended button da=
ta
[   14.864110] psmouse serio1: trackpoint: IBM TrackPoint firmware: 0x01, b=
uttons: 0/0
[   14.876423] input: TPPS/2 IBM TrackPoint as /devices/platform/i8042/seri=
o1/input/input3
[   14.887216] input: AT Translated Set 2 keyboard as /devices/LNXSYSTM:00/=
LNXSYBUS:00/PNP0A03:00/device:07/VMBUS:01/d34b2567-b9b6-42b9-8778-0a4ec0b95=
5bf/serio2/input/input5
[   44.644061] random: crng init done
[   66.524169] hv_utils: KVP IC version 4.0
Begin: Loading essential drivers ... done.
Begin: Running /scripts/init-premount ... done.
Begin: Mounting root file system ... Begin: Running /scripts/local-top ... =
done.
Begin: Running /scripts/local-premount ... done.
Begin: Waiting for root file system ... Begin: Running /scripts/local-block=
 ... done.
Begin: Running /scripts/local-block ... done.
Begin: Running /scripts/local-block ... done.
Begin: Running /scripts/local-block ... done.
[  242.652127] INFO: task systemd-udevd:183 blocked for more than 120 secon=
ds.
[  242.661008]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  242.697270] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables =
this message.
[  242.707872] systemd-udevd   D    0   183    170 0x00000004
[  242.714654] Call Trace:
[  242.717904]  __schedule+0x27d/0x8d0
[  242.724618]  schedule+0x36/0x80
[  242.729042]  schedule_timeout+0x235/0x3f0
[  242.736667]  ? sched_clock+0x9/0x10
[  242.741144]  ? try_to_wake_up+0x4a/0x460
[  242.745580]  wait_for_completion+0xa5/0x120
[  242.751417]  ? wake_up_q+0x70/0x70
[  242.759394]  flush_work+0x11a/0x1c0
[  242.764846]  ? worker_detach_from_pool+0xb0/0xb0
[  242.770943]  __cancel_work_timer+0xf3/0x1b0
[  242.776073]  ? disk_map_sector_rcu+0x70/0x70
[  242.783857]  cancel_delayed_work_sync+0x13/0x20
[  242.789438]  disk_block_events+0x34/0x40
[  242.794641]  __blkdev_get+0x10c/0x470
[  242.799227]  blkdev_get+0x11a/0x320
[  242.803161]  ? unlock_new_inode+0x49/0x80
[  242.807451]  ? bdget+0x110/0x130
[  242.811415]  blkdev_open+0x5b/0x70
[  242.820215]  do_dentry_open+0x208/0x310
[  242.825336]  ? blkdev_get_by_dev+0x50/0x50
[  242.836362]  vfs_open+0x4c/0x70
[  242.839839]  ? may_open+0x9b/0x100
[  242.843803]  path_openat+0x297/0x13e0
[  242.848410]  ? _copy_to_user+0x2e/0x40
[  242.852966]  ? move_addr_to_user+0xa3/0xc0
[  242.857961]  do_filp_open+0x7e/0xe0
[  242.862154]  ? _cond_resched+0x1a/0x50
[  242.867045]  ? kmem_cache_alloc+0x156/0x1b0
[  242.872004]  ? getname_flags+0x56/0x1f0
[  242.881609]  ? __alloc_fd+0x46/0x170
[  242.891812]  do_sys_open+0x11b/0x1f0
[  242.896506]  SyS_open+0x1e/0x20
[  242.900365]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  242.905607] RIP: 0033:0x7efd3827aca0
[  242.909773] RSP: 002b:00007ffd7ceaeb38 EFLAGS: 00000246 ORIG_RAX: 000000=
0000000002
[  242.918778] RAX: ffffffffffffffda RBX: 0000555fdf1a2d40 RCX: 00007efd382=
7aca0
[  242.925725] RDX: 0000555fde106058 RSI: 00000000000a0800 RDI: 0000555fdf1=
a2ff0
[  242.929715] RBP: 0000000000000000 R08: 0000000000000073 R09: 00000000000=
00003
[  242.933921] R10: 00007efd37fe6520 R11: 0000000000000246 R12: 00000000000=
00000
[  242.942260] R13: 0000555fdf1a2d40 R14: 00007ffd7ceaea00 R15: 00000000000=
00000
[  242.952780] INFO: task systemd-udevd:191 blocked for more than 120 secon=
ds.
[  242.960994]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  242.966936] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables =
this message.
[  242.975395] systemd-udevd   D    0   191    170 0x00000004
[  242.983460] Call Trace:
[  242.986694]  __schedule+0x27d/0x8d0
[  242.992603]  ? mutex_lock+0x12/0x40
[  242.996467]  ? kprobes_module_callback+0x15b/0x1d0
[  243.001934]  schedule+0x36/0x80
[  243.005908]  async_synchronize_cookie_domain+0x91/0x130
[  243.012331]  ? wake_atomic_t_function+0x60/0x60
[  243.017690]  async_synchronize_full+0x17/0x20
[  243.022662]  do_init_module+0xc1/0x1ff
[  243.026965]  load_module+0x2313/0x29a0
[  243.031232]  ? __symbol_put+0x40/0x40
[  243.035347]  ? security_kernel_post_read_file+0x6b/0x80
[  243.042541]  SYSC_finit_module+0xbc/0xf0
[  243.045130]  SyS_finit_module+0xe/0x10
[  243.049116]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  243.054517] RIP: 0033:0x7efd37fa1c19
[  243.058744] RSP: 002b:00007ffd7ceae188 EFLAGS: 00000246 ORIG_RAX: 000000=
0000000139
[  243.067483] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007efd37f=
a1c19
[  243.075801] RDX: 0000000000000000 RSI: 00007efd38497e2a RDI: 00000000000=
0000c
[  243.084162] RBP: 00007ffd7cead190 R08: 0000000000000000 R09: 00000000000=
00000
[  243.093738] R10: 000000000000000c R11: 0000000000000246 R12: 0000555fdf1=
a3300
[  243.103158] R13: 00007ffd7cead170 R14: 0000000000000005 R15: 000000000ab=
a9500
[  363.484110] INFO: task systemd-udevd:183 blocked for more than 120 secon=
ds.
[  363.493200]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  363.500109] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables =
this message.
[  363.509635] systemd-udevd   D    0   183    170 0x00000004
[  363.516428] Call Trace:
[  363.519492]  __schedule+0x27d/0x8d0
[  363.523850]  schedule+0x36/0x80
[  363.529927]  schedule_timeout+0x235/0x3f0
[  363.534884]  ? sched_clock+0x9/0x10
[  363.539006]  ? try_to_wake_up+0x4a/0x460
[  363.543686]  wait_for_completion+0xa5/0x120
[  363.548831]  ? wake_up_q+0x70/0x70
[  363.552993]  flush_work+0x11a/0x1c0
[  363.557236]  ? worker_detach_from_pool+0xb0/0xb0
[  363.562851]  __cancel_work_timer+0xf3/0x1b0
[  363.567942]  ? disk_map_sector_rcu+0x70/0x70
[  363.572973]  cancel_delayed_work_sync+0x13/0x20
[  363.580271]  disk_block_events+0x34/0x40
[  363.585284]  __blkdev_get+0x10c/0x470
[  363.589774]  blkdev_get+0x11a/0x320
[  363.593851]  ? unlock_new_inode+0x49/0x80
[  363.598509]  ? bdget+0x110/0x130
[  363.602272]  blkdev_open+0x5b/0x70
[  363.606254]  do_dentry_open+0x208/0x310
[  363.610722]  ? blkdev_get_by_dev+0x50/0x50
[  363.615493]  vfs_open+0x4c/0x70
[  363.619287]  ? may_open+0x9b/0x100
[  363.623386]  path_openat+0x297/0x13e0
[  363.627752]  ? _copy_to_user+0x2e/0x40
[  363.633978]  ? move_addr_to_user+0xa3/0xc0
[  363.638833]  do_filp_open+0x7e/0xe0
[  363.642809]  ? _cond_resched+0x1a/0x50
[  363.647070]  ? kmem_cache_alloc+0x156/0x1b0
[  363.651785]  ? getname_flags+0x56/0x1f0
[  363.656285]  ? __alloc_fd+0x46/0x170
[  363.660490]  do_sys_open+0x11b/0x1f0
[  363.664655]  SyS_open+0x1e/0x20
[  363.668405]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  363.674075] RIP: 0033:0x7efd3827aca0
[  363.678582] RSP: 002b:00007ffd7ceaeb38 EFLAGS: 00000246 ORIG_RAX: 000000=
0000000002
[  363.689873] RAX: ffffffffffffffda RBX: 0000555fdf1a2d40 RCX: 00007efd382=
7aca0
[  363.698127] RDX: 0000555fde106058 RSI: 00000000000a0800 RDI: 0000555fdf1=
a2ff0
[  363.706494] RBP: 0000000000000000 R08: 0000000000000073 R09: 00000000000=
00003
[  363.714696] R10: 00007efd37fe6520 R11: 0000000000000246 R12: 00000000000=
00000
[  363.722922] R13: 0000555fdf1a2d40 R14: 00007ffd7ceaea00 R15: 00000000000=
00000
[  363.731123] INFO: task systemd-udevd:191 blocked for more than 120 secon=
ds.
[  363.739454]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  363.745246] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables =
this message.
[  363.753890] systemd-udevd   D    0   191    170 0x00000004
[  363.760085] Call Trace:
[  363.763006]  __schedule+0x27d/0x8d0
[  363.767016]  ? mutex_lock+0x12/0x40
[  363.770943]  ? kprobes_module_callback+0x15b/0x1d0
[  363.780593]  schedule+0x36/0x80
[  363.785119]  async_synchronize_cookie_domain+0x91/0x130
[  363.793021]  ? wake_atomic_t_function+0x60/0x60
[  363.798351]  async_synchronize_full+0x17/0x20
[  363.803405]  do_init_module+0xc1/0x1ff
[  363.807564]  load_module+0x2313/0x29a0
[  363.811786]  ? __symbol_put+0x40/0x40
[  363.815895]  ? security_kernel_post_read_file+0x6b/0x80
[  363.821852]  SYSC_finit_module+0xbc/0xf0
[  363.826433]  SyS_finit_module+0xe/0x10
[  363.830732]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  363.836311] RIP: 0033:0x7efd37fa1c19
[  363.842740] RSP: 002b:00007ffd7ceae188 EFLAGS: 00000246 ORIG_RAX: 000000=
0000000139
[  363.852003] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007efd37f=
a1c19
[  363.860774] RDX: 0000000000000000 RSI: 00007efd38497e2a RDI: 00000000000=
0000c
[  363.869669] RBP: 00007ffd7cead190 R08: 0000000000000000 R09: 00000000000=
00000
[  363.878494] R10: 000000000000000c R11: 0000000000000246 R12: 0000555fdf1=
a3300
[  363.887931] R13: 00007ffd7cead170 R14: 0000000000000005 R15: 000000000ab=
a9500
[  484.316121] INFO: task systemd-udevd:183 blocked for more than 120 secon=
ds.
[  484.327141]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  484.333747] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables =
this message.
[  484.342486] systemd-udevd   D    0   183    170 0x00000004
[  484.348790] Call Trace:
[  484.351712]  __schedule+0x27d/0x8d0
[  484.355904]  schedule+0x36/0x80
[  484.359718]  schedule_timeout+0x235/0x3f0
[  484.364932]  ? sched_clock+0x9/0x10
[  484.368892]  ? try_to_wake_up+0x4a/0x460
[  484.376582]  wait_for_completion+0xa5/0x120
[  484.383779]  ? wake_up_q+0x70/0x70
[  484.389885]  flush_work+0x11a/0x1c0
[  484.394048]  ? worker_detach_from_pool+0xb0/0xb0
[  484.399464]  __cancel_work_timer+0xf3/0x1b0
[  484.404326]  ? disk_map_sector_rcu+0x70/0x70
[  484.409379]  cancel_delayed_work_sync+0x13/0x20
[  484.414449]  disk_block_events+0x34/0x40
[  484.418615]  __blkdev_get+0x10c/0x470
[  484.423391]  blkdev_get+0x11a/0x320
[  484.429312]  ? unlock_new_inode+0x49/0x80
[  484.433618]  ? bdget+0x110/0x130
[  484.437300]  blkdev_open+0x5b/0x70
[  484.441376]  do_dentry_open+0x208/0x310
[  484.446108]  ? blkdev_get_by_dev+0x50/0x50
[  484.450946]  vfs_open+0x4c/0x70
[  484.454693]  ? may_open+0x9b/0x100
[  484.458682]  path_openat+0x297/0x13e0
[  484.462656]  ? _copy_to_user+0x2e/0x40
[  484.467125]  ? move_addr_to_user+0xa3/0xc0
[  484.472370]  do_filp_open+0x7e/0xe0
[  484.479195]  ? _cond_resched+0x1a/0x50
[  484.484027]  ? kmem_cache_alloc+0x156/0x1b0
[  484.488851]  ? getname_flags+0x56/0x1f0
[  484.493049]  ? __alloc_fd+0x46/0x170
[  484.496899]  do_sys_open+0x11b/0x1f0
[  484.500784]  SyS_open+0x1e/0x20
[  484.504284]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  484.510767] RIP: 0033:0x7efd3827aca0
[  484.515460] RSP: 002b:00007ffd7ceaeb38 EFLAGS: 00000246 ORIG_RAX: 000000=
0000000002
[  484.527246] RAX: ffffffffffffffda RBX: 0000555fdf1a2d40 RCX: 00007efd382=
7aca0
[  484.537307] RDX: 0000555fde106058 RSI: 00000000000a0800 RDI: 0000555fdf1=
a2ff0
[  484.545620] RBP: 0000000000000000 R08: 0000000000000073 R09: 00000000000=
00003
[  484.554372] R10: 00007efd37fe6520 R11: 0000000000000246 R12: 00000000000=
00000
[  484.563267] R13: 0000555fdf1a2d40 R14: 00007ffd7ceaea00 R15: 00000000000=
00000
[  484.571966] INFO: task systemd-udevd:191 blocked for more than 120 secon=
ds.
[  484.580711]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  484.589555] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables =
this message.
[  484.598799] systemd-udevd   D    0   191    170 0x00000004
[  484.605231] Call Trace:
[  484.608308]  __schedule+0x27d/0x8d0
[  484.612702]  ? mutex_lock+0x12/0x40
[  484.617071]  ? kprobes_module_callback+0x15b/0x1d0
[  484.621625]  schedule+0x36/0x80
[  484.623511]  async_synchronize_cookie_domain+0x91/0x130
[  484.626443]  ? wake_atomic_t_function+0x60/0x60
[  484.629059]  async_synchronize_full+0x17/0x20
[  484.635723]  do_init_module+0xc1/0x1ff
[  484.640226]  load_module+0x2313/0x29a0
[  484.645030]  ? __symbol_put+0x40/0x40
[  484.649870]  ? security_kernel_post_read_file+0x6b/0x80
[  484.658016]  SYSC_finit_module+0xbc/0xf0
[  484.662676]  SyS_finit_module+0xe/0x10
[  484.667374]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  484.673822] RIP: 0033:0x7efd37fa1c19
[  484.677771] RSP: 002b:00007ffd7ceae188 EFLAGS: 00000246 ORIG_RAX: 000000=
0000000139
[  484.688668] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007efd37f=
a1c19
[  484.697959] RDX: 0000000000000000 RSI: 00007efd38497e2a RDI: 00000000000=
0000c
[  484.707793] RBP: 00007ffd7cead190 R08: 0000000000000000 R09: 00000000000=
00000
[  484.716695] R10: 000000000000000c R11: 0000000000000246 R12: 0000555fdf1=
a3300
[  484.724415] R13: 00007ffd7cead170 R14: 0000000000000005 R15: 000000000ab=
a9500

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

> From: linux-kernel-owner@vger.kernel.org [mailto:linux-kernel-
> owner@vger.kernel.org] On Behalf Of Hannes Reinecke
> Sent: Wednesday, February 1, 2017 00:15
> To: Bart Van Assche <Bart.VanAssche@sandisk.com>; hare@suse.de;
> axboe@kernel.dk
> Cc: hch@lst.de; linux-kernel@vger.kernel.org; linux-block@vger.kernel.org;
> jth@kernel.org
> Subject: Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue
> elements
> 
> On 01/31/2017 01:31 AM, Bart Van Assche wrote:
> > On Wed, 2017-01-18 at 10:48 +0100, Hannes Reinecke wrote:
> >> @@ -1488,26 +1487,13 @@ static unsigned long
> disk_events_poll_jiffies(struct gendisk *disk)
> >>  void disk_block_events(struct gendisk *disk)
> >>  {
> >>         struct disk_events *ev = disk->ev;
> >> -       unsigned long flags;
> >> -       bool cancel;
> >>
> >>         if (!ev)
> >>                 return;
> >>
> >> -       /*
> >> -        * Outer mutex ensures that the first blocker completes canceling
> >> -        * the event work before further blockers are allowed to finish.
> >> -        */
> >> -       mutex_lock(&ev->block_mutex);
> >> -
> >> -       spin_lock_irqsave(&ev->lock, flags);
> >> -       cancel = !ev->block++;
> >> -       spin_unlock_irqrestore(&ev->lock, flags);
> >> -
> >> -       if (cancel)
> >> +       if (atomic_inc_return(&ev->block) == 1)
> >>                 cancel_delayed_work_sync(&disk->ev->dwork);
> >>
> >> -       mutex_unlock(&ev->block_mutex);
> >>  }
> >
> > Hello Hannes,
> >
> > I have already encountered a few times a deadlock that was caused by the
> > event checking code so I agree with you that it would be a big step forward
> > if such deadlocks wouldn't occur anymore. However, this patch realizes a
> > change that has not been described in the patch description, namely that
> > disk_block_events() calls are no longer serialized. Are you sure it is safe
> > to drop the serialization of disk_block_events() calls?
> >
> Well, this whole synchronization stuff it a bit weird; I so totally fail
> to see the rationale for it.
> But anyway, once we've converted ev->block to atomics I _think_ the
> mutex_lock can remain; will be checking.
> 
> Cheers,
> 
> Hannes
> --

Hi, I think I got the same calltrace with today's linux-next (next-20170203).

The issue happened every time when my Linux virtual machine booted and
Hannes's patch could NOT help.

The calltrace is pasted below.

Thanks,
-- Dexuan

[    9.718802] scsi host2: storvsc_host_t
[    9.723854] scsi 2:0:0:0: Direct-Access     Msft     Virtual Disk     1.0  PQ: 0 ANSI: 5
[    9.753161] sd 2:0:0:0: Attached scsi generic sg1 type 0
[    9.766383] scsi host3: storvsc_host_t
[    9.771759] scsi 3:0:0:0: Direct-Access     Msft     Virtual Disk     1.0  PQ: 0 ANSI: 5
[    9.781836] hv_utils: VSS IC version 5.0
[    9.822511] sd 3:0:0:0: Attached scsi generic sg2 type 0
[    9.829039] sd 3:0:0:0: [sdb] 266338304 512-byte logical blocks: (136 GB/127 GiB)
[    9.838525] sd 3:0:0:0: [sdb] 4096-byte physical blocks
[    9.845350] sd 3:0:0:0: [sdb] Write Protect is off
[    9.851077] sd 3:0:0:0: [sdb] Mode Sense: 0f 00 00 00
[    9.859765] sd 3:0:0:0: [sdb] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
[    9.872728]  sdb: sdb1
[    9.877279] sd 3:0:0:0: [sdb] Attached SCSI disk
[    9.964093] psmouse serio1: trackpoint: failed to get extended button data
[   14.864110] psmouse serio1: trackpoint: IBM TrackPoint firmware: 0x01, buttons: 0/0
[   14.876423] input: TPPS/2 IBM TrackPoint as /devices/platform/i8042/serio1/input/input3
[   14.887216] input: AT Translated Set 2 keyboard as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A03:00/device:07/VMBUS:01/d34b2567-b9b6-42b9-8778-0a4ec0b955bf/serio2/input/input5
[   44.644061] random: crng init done
[   66.524169] hv_utils: KVP IC version 4.0
Begin: Loading essential drivers ... done.
Begin: Running /scripts/init-premount ... done.
Begin: Mounting root file system ... Begin: Running /scripts/local-top ... done.
Begin: Running /scripts/local-premount ... done.
Begin: Waiting for root file system ... Begin: Running /scripts/local-block ... done.
Begin: Running /scripts/local-block ... done.
Begin: Running /scripts/local-block ... done.
Begin: Running /scripts/local-block ... done.
[  242.652127] INFO: task systemd-udevd:183 blocked for more than 120 seconds.
[  242.661008]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  242.697270] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  242.707872] systemd-udevd   D    0   183    170 0x00000004
[  242.714654] Call Trace:
[  242.717904]  __schedule+0x27d/0x8d0
[  242.724618]  schedule+0x36/0x80
[  242.729042]  schedule_timeout+0x235/0x3f0
[  242.736667]  ? sched_clock+0x9/0x10
[  242.741144]  ? try_to_wake_up+0x4a/0x460
[  242.745580]  wait_for_completion+0xa5/0x120
[  242.751417]  ? wake_up_q+0x70/0x70
[  242.759394]  flush_work+0x11a/0x1c0
[  242.764846]  ? worker_detach_from_pool+0xb0/0xb0
[  242.770943]  __cancel_work_timer+0xf3/0x1b0
[  242.776073]  ? disk_map_sector_rcu+0x70/0x70
[  242.783857]  cancel_delayed_work_sync+0x13/0x20
[  242.789438]  disk_block_events+0x34/0x40
[  242.794641]  __blkdev_get+0x10c/0x470
[  242.799227]  blkdev_get+0x11a/0x320
[  242.803161]  ? unlock_new_inode+0x49/0x80
[  242.807451]  ? bdget+0x110/0x130
[  242.811415]  blkdev_open+0x5b/0x70
[  242.820215]  do_dentry_open+0x208/0x310
[  242.825336]  ? blkdev_get_by_dev+0x50/0x50
[  242.836362]  vfs_open+0x4c/0x70
[  242.839839]  ? may_open+0x9b/0x100
[  242.843803]  path_openat+0x297/0x13e0
[  242.848410]  ? _copy_to_user+0x2e/0x40
[  242.852966]  ? move_addr_to_user+0xa3/0xc0
[  242.857961]  do_filp_open+0x7e/0xe0
[  242.862154]  ? _cond_resched+0x1a/0x50
[  242.867045]  ? kmem_cache_alloc+0x156/0x1b0
[  242.872004]  ? getname_flags+0x56/0x1f0
[  242.881609]  ? __alloc_fd+0x46/0x170
[  242.891812]  do_sys_open+0x11b/0x1f0
[  242.896506]  SyS_open+0x1e/0x20
[  242.900365]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  242.905607] RIP: 0033:0x7efd3827aca0
[  242.909773] RSP: 002b:00007ffd7ceaeb38 EFLAGS: 00000246 ORIG_RAX: 0000000000000002
[  242.918778] RAX: ffffffffffffffda RBX: 0000555fdf1a2d40 RCX: 00007efd3827aca0
[  242.925725] RDX: 0000555fde106058 RSI: 00000000000a0800 RDI: 0000555fdf1a2ff0
[  242.929715] RBP: 0000000000000000 R08: 0000000000000073 R09: 0000000000000003
[  242.933921] R10: 00007efd37fe6520 R11: 0000000000000246 R12: 0000000000000000
[  242.942260] R13: 0000555fdf1a2d40 R14: 00007ffd7ceaea00 R15: 0000000000000000
[  242.952780] INFO: task systemd-udevd:191 blocked for more than 120 seconds.
[  242.960994]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  242.966936] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  242.975395] systemd-udevd   D    0   191    170 0x00000004
[  242.983460] Call Trace:
[  242.986694]  __schedule+0x27d/0x8d0
[  242.992603]  ? mutex_lock+0x12/0x40
[  242.996467]  ? kprobes_module_callback+0x15b/0x1d0
[  243.001934]  schedule+0x36/0x80
[  243.005908]  async_synchronize_cookie_domain+0x91/0x130
[  243.012331]  ? wake_atomic_t_function+0x60/0x60
[  243.017690]  async_synchronize_full+0x17/0x20
[  243.022662]  do_init_module+0xc1/0x1ff
[  243.026965]  load_module+0x2313/0x29a0
[  243.031232]  ? __symbol_put+0x40/0x40
[  243.035347]  ? security_kernel_post_read_file+0x6b/0x80
[  243.042541]  SYSC_finit_module+0xbc/0xf0
[  243.045130]  SyS_finit_module+0xe/0x10
[  243.049116]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  243.054517] RIP: 0033:0x7efd37fa1c19
[  243.058744] RSP: 002b:00007ffd7ceae188 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[  243.067483] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007efd37fa1c19
[  243.075801] RDX: 0000000000000000 RSI: 00007efd38497e2a RDI: 000000000000000c
[  243.084162] RBP: 00007ffd7cead190 R08: 0000000000000000 R09: 0000000000000000
[  243.093738] R10: 000000000000000c R11: 0000000000000246 R12: 0000555fdf1a3300
[  243.103158] R13: 00007ffd7cead170 R14: 0000000000000005 R15: 000000000aba9500
[  363.484110] INFO: task systemd-udevd:183 blocked for more than 120 seconds.
[  363.493200]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  363.500109] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  363.509635] systemd-udevd   D    0   183    170 0x00000004
[  363.516428] Call Trace:
[  363.519492]  __schedule+0x27d/0x8d0
[  363.523850]  schedule+0x36/0x80
[  363.529927]  schedule_timeout+0x235/0x3f0
[  363.534884]  ? sched_clock+0x9/0x10
[  363.539006]  ? try_to_wake_up+0x4a/0x460
[  363.543686]  wait_for_completion+0xa5/0x120
[  363.548831]  ? wake_up_q+0x70/0x70
[  363.552993]  flush_work+0x11a/0x1c0
[  363.557236]  ? worker_detach_from_pool+0xb0/0xb0
[  363.562851]  __cancel_work_timer+0xf3/0x1b0
[  363.567942]  ? disk_map_sector_rcu+0x70/0x70
[  363.572973]  cancel_delayed_work_sync+0x13/0x20
[  363.580271]  disk_block_events+0x34/0x40
[  363.585284]  __blkdev_get+0x10c/0x470
[  363.589774]  blkdev_get+0x11a/0x320
[  363.593851]  ? unlock_new_inode+0x49/0x80
[  363.598509]  ? bdget+0x110/0x130
[  363.602272]  blkdev_open+0x5b/0x70
[  363.606254]  do_dentry_open+0x208/0x310
[  363.610722]  ? blkdev_get_by_dev+0x50/0x50
[  363.615493]  vfs_open+0x4c/0x70
[  363.619287]  ? may_open+0x9b/0x100
[  363.623386]  path_openat+0x297/0x13e0
[  363.627752]  ? _copy_to_user+0x2e/0x40
[  363.633978]  ? move_addr_to_user+0xa3/0xc0
[  363.638833]  do_filp_open+0x7e/0xe0
[  363.642809]  ? _cond_resched+0x1a/0x50
[  363.647070]  ? kmem_cache_alloc+0x156/0x1b0
[  363.651785]  ? getname_flags+0x56/0x1f0
[  363.656285]  ? __alloc_fd+0x46/0x170
[  363.660490]  do_sys_open+0x11b/0x1f0
[  363.664655]  SyS_open+0x1e/0x20
[  363.668405]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  363.674075] RIP: 0033:0x7efd3827aca0
[  363.678582] RSP: 002b:00007ffd7ceaeb38 EFLAGS: 00000246 ORIG_RAX: 0000000000000002
[  363.689873] RAX: ffffffffffffffda RBX: 0000555fdf1a2d40 RCX: 00007efd3827aca0
[  363.698127] RDX: 0000555fde106058 RSI: 00000000000a0800 RDI: 0000555fdf1a2ff0
[  363.706494] RBP: 0000000000000000 R08: 0000000000000073 R09: 0000000000000003
[  363.714696] R10: 00007efd37fe6520 R11: 0000000000000246 R12: 0000000000000000
[  363.722922] R13: 0000555fdf1a2d40 R14: 00007ffd7ceaea00 R15: 0000000000000000
[  363.731123] INFO: task systemd-udevd:191 blocked for more than 120 seconds.
[  363.739454]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  363.745246] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  363.753890] systemd-udevd   D    0   191    170 0x00000004
[  363.760085] Call Trace:
[  363.763006]  __schedule+0x27d/0x8d0
[  363.767016]  ? mutex_lock+0x12/0x40
[  363.770943]  ? kprobes_module_callback+0x15b/0x1d0
[  363.780593]  schedule+0x36/0x80
[  363.785119]  async_synchronize_cookie_domain+0x91/0x130
[  363.793021]  ? wake_atomic_t_function+0x60/0x60
[  363.798351]  async_synchronize_full+0x17/0x20
[  363.803405]  do_init_module+0xc1/0x1ff
[  363.807564]  load_module+0x2313/0x29a0
[  363.811786]  ? __symbol_put+0x40/0x40
[  363.815895]  ? security_kernel_post_read_file+0x6b/0x80
[  363.821852]  SYSC_finit_module+0xbc/0xf0
[  363.826433]  SyS_finit_module+0xe/0x10
[  363.830732]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  363.836311] RIP: 0033:0x7efd37fa1c19
[  363.842740] RSP: 002b:00007ffd7ceae188 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[  363.852003] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007efd37fa1c19
[  363.860774] RDX: 0000000000000000 RSI: 00007efd38497e2a RDI: 000000000000000c
[  363.869669] RBP: 00007ffd7cead190 R08: 0000000000000000 R09: 0000000000000000
[  363.878494] R10: 000000000000000c R11: 0000000000000246 R12: 0000555fdf1a3300
[  363.887931] R13: 00007ffd7cead170 R14: 0000000000000005 R15: 000000000aba9500
[  484.316121] INFO: task systemd-udevd:183 blocked for more than 120 seconds.
[  484.327141]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  484.333747] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  484.342486] systemd-udevd   D    0   183    170 0x00000004
[  484.348790] Call Trace:
[  484.351712]  __schedule+0x27d/0x8d0
[  484.355904]  schedule+0x36/0x80
[  484.359718]  schedule_timeout+0x235/0x3f0
[  484.364932]  ? sched_clock+0x9/0x10
[  484.368892]  ? try_to_wake_up+0x4a/0x460
[  484.376582]  wait_for_completion+0xa5/0x120
[  484.383779]  ? wake_up_q+0x70/0x70
[  484.389885]  flush_work+0x11a/0x1c0
[  484.394048]  ? worker_detach_from_pool+0xb0/0xb0
[  484.399464]  __cancel_work_timer+0xf3/0x1b0
[  484.404326]  ? disk_map_sector_rcu+0x70/0x70
[  484.409379]  cancel_delayed_work_sync+0x13/0x20
[  484.414449]  disk_block_events+0x34/0x40
[  484.418615]  __blkdev_get+0x10c/0x470
[  484.423391]  blkdev_get+0x11a/0x320
[  484.429312]  ? unlock_new_inode+0x49/0x80
[  484.433618]  ? bdget+0x110/0x130
[  484.437300]  blkdev_open+0x5b/0x70
[  484.441376]  do_dentry_open+0x208/0x310
[  484.446108]  ? blkdev_get_by_dev+0x50/0x50
[  484.450946]  vfs_open+0x4c/0x70
[  484.454693]  ? may_open+0x9b/0x100
[  484.458682]  path_openat+0x297/0x13e0
[  484.462656]  ? _copy_to_user+0x2e/0x40
[  484.467125]  ? move_addr_to_user+0xa3/0xc0
[  484.472370]  do_filp_open+0x7e/0xe0
[  484.479195]  ? _cond_resched+0x1a/0x50
[  484.484027]  ? kmem_cache_alloc+0x156/0x1b0
[  484.488851]  ? getname_flags+0x56/0x1f0
[  484.493049]  ? __alloc_fd+0x46/0x170
[  484.496899]  do_sys_open+0x11b/0x1f0
[  484.500784]  SyS_open+0x1e/0x20
[  484.504284]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  484.510767] RIP: 0033:0x7efd3827aca0
[  484.515460] RSP: 002b:00007ffd7ceaeb38 EFLAGS: 00000246 ORIG_RAX: 0000000000000002
[  484.527246] RAX: ffffffffffffffda RBX: 0000555fdf1a2d40 RCX: 00007efd3827aca0
[  484.537307] RDX: 0000555fde106058 RSI: 00000000000a0800 RDI: 0000555fdf1a2ff0
[  484.545620] RBP: 0000000000000000 R08: 0000000000000073 R09: 0000000000000003
[  484.554372] R10: 00007efd37fe6520 R11: 0000000000000246 R12: 0000000000000000
[  484.563267] R13: 0000555fdf1a2d40 R14: 00007ffd7ceaea00 R15: 0000000000000000
[  484.571966] INFO: task systemd-udevd:191 blocked for more than 120 seconds.
[  484.580711]       Not tainted 4.10.0-rc6-next-20170203+ #2
[  484.589555] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  484.598799] systemd-udevd   D    0   191    170 0x00000004
[  484.605231] Call Trace:
[  484.608308]  __schedule+0x27d/0x8d0
[  484.612702]  ? mutex_lock+0x12/0x40
[  484.617071]  ? kprobes_module_callback+0x15b/0x1d0
[  484.621625]  schedule+0x36/0x80
[  484.623511]  async_synchronize_cookie_domain+0x91/0x130
[  484.626443]  ? wake_atomic_t_function+0x60/0x60
[  484.629059]  async_synchronize_full+0x17/0x20
[  484.635723]  do_init_module+0xc1/0x1ff
[  484.640226]  load_module+0x2313/0x29a0
[  484.645030]  ? __symbol_put+0x40/0x40
[  484.649870]  ? security_kernel_post_read_file+0x6b/0x80
[  484.658016]  SYSC_finit_module+0xbc/0xf0
[  484.662676]  SyS_finit_module+0xe/0x10
[  484.667374]  entry_SYSCALL_64_fastpath+0x1e/0xad
[  484.673822] RIP: 0033:0x7efd37fa1c19
[  484.677771] RSP: 002b:00007ffd7ceae188 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[  484.688668] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007efd37fa1c19
[  484.697959] RDX: 0000000000000000 RSI: 00007efd38497e2a RDI: 000000000000000c
[  484.707793] RBP: 00007ffd7cead190 R08: 0000000000000000 R09: 0000000000000000
[  484.716695] R10: 000000000000000c R11: 0000000000000246 R12: 0000555fdf1a3300
[  484.724415] R13: 00007ffd7cead170 R14: 0000000000000005 R15: 000000000aba9500

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

> From: linux-block-owner@vger.kernel.org [mailto:linux-block-
> owner@vger.kernel.org] On Behalf Of Dexuan Cui
> Sent: Friday, February 3, 2017 20:23
> To: Hannes Reinecke <hare@suse.com>; Bart Van Assche
> <Bart.VanAssche@sandisk.com>; hare@suse.de; axboe@kernel.dk
> Cc: hch@lst.de; linux-kernel@vger.kernel.org; linux-block@vger.kernel.org=
;
> jth@kernel.org
> Subject: RE: [PATCH] genhd: Do not hold event lock when scheduling workqu=
eue
> elements
>=20
> > From: linux-kernel-owner@vger.kernel.org [mailto:linux-kernel-
> > owner@vger.kernel.org] On Behalf Of Hannes Reinecke
> > Sent: Wednesday, February 1, 2017 00:15
> > To: Bart Van Assche <Bart.VanAssche@sandisk.com>; hare@suse.de;
> > axboe@kernel.dk
> > Cc: hch@lst.de; linux-kernel@vger.kernel.org; linux-block@vger.kernel.o=
rg;
> > jth@kernel.org
> > Subject: Re: [PATCH] genhd: Do not hold event lock when scheduling
> workqueue
> > elements
> >
> > On 01/31/2017 01:31 AM, Bart Van Assche wrote:
> > > On Wed, 2017-01-18 at 10:48 +0100, Hannes Reinecke wrote:
> > >> @@ -1488,26 +1487,13 @@ static unsigned long
> > disk_events_poll_jiffies(struct gendisk *disk)
> > >>  void disk_block_events(struct gendisk *disk)
> > >>  {
> > >>         struct disk_events *ev =3D disk->ev;
> > >> -       unsigned long flags;
> > >> -       bool cancel;
> > >>
> > >>         if (!ev)
> > >>                 return;
> > >>
> > >> -       /*
> > >> -        * Outer mutex ensures that the first blocker completes canc=
eling
> > >> -        * the event work before further blockers are allowed to fin=
ish.
> > >> -        */
> > >> -       mutex_lock(&ev->block_mutex);
> > >> -
> > >> -       spin_lock_irqsave(&ev->lock, flags);
> > >> -       cancel =3D !ev->block++;
> > >> -       spin_unlock_irqrestore(&ev->lock, flags);
> > >> -
> > >> -       if (cancel)
> > >> +       if (atomic_inc_return(&ev->block) =3D=3D 1)
> > >>                 cancel_delayed_work_sync(&disk->ev->dwork);
> > >>
> > >> -       mutex_unlock(&ev->block_mutex);
> > >>  }
> > >
> > > Hello Hannes,
> > >
> > > I have already encountered a few times a deadlock that was caused by =
the
> > > event checking code so I agree with you that it would be a big step f=
orward
> > > if such deadlocks wouldn't occur anymore. However, this patch realize=
s a
> > > change that has not been described in the patch description, namely t=
hat
> > > disk_block_events() calls are no longer serialized. Are you sure it i=
s safe
> > > to drop the serialization of disk_block_events() calls?
> > >
> > Well, this whole synchronization stuff it a bit weird; I so totally fai=
l
> > to see the rationale for it.
> > But anyway, once we've converted ev->block to atomics I _think_ the
> > mutex_lock can remain; will be checking.
> >
> > Cheers,
> >
> > Hannes
> > --
>=20
> Hi, I think I got the same calltrace with today's linux-next (next-201702=
03).
>=20
> The issue happened every time when my Linux virtual machine booted and
> Hannes's patch could NOT help.
>=20
> The calltrace is pasted below.
>=20
> -- Dexuan
=20
Any news on this thread?

The issue is still blocking Linux from booting up normally in my test. :-(

Have we identified the faulty patch?
If so, at least I can try to revert it to boot up.

Thanks,
-- Dexuan

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

> From: linux-block-owner@vger.kernel.org [mailto:linux-block-
> owner@vger.kernel.org] On Behalf Of Dexuan Cui
> Sent: Friday, February 3, 2017 20:23
> To: Hannes Reinecke <hare@suse.com>; Bart Van Assche
> <Bart.VanAssche@sandisk.com>; hare@suse.de; axboe@kernel.dk
> Cc: hch@lst.de; linux-kernel@vger.kernel.org; linux-block@vger.kernel.org;
> jth@kernel.org
> Subject: RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue
> elements
> 
> > From: linux-kernel-owner@vger.kernel.org [mailto:linux-kernel-
> > owner@vger.kernel.org] On Behalf Of Hannes Reinecke
> > Sent: Wednesday, February 1, 2017 00:15
> > To: Bart Van Assche <Bart.VanAssche@sandisk.com>; hare@suse.de;
> > axboe@kernel.dk
> > Cc: hch@lst.de; linux-kernel@vger.kernel.org; linux-block@vger.kernel.org;
> > jth@kernel.org
> > Subject: Re: [PATCH] genhd: Do not hold event lock when scheduling
> workqueue
> > elements
> >
> > On 01/31/2017 01:31 AM, Bart Van Assche wrote:
> > > On Wed, 2017-01-18 at 10:48 +0100, Hannes Reinecke wrote:
> > >> @@ -1488,26 +1487,13 @@ static unsigned long
> > disk_events_poll_jiffies(struct gendisk *disk)
> > >>  void disk_block_events(struct gendisk *disk)
> > >>  {
> > >>         struct disk_events *ev = disk->ev;
> > >> -       unsigned long flags;
> > >> -       bool cancel;
> > >>
> > >>         if (!ev)
> > >>                 return;
> > >>
> > >> -       /*
> > >> -        * Outer mutex ensures that the first blocker completes canceling
> > >> -        * the event work before further blockers are allowed to finish.
> > >> -        */
> > >> -       mutex_lock(&ev->block_mutex);
> > >> -
> > >> -       spin_lock_irqsave(&ev->lock, flags);
> > >> -       cancel = !ev->block++;
> > >> -       spin_unlock_irqrestore(&ev->lock, flags);
> > >> -
> > >> -       if (cancel)
> > >> +       if (atomic_inc_return(&ev->block) == 1)
> > >>                 cancel_delayed_work_sync(&disk->ev->dwork);
> > >>
> > >> -       mutex_unlock(&ev->block_mutex);
> > >>  }
> > >
> > > Hello Hannes,
> > >
> > > I have already encountered a few times a deadlock that was caused by the
> > > event checking code so I agree with you that it would be a big step forward
> > > if such deadlocks wouldn't occur anymore. However, this patch realizes a
> > > change that has not been described in the patch description, namely that
> > > disk_block_events() calls are no longer serialized. Are you sure it is safe
> > > to drop the serialization of disk_block_events() calls?
> > >
> > Well, this whole synchronization stuff it a bit weird; I so totally fail
> > to see the rationale for it.
> > But anyway, once we've converted ev->block to atomics I _think_ the
> > mutex_lock can remain; will be checking.
> >
> > Cheers,
> >
> > Hannes
> > --
> 
> Hi, I think I got the same calltrace with today's linux-next (next-20170203).
> 
> The issue happened every time when my Linux virtual machine booted and
> Hannes's patch could NOT help.
> 
> The calltrace is pasted below.
> 
> -- Dexuan
 
Any news on this thread?

The issue is still blocking Linux from booting up normally in my test. :-(

Have we identified the faulty patch?
If so, at least I can try to revert it to boot up.

Thanks,
-- Dexuan

Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Bart Van Assche]

T24gVHVlLCAyMDE3LTAyLTA3IGF0IDAyOjIzICswMDAwLCBEZXh1YW4gQ3VpIHdyb3RlOg0KPiBB
bnkgbmV3cyBvbiB0aGlzIHRocmVhZD8NCj4gDQo+IFRoZSBpc3N1ZSBpcyBzdGlsbCBibG9ja2lu
ZyBMaW51eCBmcm9tIGJvb3RpbmcgdXAgbm9ybWFsbHkgaW4gbXkgdGVzdC4gOi0oDQo+IA0KPiBI
YXZlIHdlIGlkZW50aWZpZWQgdGhlIGZhdWx0eSBwYXRjaD8NCj4gSWYgc28sIGF0IGxlYXN0IEkg
Y2FuIHRyeSB0byByZXZlcnQgaXQgdG8gYm9vdCB1cC4NCg0KSXQncyBpbnRlcmVzdGluZyB0aGF0
IHlvdSBoYXZlIGEgcmVwcm9kdWNpYmxlIHRlc3RjYXNlLiBJZiB5b3UgY2FuIHRlbGwgbWUgaG93
IHRvDQpyZXByb2R1Y2UgdGhpcyBJJ2xsIGhhdmUgYSBsb29rIGF0IGl0IHRvZ2V0aGVyIHdpdGgg
SGFubmVzLg0KDQpCYXJ0LgpXZXN0ZXJuIERpZ2l0YWwgQ29ycG9yYXRpb24gKGFuZCBpdHMgc3Vi
c2lkaWFyaWVzKSBFLW1haWwgQ29uZmlkZW50aWFsaXR5IE5vdGljZSAmIERpc2NsYWltZXI6CgpU
aGlzIGUtbWFpbCBhbmQgYW55IGZpbGVzIHRyYW5zbWl0dGVkIHdpdGggaXQgbWF5IGNvbnRhaW4g
Y29uZmlkZW50aWFsIG9yIGxlZ2FsbHkgcHJpdmlsZWdlZCBpbmZvcm1hdGlvbiBvZiBXREMgYW5k
L29yIGl0cyBhZmZpbGlhdGVzLCBhbmQgYXJlIGludGVuZGVkIHNvbGVseSBmb3IgdGhlIHVzZSBv
ZiB0aGUgaW5kaXZpZHVhbCBvciBlbnRpdHkgdG8gd2hpY2ggdGhleSBhcmUgYWRkcmVzc2VkLiBJ
ZiB5b3UgYXJlIG5vdCB0aGUgaW50ZW5kZWQgcmVjaXBpZW50LCBhbnkgZGlzY2xvc3VyZSwgY29w
eWluZywgZGlzdHJpYnV0aW9uIG9yIGFueSBhY3Rpb24gdGFrZW4gb3Igb21pdHRlZCB0byBiZSB0
YWtlbiBpbiByZWxpYW5jZSBvbiBpdCwgaXMgcHJvaGliaXRlZC4gSWYgeW91IGhhdmUgcmVjZWl2
ZWQgdGhpcyBlLW1haWwgaW4gZXJyb3IsIHBsZWFzZSBub3RpZnkgdGhlIHNlbmRlciBpbW1lZGlh
dGVseSBhbmQgZGVsZXRlIHRoZSBlLW1haWwgaW4gaXRzIGVudGlyZXR5IGZyb20geW91ciBzeXN0
ZW0uCg==

Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Bart Van Assche]

On Tue, 2017-02-07 at 02:23 +0000, Dexuan Cui wrote:
> Any news on this thread?
> 
> The issue is still blocking Linux from booting up normally in my test. :-(
> 
> Have we identified the faulty patch?
> If so, at least I can try to revert it to boot up.

It's interesting that you have a reproducible testcase. If you can tell me how to
reproduce this I'll have a look at it together with Hannes.

Bart.

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

[-- Attachment #1: Type: text/plain, Size: 3070 bytes --]

> From: Bart Van Assche [mailto:Bart.VanAssche@sandisk.com]
>
> On Tue, 2017-02-07 at 02:23 +0000, Dexuan Cui wrote:
> > Any news on this thread?
> >
> > The issue is still blocking Linux from booting up normally in my test. :-(
> >
> > Have we identified the faulty patch?
> > If so, at least I can try to revert it to boot up.
>
> It's interesting that you have a reproducible testcase. If you can tell me how to
> reproduce this I'll have a look at it together with Hannes.
>
> Bart.

I'm running a Ubuntu 16.04 guest on Hyper-V with the guest kernel replaced
with the linux-next kernel.

I can boot the guest with linux-next's next-20170130 without any issue,
but since next-20170131 I haven't succeeded in booting the guest.

With next-20170203 (mentioned in my mail last Friday), I got the same
calltrace as Hannes.

With today's linux-next (next-20170206), actually the calltrace changed to
the below:

(Please see the attached files for the kernel config and the full kernel log.)
(I applied Hannes's patch in this thread, but the situation remained the same.)

[  121.824158] INFO: task systemd-udevd:91 blocked for more than 60 seconds.
[  121.854885]       Not tainted 4.10.0-rc6-next-20170206+ #1
[  121.885004] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  121.927618] systemd-udevd   D12816    91     86 0x00000000
[  121.952912] Call Trace:
[  121.964366]  __schedule+0x2a9/0x900
[  121.979931]  schedule+0x36/0x80
[  121.995288]  async_synchronize_cookie_domain+0x91/0x130
[  122.023036]  ? remove_wait_queue+0x70/0x70
[  122.051383]  async_synchronize_full+0x17/0x20
[  122.076925]  do_init_module+0xc1/0x1f9
[  122.097530]  load_module+0x24bc/0x2980
[  122.118418]  ? ref_module+0x1c0/0x1c0
[  122.139060]  SYSC_finit_module+0xbc/0xf0
[  122.161566]  SyS_finit_module+0xe/0x10
[  122.185397]  entry_SYSCALL_64_fastpath+0x1e/0xb2
[  122.221880] RIP: 0033:0x7f1d69105c19
[  122.248526] RSP: 002b:00007ffe34dc3928 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[  122.283349] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007f1d69105c19
[  122.315987] RDX: 0000000000000000 RSI: 00007f1d695fbe2a RDI: 000000000000000c
[  122.354369] RBP: 00007ffe34dc2930 R08: 0000000000000000 R09: 0000000000000000
[  122.407496] R10: 000000000000000c R11: 0000000000000246 R12: 000055f0b9b910a0
[  122.443667] R13: 00007ffe34dc2910 R14: 0000000000000005 R15: 000000000aba9500
[  122.475741]
[  122.475741] Showing all locks held in the system:
[  122.503742] 2 locks held by khungtaskd/17:
[  122.524260]  #0:  (rcu_read_lock){......}, at: [<ffffffff9a10d5f1>] watchdog+0xa1/0x3d0
[  122.569110]  #1:  (tasklist_lock){......}, at: [<ffffffff9a0aaf8d>] debug_show_all_locks+0x3d/0x1a0
[  122.623903] 2 locks held by kworker/u128:1/61:
[  122.654030]  #0:  ("events_unbound"){......}, at: [<ffffffff9a079035>] process_one_work+0x175/0x540
[  122.710469]  #1:  ((&entry->work)){......}, at: [<ffffffff9a079035>] process_one_work+0x175/0x540
[  122.770659]

Thanks,
-- Dexuan

[-- Attachment #2: kernel.config.zip --]
[-- Type: application/x-zip-compressed, Size: 22441 bytes --]

[-- Attachment #3: putty.log.zip --]
[-- Type: application/x-zip-compressed, Size: 10588 bytes --]

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

PiBGcm9tOiBsaW51eC1ibG9jay1vd25lckB2Z2VyLmtlcm5lbC5vcmcgW21haWx0bzpsaW51eC1i
bG9jay0NCj4gb3duZXJAdmdlci5rZXJuZWwub3JnXSBPbiBCZWhhbGYgT2YgRGV4dWFuIEN1aQ0K
PiB3aXRoIHRoZSBsaW51eC1uZXh0IGtlcm5lbC4NCj4gDQo+IEkgY2FuIGJvb3QgdGhlIGd1ZXN0
IHdpdGggbGludXgtbmV4dCdzIG5leHQtMjAxNzAxMzAgd2l0aG91dCBhbnkgaXNzdWUsDQo+IGJ1
dCBzaW5jZSBuZXh0LTIwMTcwMTMxIEkgaGF2ZW4ndCBzdWNjZWVkZWQgaW4gYm9vdGluZyB0aGUg
Z3Vlc3QuDQo+IA0KPiBXaXRoIG5leHQtMjAxNzAyMDMgKG1lbnRpb25lZCBpbiBteSBtYWlsIGxh
c3QgRnJpZGF5KSwgSSBnb3QgdGhlIHNhbWUNCj4gY2FsbHRyYWNlIGFzIEhhbm5lcy4NCj4gDQo+
IFdpdGggdG9kYXkncyBsaW51eC1uZXh0IChuZXh0LTIwMTcwMjA2KSwgYWN0dWFsbHkgdGhlIGNh
bGx0cmFjZSBjaGFuZ2VkIHRvDQo+IHRoZSBiZWxvdy4NCj4gWyAgMTIyLjAyMzAzNl0gID8gcmVt
b3ZlX3dhaXRfcXVldWUrMHg3MC8weDcwDQo+IFsgIDEyMi4wNTEzODNdICBhc3luY19zeW5jaHJv
bml6ZV9mdWxsKzB4MTcvMHgyMA0KPiBbICAxMjIuMDc2OTI1XSAgZG9faW5pdF9tb2R1bGUrMHhj
MS8weDFmOQ0KPiBbICAxMjIuMDk3NTMwXSAgbG9hZF9tb2R1bGUrMHgyNGJjLzB4Mjk4MA0KIA0K
SSBkb24ndCBrbm93IHdoeSBpdCBoYW5ncyBoZXJlLCBidXQgdGhpcyBpcyB0aGUgc2FtZSBjYWxs
dHJhY2UgaW4gbXkNCmxhc3QtRnJpZGF5IG1haWwsIHdoaWNoIGNvbnRhaW5zIDIgY2FsbHRyYWNl
cy4gSXQgbG9va3MgdGhlIG90aGVyIGNhbGx0cmFjZSBoYXMNCmJlZW4gcmVzb2x2ZWQgYnkgc29t
ZSBjaGFuZ2VzIGJldHdlZW4gbmV4dC0yMDE3MDIwMyBhbmQgdG9kYXkuDQoNCkhlcmUgdGhlIGtl
cm5lbCBpcyB0cnlpbmcgdG8gbG9hZCB0aGUgSHlwZXItViBzdG9yYWdlIGRyaXZlciAoaHZfc3Rv
cnZzYyksIGFuZA0KdGhlIGRyaXZlcidzIF9faW5pdCBhbmQgLnByb2JlIGhhdmUgZmluaXNoZWQg
c3VjY2Vzc2Z1bGx5IGFuZCB0aGVuIHRoZSBrZXJuZWwNCmhhbmdzIGhlcmUuDQoNCkkgYmVsaWV2
ZSBzb21ldGhpbmcgaXMgYnJva2VuIHJlY2VudGx5LCBiZWNhdXNlIEkgZG9uJ3QgaGF2ZSBhbnkg
aXNzdWUgYmVmb3JlDQpKYW4gMzEuIA0KDQpUaGFua3MsDQotLSBEZXh1YW4NCg==

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

> From: linux-block-owner@vger.kernel.org [mailto:linux-block-
> owner@vger.kernel.org] On Behalf Of Dexuan Cui
> with the linux-next kernel.
> 
> I can boot the guest with linux-next's next-20170130 without any issue,
> but since next-20170131 I haven't succeeded in booting the guest.
> 
> With next-20170203 (mentioned in my mail last Friday), I got the same
> calltrace as Hannes.
> 
> With today's linux-next (next-20170206), actually the calltrace changed to
> the below.
> [  122.023036]  ? remove_wait_queue+0x70/0x70
> [  122.051383]  async_synchronize_full+0x17/0x20
> [  122.076925]  do_init_module+0xc1/0x1f9
> [  122.097530]  load_module+0x24bc/0x2980
 
I don't know why it hangs here, but this is the same calltrace in my
last-Friday mail, which contains 2 calltraces. It looks the other calltrace has
been resolved by some changes between next-20170203 and today.

Here the kernel is trying to load the Hyper-V storage driver (hv_storvsc), and
the driver's __init and .probe have finished successfully and then the kernel
hangs here.

I believe something is broken recently, because I don't have any issue before
Jan 31. 

Thanks,
-- Dexuan

Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Jens Axboe]

On 02/06/2017 11:29 PM, Dexuan Cui wrote:
>> From: linux-block-owner@vger.kernel.org [mailto:linux-block-
>> owner@vger.kernel.org] On Behalf Of Dexuan Cui
>> with the linux-next kernel.
>>
>> I can boot the guest with linux-next's next-20170130 without any issue,
>> but since next-20170131 I haven't succeeded in booting the guest.
>>
>> With next-20170203 (mentioned in my mail last Friday), I got the same
>> calltrace as Hannes.
>>
>> With today's linux-next (next-20170206), actually the calltrace changed to
>> the below.
>> [  122.023036]  ? remove_wait_queue+0x70/0x70
>> [  122.051383]  async_synchronize_full+0x17/0x20
>> [  122.076925]  do_init_module+0xc1/0x1f9
>> [  122.097530]  load_module+0x24bc/0x2980
>  
> I don't know why it hangs here, but this is the same calltrace in my
> last-Friday mail, which contains 2 calltraces. It looks the other calltrace has
> been resolved by some changes between next-20170203 and today.
> 
> Here the kernel is trying to load the Hyper-V storage driver (hv_storvsc), and
> the driver's __init and .probe have finished successfully and then the kernel
> hangs here.
> 
> I believe something is broken recently, because I don't have any issue before
> Jan 31. 

Can you try and bisect it?

-- 
Jens Axboe

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

PiBGcm9tOiBKZW5zIEF4Ym9lIFttYWlsdG86YXhib2VAa2VybmVsLmRrXQ0KPiBTZW50OiBXZWRu
ZXNkYXksIEZlYnJ1YXJ5IDgsIDIwMTcgMDA6MDkNCj4gVG86IERleHVhbiBDdWkgPGRlY3VpQG1p
Y3Jvc29mdC5jb20+OyBCYXJ0IFZhbiBBc3NjaGUNCj4gPEJhcnQuVmFuQXNzY2hlQHNhbmRpc2su
Y29tPjsgaGFyZUBzdXNlLmNvbTsgaGFyZUBzdXNlLmRlDQo+IENjOiBoY2hAbHN0LmRlOyBsaW51
eC1rZXJuZWxAdmdlci5rZXJuZWwub3JnOyBsaW51eC1ibG9ja0B2Z2VyLmtlcm5lbC5vcmc7DQo+
IGp0aEBrZXJuZWwub3JnDQo+IFN1YmplY3Q6IFJlOiBbUEFUQ0hdIGdlbmhkOiBEbyBub3QgaG9s
ZCBldmVudCBsb2NrIHdoZW4gc2NoZWR1bGluZyB3b3JrcXVldWUNCj4gZWxlbWVudHMNCj4gDQo+
IE9uIDAyLzA2LzIwMTcgMTE6MjkgUE0sIERleHVhbiBDdWkgd3JvdGU6DQo+ID4+IEZyb206IGxp
bnV4LWJsb2NrLW93bmVyQHZnZXIua2VybmVsLm9yZyBbbWFpbHRvOmxpbnV4LWJsb2NrLQ0KPiA+
PiBvd25lckB2Z2VyLmtlcm5lbC5vcmddIE9uIEJlaGFsZiBPZiBEZXh1YW4gQ3VpDQo+ID4+IHdp
dGggdGhlIGxpbnV4LW5leHQga2VybmVsLg0KPiA+Pg0KPiA+PiBJIGNhbiBib290IHRoZSBndWVz
dCB3aXRoIGxpbnV4LW5leHQncyBuZXh0LTIwMTcwMTMwIHdpdGhvdXQgYW55IGlzc3VlLA0KPiA+
PiBidXQgc2luY2UgbmV4dC0yMDE3MDEzMSBJIGhhdmVuJ3Qgc3VjY2VlZGVkIGluIGJvb3Rpbmcg
dGhlIGd1ZXN0Lg0KPiA+Pg0KPiA+PiBXaXRoIG5leHQtMjAxNzAyMDMgKG1lbnRpb25lZCBpbiBt
eSBtYWlsIGxhc3QgRnJpZGF5KSwgSSBnb3QgdGhlIHNhbWUNCj4gPj4gY2FsbHRyYWNlIGFzIEhh
bm5lcy4NCj4gPj4NCj4gPj4gV2l0aCB0b2RheSdzIGxpbnV4LW5leHQgKG5leHQtMjAxNzAyMDYp
LCBhY3R1YWxseSB0aGUgY2FsbHRyYWNlIGNoYW5nZWQgdG8NCj4gPj4gdGhlIGJlbG93Lg0KPiA+
PiBbICAxMjIuMDIzMDM2XSAgPyByZW1vdmVfd2FpdF9xdWV1ZSsweDcwLzB4NzANCj4gPj4gWyAg
MTIyLjA1MTM4M10gIGFzeW5jX3N5bmNocm9uaXplX2Z1bGwrMHgxNy8weDIwDQo+ID4+IFsgIDEy
Mi4wNzY5MjVdICBkb19pbml0X21vZHVsZSsweGMxLzB4MWY5DQo+ID4+IFsgIDEyMi4wOTc1MzBd
ICBsb2FkX21vZHVsZSsweDI0YmMvMHgyOTgwDQo+ID4NCj4gPiBJIGRvbid0IGtub3cgd2h5IGl0
IGhhbmdzIGhlcmUsIGJ1dCB0aGlzIGlzIHRoZSBzYW1lIGNhbGx0cmFjZSBpbiBteQ0KPiA+IGxh
c3QtRnJpZGF5IG1haWwsIHdoaWNoIGNvbnRhaW5zIDIgY2FsbHRyYWNlcy4gSXQgbG9va3MgdGhl
IG90aGVyIGNhbGx0cmFjZSBoYXMNCj4gPiBiZWVuIHJlc29sdmVkIGJ5IHNvbWUgY2hhbmdlcyBi
ZXR3ZWVuIG5leHQtMjAxNzAyMDMgYW5kIHRvZGF5Lg0KPiA+DQo+ID4gSGVyZSB0aGUga2VybmVs
IGlzIHRyeWluZyB0byBsb2FkIHRoZSBIeXBlci1WIHN0b3JhZ2UgZHJpdmVyIChodl9zdG9ydnNj
KSwgYW5kDQo+ID4gdGhlIGRyaXZlcidzIF9faW5pdCBhbmQgLnByb2JlIGhhdmUgZmluaXNoZWQg
c3VjY2Vzc2Z1bGx5IGFuZCB0aGVuIHRoZSBrZXJuZWwNCj4gPiBoYW5ncyBoZXJlLg0KPiA+DQo+
ID4gSSBiZWxpZXZlIHNvbWV0aGluZyBpcyBicm9rZW4gcmVjZW50bHksIGJlY2F1c2UgSSBkb24n
dCBoYXZlIGFueSBpc3N1ZSBiZWZvcmUNCj4gPiBKYW4gMzEuDQo+IA0KPiBDYW4geW91IHRyeSBh
bmQgYmlzZWN0IGl0Pw0KPiANCj4gSmVucyBBeGJvZQ0KDQpJIGJpc2VjdGVkIGl0IG9uIHRoZSBi
cmFuY2ggZm9yLTQuMTEvbmV4dCBvZiB0aGUgbGludXgtYmxvY2sgcmVwbyBhbmQgdGhlIGxvZyBz
aG93cw0KdGhlIGZpcnN0IGJhZCBjb21taXQgaXMgDQpbZTljNzg3ZTZdIHNjc2k6IGFsbG9jYXRl
IHNjc2lfY21uZCBzdHJ1Y3R1cmVzIGFzIHBhcnQgb2Ygc3RydWN0IHJlcXVlc3QNCg0KIyBnaXQg
YmlzZWN0IGxvZw0KZ2l0IGJpc2VjdCBzdGFydA0KIyBiYWQ6IFs4MGM2YjE1NzMyZjBkODgzMDAz
MjE0OWNiY2JjOGQ2N2UwNzRiNWU4XSBibGstbXEtc2NoZWQ6ICh1bilyZWdpc3RlciBlbGV2YXRv
ciB3aGVuICh1bilyZWdpc3RlcmluZyBxdWV1ZQ0KZ2l0IGJpc2VjdCBiYWQgODBjNmIxNTczMmYw
ZDg4MzAwMzIxNDljYmNiYzhkNjdlMDc0YjVlOA0KIyBnb29kOiBbMzA5YmQ5NmFmOWUyNmRhMzAz
ODY2MWJmNWNkYWQ3ODBlZWY0OWRkOV0gbWQ6IGNsZWFudXAgYmlvIG9wIC8gZmxhZ3MgaGFuZGxp
bmcgaW4gcmFpZDFfd3JpdGVfcmVxdWVzdA0KZ2l0IGJpc2VjdCBnb29kIDMwOWJkOTZhZjllMjZk
YTMwMzg2NjFiZjVjZGFkNzgwZWVmNDlkZDkNCiMgYmFkOiBbMjc0MTBhODkyN2ZiODliZDE1MGRl
MDhkNzQ5YThlZDdmNjdiNzczOV0gbmJkOiByZW1vdmUgUkVRX1RZUEVfRFJWX1BSSVYgbGVmdG92
ZXJzDQpnaXQgYmlzZWN0IGJhZCAyNzQxMGE4OTI3ZmI4OWJkMTUwZGUwOGQ3NDlhOGVkN2Y2N2I3
NzM5DQojIGJhZDogW2U5Yzc4N2U2NWMwYzM2NTI5NzQ1YmU0N2Q0OTBkOTk4YjRiNmU1ODldIHNj
c2k6IGFsbG9jYXRlIHNjc2lfY21uZCBzdHJ1Y3R1cmVzIGFzIHBhcnQgb2Ygc3RydWN0IHJlcXVl
c3QNCmdpdCBiaXNlY3QgYmFkIGU5Yzc4N2U2NWMwYzM2NTI5NzQ1YmU0N2Q0OTBkOTk4YjRiNmU1
ODkNCiMgZ29vZDogWzMyNzgyNTU3NDEzMjZiNmQ2NmQ4Y2E3ZDFjYjJjNTc2MzNlZTQzZDldIHNj
c2lfZGhfcmRhYzogc3dpdGNoIHRvIHNjc2lfZXhlY3V0ZV9yZXFfZmxhZ3MoKQ0KZ2l0IGJpc2Vj
dCBnb29kIDMyNzgyNTU3NDEzMjZiNmQ2NmQ4Y2E3ZDFjYjJjNTc2MzNlZTQzZDkNCiMgZ29vZDog
WzBmYmMzZTBmZjYyM2YxMDEyZTdjMmFmOTZlNzgxZWViMjZiY2MwZDddIHNjc2k6IHJlbW92ZSBn
ZnBfZmxhZ3MgbWVtYmVyIGluIHNjc2lfaG9zdF9jbWRfcG9vbA0KZ2l0IGJpc2VjdCBnb29kIDBm
YmMzZTBmZjYyM2YxMDEyZTdjMmFmOTZlNzgxZWViMjZiY2MwZDcNCiMgZ29vZDogW2VlZmY2OGM1
NjE4YzhkMDkyMGIxNDUzM2M3MGIyZGYwMDdiZDk0YjRdIHNjc2k6IHJlbW92ZSBzY3NpX2NtZF9k
bWFfcG9vbA0KZ2l0IGJpc2VjdCBnb29kIGVlZmY2OGM1NjE4YzhkMDkyMGIxNDUzM2M3MGIyZGYw
MDdiZDk0YjQNCiMgZ29vZDogW2Q0ODc3N2E2MzNkNmZhN2NjZGUwZjBlNjUwOWYwYzAxZmJmYzUy
OTldIHNjc2k6IHJlbW92ZSBfX3Njc2lfYWxsb2NfcXVldWUNCmdpdCBiaXNlY3QgZ29vZCBkNDg3
NzdhNjMzZDZmYTdjY2RlMGYwZTY1MDlmMGMwMWZiZmM1Mjk5DQojIGZpcnN0IGJhZCBjb21taXQ6
IFtlOWM3ODdlNjVjMGMzNjUyOTc0NWJlNDdkNDkwZDk5OGI0YjZlNTg5XSBzY3NpOiBhbGxvY2F0
ZSBzY3NpX2NtbmQgc3RydWN0dXJlcyBhcyBwYXJ0IG9mIHN0cnVjdCByZXF1ZXN0DQoNClRoYW5r
cywNCi0tIERleHVhbg0K

RE: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements

[Dexuan Cui]

> From: Jens Axboe [mailto:axboe@kernel.dk]
> Sent: Wednesday, February 8, 2017 00:09
> To: Dexuan Cui <decui@microsoft.com>; Bart Van Assche
> <Bart.VanAssche@sandisk.com>; hare@suse.com; hare@suse.de
> Cc: hch@lst.de; linux-kernel@vger.kernel.org; linux-block@vger.kernel.org;
> jth@kernel.org
> Subject: Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue
> elements
> 
> On 02/06/2017 11:29 PM, Dexuan Cui wrote:
> >> From: linux-block-owner@vger.kernel.org [mailto:linux-block-
> >> owner@vger.kernel.org] On Behalf Of Dexuan Cui
> >> with the linux-next kernel.
> >>
> >> I can boot the guest with linux-next's next-20170130 without any issue,
> >> but since next-20170131 I haven't succeeded in booting the guest.
> >>
> >> With next-20170203 (mentioned in my mail last Friday), I got the same
> >> calltrace as Hannes.
> >>
> >> With today's linux-next (next-20170206), actually the calltrace changed to
> >> the below.
> >> [  122.023036]  ? remove_wait_queue+0x70/0x70
> >> [  122.051383]  async_synchronize_full+0x17/0x20
> >> [  122.076925]  do_init_module+0xc1/0x1f9
> >> [  122.097530]  load_module+0x24bc/0x2980
> >
> > I don't know why it hangs here, but this is the same calltrace in my
> > last-Friday mail, which contains 2 calltraces. It looks the other calltrace has
> > been resolved by some changes between next-20170203 and today.
> >
> > Here the kernel is trying to load the Hyper-V storage driver (hv_storvsc), and
> > the driver's __init and .probe have finished successfully and then the kernel
> > hangs here.
> >
> > I believe something is broken recently, because I don't have any issue before
> > Jan 31.
> 
> Can you try and bisect it?
> 
> Jens Axboe

I bisected it on the branch for-4.11/next of the linux-block repo and the log shows
the first bad commit is 
[e9c787e6] scsi: allocate scsi_cmnd structures as part of struct request

# git bisect log
git bisect start
# bad: [80c6b15732f0d8830032149cbcbc8d67e074b5e8] blk-mq-sched: (un)register elevator when (un)registering queue
git bisect bad 80c6b15732f0d8830032149cbcbc8d67e074b5e8
# good: [309bd96af9e26da3038661bf5cdad780eef49dd9] md: cleanup bio op / flags handling in raid1_write_request
git bisect good 309bd96af9e26da3038661bf5cdad780eef49dd9
# bad: [27410a8927fb89bd150de08d749a8ed7f67b7739] nbd: remove REQ_TYPE_DRV_PRIV leftovers
git bisect bad 27410a8927fb89bd150de08d749a8ed7f67b7739
# bad: [e9c787e65c0c36529745be47d490d998b4b6e589] scsi: allocate scsi_cmnd structures as part of struct request
git bisect bad e9c787e65c0c36529745be47d490d998b4b6e589
# good: [3278255741326b6d66d8ca7d1cb2c57633ee43d9] scsi_dh_rdac: switch to scsi_execute_req_flags()
git bisect good 3278255741326b6d66d8ca7d1cb2c57633ee43d9
# good: [0fbc3e0ff623f1012e7c2af96e781eeb26bcc0d7] scsi: remove gfp_flags member in scsi_host_cmd_pool
git bisect good 0fbc3e0ff623f1012e7c2af96e781eeb26bcc0d7
# good: [eeff68c5618c8d0920b14533c70b2df007bd94b4] scsi: remove scsi_cmd_dma_pool
git bisect good eeff68c5618c8d0920b14533c70b2df007bd94b4
# good: [d48777a633d6fa7ccde0f0e6509f0c01fbfc5299] scsi: remove __scsi_alloc_queue
git bisect good d48777a633d6fa7ccde0f0e6509f0c01fbfc5299
# first bad commit: [e9c787e65c0c36529745be47d490d998b4b6e589] scsi: allocate scsi_cmnd structures as part of struct request

Thanks,
-- Dexuan

Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Jens Axboe]

On 02/08/2017 03:48 AM, Dexuan Cui wrote:
>> From: Jens Axboe [mailto:axboe@kernel.dk]
>> Sent: Wednesday, February 8, 2017 00:09
>> To: Dexuan Cui <decui@microsoft.com>; Bart Van Assche
>> <Bart.VanAssche@sandisk.com>; hare@suse.com; hare@suse.de
>> Cc: hch@lst.de; linux-kernel@vger.kernel.org; linux-block@vger.kernel.org;
>> jth@kernel.org
>> Subject: Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue
>> elements
>>
>> On 02/06/2017 11:29 PM, Dexuan Cui wrote:
>>>> From: linux-block-owner@vger.kernel.org [mailto:linux-block-
>>>> owner@vger.kernel.org] On Behalf Of Dexuan Cui
>>>> with the linux-next kernel.
>>>>
>>>> I can boot the guest with linux-next's next-20170130 without any issue,
>>>> but since next-20170131 I haven't succeeded in booting the guest.
>>>>
>>>> With next-20170203 (mentioned in my mail last Friday), I got the same
>>>> calltrace as Hannes.
>>>>
>>>> With today's linux-next (next-20170206), actually the calltrace changed to
>>>> the below.
>>>> [  122.023036]  ? remove_wait_queue+0x70/0x70
>>>> [  122.051383]  async_synchronize_full+0x17/0x20
>>>> [  122.076925]  do_init_module+0xc1/0x1f9
>>>> [  122.097530]  load_module+0x24bc/0x2980
>>>
>>> I don't know why it hangs here, but this is the same calltrace in my
>>> last-Friday mail, which contains 2 calltraces. It looks the other calltrace has
>>> been resolved by some changes between next-20170203 and today.
>>>
>>> Here the kernel is trying to load the Hyper-V storage driver (hv_storvsc), and
>>> the driver's __init and .probe have finished successfully and then the kernel
>>> hangs here.
>>>
>>> I believe something is broken recently, because I don't have any issue before
>>> Jan 31.
>>
>> Can you try and bisect it?
>>
>> Jens Axboe
> 
> I bisected it on the branch for-4.11/next of the linux-block repo and the log shows
> the first bad commit is 
> [e9c787e6] scsi: allocate scsi_cmnd structures as part of struct request
> 
> # git bisect log
> git bisect start
> # bad: [80c6b15732f0d8830032149cbcbc8d67e074b5e8] blk-mq-sched: (un)register elevator when (un)registering queue
> git bisect bad 80c6b15732f0d8830032149cbcbc8d67e074b5e8
> # good: [309bd96af9e26da3038661bf5cdad780eef49dd9] md: cleanup bio op / flags handling in raid1_write_request
> git bisect good 309bd96af9e26da3038661bf5cdad780eef49dd9
> # bad: [27410a8927fb89bd150de08d749a8ed7f67b7739] nbd: remove REQ_TYPE_DRV_PRIV leftovers
> git bisect bad 27410a8927fb89bd150de08d749a8ed7f67b7739
> # bad: [e9c787e65c0c36529745be47d490d998b4b6e589] scsi: allocate scsi_cmnd structures as part of struct request
> git bisect bad e9c787e65c0c36529745be47d490d998b4b6e589
> # good: [3278255741326b6d66d8ca7d1cb2c57633ee43d9] scsi_dh_rdac: switch to scsi_execute_req_flags()
> git bisect good 3278255741326b6d66d8ca7d1cb2c57633ee43d9
> # good: [0fbc3e0ff623f1012e7c2af96e781eeb26bcc0d7] scsi: remove gfp_flags member in scsi_host_cmd_pool
> git bisect good 0fbc3e0ff623f1012e7c2af96e781eeb26bcc0d7
> # good: [eeff68c5618c8d0920b14533c70b2df007bd94b4] scsi: remove scsi_cmd_dma_pool
> git bisect good eeff68c5618c8d0920b14533c70b2df007bd94b4
> # good: [d48777a633d6fa7ccde0f0e6509f0c01fbfc5299] scsi: remove __scsi_alloc_queue
> git bisect good d48777a633d6fa7ccde0f0e6509f0c01fbfc5299
> # first bad commit: [e9c787e65c0c36529745be47d490d998b4b6e589] scsi: allocate scsi_cmnd structures as part of struct request

Christoph?

I've changed the subject line, this issue has nothing to do with the
issue that Hannes was attempting to fix.

-- 
Jens Axboe

Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[hch]

On Wed, Feb 08, 2017 at 10:43:59AM -0700, Jens Axboe wrote:
> I've changed the subject line, this issue has nothing to do with the
> issue that Hannes was attempting to fix.

Nothing really useful in the thread.  Dexuan, can you throw in some
prints to see which command times out?

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

> From: hch@lst.de [mailto:hch@lst.de]
> Sent: Thursday, February 9, 2017 02:03
> To: Jens Axboe <axboe@kernel.dk>
> Cc: Dexuan Cui <decui@microsoft.com>; Bart Van Assche
> <Bart.VanAssche@sandisk.com>; hare@suse.com; hare@suse.de; Martin K.
> Petersen <martin.petersen@oracle.com>; hch@lst.de; linux-
> kernel@vger.kernel.org; linux-block@vger.kernel.org; jth@kernel.org
> Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event l=
ock
> when scheduling workqueue elements")
>=20
> On Wed, Feb 08, 2017 at 10:43:59AM -0700, Jens Axboe wrote:
> > I've changed the subject line, this issue has nothing to do with the
> > issue that Hannes was attempting to fix.
>=20
> Nothing really useful in the thread.  Dexuan, can you throw in some
> prints to see which command times out?

My colleagues have sent you the logs in another thread.

Thanks for looking into this!

-- Dexuan

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

> From: hch@lst.de [mailto:hch@lst.de]
> Sent: Thursday, February 9, 2017 02:03
> To: Jens Axboe <axboe@kernel.dk>
> Cc: Dexuan Cui <decui@microsoft.com>; Bart Van Assche
> <Bart.VanAssche@sandisk.com>; hare@suse.com; hare@suse.de; Martin K.
> Petersen <martin.petersen@oracle.com>; hch@lst.de; linux-
> kernel@vger.kernel.org; linux-block@vger.kernel.org; jth@kernel.org
> Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock
> when scheduling workqueue elements")
> 
> On Wed, Feb 08, 2017 at 10:43:59AM -0700, Jens Axboe wrote:
> > I've changed the subject line, this issue has nothing to do with the
> > issue that Hannes was attempting to fix.
> 
> Nothing really useful in the thread.  Dexuan, can you throw in some
> prints to see which command times out?

My colleagues have sent you the logs in another thread.

Thanks for looking into this!

-- Dexuan

Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[hch]

Hi Dexuan,

I've spent some time with the logs and looking over the code and
couldn't find any smoking gun.  I start to wonder if it might just
be a timing issue?

Can you try one or two things for me:

 1) run with the blk-mq I/O path for scsi by either enabling it a boot /
    module load time with the scsi_mod.use_blk_mq=Y option, or at compile
    time by enabling the CONFIG_SCSI_MQ_DEFAULT option.  If that fails
    with the commit a blk-mq run before the commit would also be useful.
 2) if possible run a VM config without the virtual CD-ROM drive -
    a lot of the scsi log chatter is about handling timeouts on the
    CD drive, so that might be able to isolate issues a bit better.

Note that I'll be offline from this afternoon European time until Sunday
night as I'm out in the mountains at a lodge without internet access,
but this issue will be my priority once back.

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

[-- Attachment #1: Type: text/plain, Size: 1687 bytes --]

> From: hch@lst.de [mailto:hch@lst.de]
>
> Hi Dexuan,
>
> I've spent some time with the logs and looking over the code and
> couldn't find any smoking gun.  I start to wonder if it might just
> be a timing issue?

Thanks a lot for digging into the issue!

I'm not sure if it's a timing issue, but the fact is that the VM hangs every time
due to the issue, when it tries to boot up.
We tried both UP VM and SMP VM, and they always failed to boot up.

> Can you try one or two things for me:

I did the following tests with
1 CPU,
IDE controller 0 with the VM's image,
IDE controller 1 with no disk,
no SCSI controller, and
no CD drive.

>  1) run with the blk-mq I/O path for scsi by either enabling it a boot /
>     module load time with the scsi_mod.use_blk_mq=Y option, or at compile
>     time by enabling the CONFIG_SCSI_MQ_DEFAULT option.  If that fails
>     with the commit a blk-mq run before the commit would also be useful.

With the scsi_mod.use_blk_mq=Y kernel parameter, the VM can boot fine!
And with =N (the default), the VM hits the issue.

>  2) if possible run a VM config without the virtual CD-ROM drive -
>     a lot of the scsi log chatter is about handling timeouts on the
>     CD drive, so that might be able to isolate issues a bit better.
Please see the attached logs.
I got the good.log by reverting the bad commit. It's so long so I only
reserve the first part of the log.

> Note that I'll be offline from this afternoon European time until Sunday
> night as I'm out in the mountains at a lodge without internet access,
> but this issue will be my priority once back.

Have a nice trip! :-)

Thanks,
-- Dexuan

[-- Attachment #2: good-bad-scsi-logs.zip --]
[-- Type: application/x-zip-compressed, Size: 80096 bytes --]

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

[-- Attachment #1: Type: text/plain, Size: 1687 bytes --]

> From: hch@lst.de [mailto:hch@lst.de]
>
> Hi Dexuan,
>
> I've spent some time with the logs and looking over the code and
> couldn't find any smoking gun.  I start to wonder if it might just
> be a timing issue?

Thanks a lot for digging into the issue!

I'm not sure if it's a timing issue, but the fact is that the VM hangs every time
due to the issue, when it tries to boot up.
We tried both UP VM and SMP VM, and they always failed to boot up.

> Can you try one or two things for me:

I did the following tests with
1 CPU,
IDE controller 0 with the VM's image,
IDE controller 1 with no disk,
no SCSI controller, and
no CD drive.

>  1) run with the blk-mq I/O path for scsi by either enabling it a boot /
>     module load time with the scsi_mod.use_blk_mq=Y option, or at compile
>     time by enabling the CONFIG_SCSI_MQ_DEFAULT option.  If that fails
>     with the commit a blk-mq run before the commit would also be useful.

With the scsi_mod.use_blk_mq=Y kernel parameter, the VM can boot fine!
And with =N (the default), the VM hits the issue.

>  2) if possible run a VM config without the virtual CD-ROM drive -
>     a lot of the scsi log chatter is about handling timeouts on the
>     CD drive, so that might be able to isolate issues a bit better.
Please see the attached logs.
I got the good.log by reverting the bad commit. It's so long so I only
reserve the first part of the log.

> Note that I'll be offline from this afternoon European time until Sunday
> night as I'm out in the mountains at a lodge without internet access,
> but this issue will be my priority once back.

Have a nice trip! :-)

Thanks,
-- Dexuan

[-- Attachment #2: good-bad-scsi-logs.zip --]
[-- Type: application/x-zip-compressed, Size: 80096 bytes --]

Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[hch]

Hi Dexuan,

can you try the hack below for now?  I disable the TUR call from
sd_check_events, which I think your VM is hanging on.  The checks
it does on the sense data look a bit fishy, but so far I've not
identified a possible root cause.

diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 40b4038c019e..1502e87c2be9 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -1457,9 +1457,13 @@ static unsigned int sd_check_events(struct gendisk *disk, unsigned int clearing)
 	retval = -ENODEV;
 
 	if (scsi_block_when_processing_errors(sdp)) {
+#if 0
 		sshdr  = kzalloc(sizeof(*sshdr), GFP_KERNEL);
 		retval = scsi_test_unit_ready(sdp, SD_TIMEOUT, SD_MAX_RETRIES,
 					      sshdr);
+#else
+		retval = 0;
+#endif
 	}
 
 	/* failed to execute TUR, assume media not present */

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

[-- Attachment #1: Type: text/plain, Size: 1107 bytes --]

> From: hch@lst.de [mailto:hch@lst.de]
> 
> Hi Dexuan,
> 
> can you try the hack below for now?  I disable the TUR call from
> sd_check_events, which I think your VM is hanging on.  The checks
> it does on the sense data look a bit fishy, but so far I've not
> identified a possible root cause.
> 
> diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
> index 40b4038c019e..1502e87c2be9 100644
> --- a/drivers/scsi/sd.c
> +++ b/drivers/scsi/sd.c
> @@ -1457,9 +1457,13 @@ static unsigned int sd_check_events(struct
> gendisk *disk, unsigned int clearing)
>  	retval = -ENODEV;
> 
>  	if (scsi_block_when_processing_errors(sdp)) {
> +#if 0
>  		sshdr  = kzalloc(sizeof(*sshdr), GFP_KERNEL);
>  		retval = scsi_test_unit_ready(sdp, SD_TIMEOUT,
> SD_MAX_RETRIES,
>  					      sshdr);
> +#else
> +		retval = 0;
> +#endif
>  	}
> 
>  	/* failed to execute TUR, assume media not present */

Unluckily, the issue is still there.
Please see the attachment for the new log.
It looks somewhere we're still sending the cmd "Test Unit Ready", which
somehow hangs.

Thanks,
-- Dexuan

[-- Attachment #2: putty.log --]
[-- Type: application/octet-stream, Size: 30844 bytes --]

[    0.000000] Linux version 4.10.0-rc5+ (root@decui-u1604) (gcc version 5.3.1 20160413 (Ubuntu 5.3.1-14ubuntu2.1) ) #19 SMP Tue Feb 14 21:59:36 CST 2017
[    0.000000] Command line: BOOT_IMAGE=/boot/hv/bzImage root=UUID=f07ac67a-0109-4738-a388-100218d2c5d2 ro ignore_loglevel scsi_mod.use_blk_mq=N scsi_mod.scsi_logging_level=0xFFFF console=ttyS0
[    0.000000] KERNEL supported cpus:
[    0.000000]   Intel GenuineIntel
[    0.000000] x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers'
[    0.000000] x86/fpu: xstate_offset[2]:  576, xstate_sizes[2]:  256
[    0.000000] x86/fpu: Enabled xstate features 0x7, context size is 832 bytes, using 'standard' format.
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009fbff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000e0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000003ffeffff] usable
[    0.000000] BIOS-e820: [mem 0x000000003fff0000-0x000000003fffefff] ACPI data
[    0.000000] BIOS-e820: [mem 0x000000003ffff000-0x000000003fffffff] ACPI NVS
[    0.000000] debug: ignoring loglevel setting.
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 2.3 present.
[    0.000000] DMI: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS 090006  04/28/2016
[    0.000000] Hypervisor detected: Microsoft HyperV
[    0.000000] HyperV: features 0x2e7f, hints 0xc2c
[    0.000000] HyperV: LAPIC Timer Frequency: 0xc3500
[    0.000000] clocksource: hyperv_clocksource: mask: 0xffffffffffffffff max_cycles: 0x24e6a1710, max_idle_ns: 440795202120 ns
[    0.000000] tsc: Marking TSC unstable due to running on Hyper-V
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x3fff0 max_arch_pfn = 0x400000000
[    0.000000] MTRR default type: uncachable
[    0.000000] MTRR fixed ranges enabled:
[    0.000000]   00000-9FFFF write-back
[    0.000000]   A0000-DFFFF uncachable
[    0.000000]   E0000-FFFFF write-back
[    0.000000] MTRR variable ranges enabled:
[    0.000000]   0 base 00000000000 mask FFF00000000 write-back
[    0.000000]   1 disabled
[    0.000000]   2 disabled
[    0.000000]   3 disabled
[    0.000000]   4 disabled
[    0.000000]   5 disabled
[    0.000000]   6 disabled
[    0.000000]   7 disabled
[    0.000000] x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WC  UC- WT  
[    0.000000] found SMP MP-table at [mem 0x000ff780-0x000ff78f] mapped at [ffff8800000ff780]
[    0.000000] Scanning 1 areas for low memory corruption
[    0.000000] Base memory trampoline at [ffff880000099000] 99000 size 24576
[    0.000000] BRK [0x0fb97000, 0x0fb97fff] PGTABLE
[    0.000000] BRK [0x0fb98000, 0x0fb98fff] PGTABLE
[    0.000000] BRK [0x0fb99000, 0x0fb99fff] PGTABLE
[    0.000000] BRK [0x0fb9a000, 0x0fb9afff] PGTABLE
[    0.000000] BRK [0x0fb9b000, 0x0fb9bfff] PGTABLE
[    0.000000] BRK [0x0fb9c000, 0x0fb9cfff] PGTABLE
[    0.000000] BRK [0x0fb9d000, 0x0fb9dfff] PGTABLE
[    0.000000] BRK [0x0fb9e000, 0x0fb9efff] PGTABLE
[    0.000000] BRK [0x0fb9f000, 0x0fb9ffff] PGTABLE
[    0.000000] BRK [0x0fba0000, 0x0fba0fff] PGTABLE
[    0.000000] BRK [0x0fba1000, 0x0fba1fff] PGTABLE
[    0.000000] BRK [0x0fba2000, 0x0fba2fff] PGTABLE
[    0.000000] RAMDISK: [mem 0x2df40000-0x32f97fff]
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000000F56F0 000014 (v00 ACPIAM)
[    0.000000] ACPI: RSDT 0x000000003FFF0000 000040 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: FACP 0x000000003FFF0200 000081 (v02 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: DSDT 0x000000003FFF1724 0033BE (v01 MSFTVM MSFTVM02 00000002 INTL 02002026)
[    0.000000] ACPI: FACS 0x000000003FFFF000 000040
[    0.000000] ACPI: WAET 0x000000003FFF1480 000028 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: SLIC 0x000000003FFF14C0 000176 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: OEM0 0x000000003FFF16C0 000064 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: SRAT 0x000000003FFF0600 0000E0 (v02 VRTUAL MICROSFT 00000001 MSFT 00000001)
[    0.000000] ACPI: APIC 0x000000003FFF0300 000252 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: OEMB 0x000000003FFFF040 000064 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] SRAT: PXM 0 -> APIC 0x00 -> Node 0
[    0.000000] No NUMA configuration found
[    0.000000] Faking a node at [mem 0x0000000000000000-0x000000003ffeffff]
[    0.000000] NODE_DATA(0) allocated [mem 0x3ffec000-0x3ffeffff]
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.000000]   DMA32    [mem 0x0000000001000000-0x000000003ffeffff]
[    0.000000]   Normal   empty
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x0000000000001000-0x000000000009efff]
[    0.000000]   node   0: [mem 0x0000000000100000-0x000000003ffeffff]
[    0.000000] Initmem setup node 0 [mem 0x0000000000001000-0x000000003ffeffff]
[    0.000000] On node 0 totalpages: 262030
[    0.000000]   DMA zone: 64 pages used for memmap
[    0.000000]   DMA zone: 21 pages reserved
[    0.000000]   DMA zone: 3998 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 4032 pages used for memmap
[    0.000000]   DMA32 zone: 258032 pages, LIFO batch:31
[    0.000000] kasan: KernelAddressSanitizer initialized
[    0.000000] ACPI: PM-Timer IO Port: 0x408
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
[    0.000000] IOAPIC[0]: apic_id 0, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] smpboot: Allowing 64 CPUs, 63 hotplug CPUs
[    0.000000] e820: [mem 0x40000000-0xffffffff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on bare hardware
[    0.000000] clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645519600211568 ns
[    0.000000] setup_percpu: NR_CPUS:256 nr_cpumask_bits:256 nr_cpu_ids:64 nr_node_ids:1
[    0.000000] percpu: Embedded 44 pages/cpu @ffff880035600000 s141000 r8192 d31032 u262144
[    0.000000] pcpu-alloc: s141000 r8192 d31032 u262144 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 00 01 02 03 04 05 06 07 [0] 08 09 10 11 12 13 14 15 
[    0.000000] pcpu-alloc: [0] 16 17 18 19 20 21 22 23 [0] 24 25 26 27 28 29 30 31 
[    0.000000] pcpu-alloc: [0] 32 33 34 35 36 37 38 39 [0] 40 41 42 43 44 45 46 47 
[    0.000000] pcpu-alloc: [0] 48 49 50 51 52 53 54 55 [0] 56 57 58 59 60 61 62 63 
[    0.000000] Built 1 zonelists in Node order, mobility grouping on.  Total pages: 257913
[    0.000000] Policy zone: DMA32
[    0.000000] Kernel command line: BOOT_IMAGE=/boot/hv/bzImage root=UUID=f07ac67a-0109-4738-a388-100218d2c5d2 ro ignore_loglevel scsi_mod.use_blk_mq=N scsi_mod.scsi_logging_level=0xFFFF console=ttyS0
[    0.000000] log_buf_len individual max cpu contribution: 4096 bytes
[    0.000000] log_buf_len total cpu_extra contributions: 258048 bytes
[    0.000000] log_buf_len min size: 262144 bytes
[    0.000000] log_buf_len: 524288 bytes
[    0.000000] early log buf free: 254188(96%)
[    0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    0.000000] Memory: 753612K/1048120K available (11344K kernel code, 4373K rwdata, 5032K rodata, 1684K init, 20112K bss, 294508K reserved, 0K cma-reserved)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=64, Nodes=1
[    0.000000] kmemleak: Kernel memory leak detector disabled
[    0.000000] Hierarchical RCU implementation.
[    0.000000] Build-time adjustment of leaf fanout to 64.
[    0.000000] RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=64.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=64
[    0.000000] NR_IRQS:16640 nr_irqs:936 16
[    0.000000] Offload RCU callbacks from all CPUs
[    0.000000] Offload RCU callbacks from CPUs: 0-63.
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [ttyS0] enabled
[    0.000000] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.000000] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.000000] ... MAX_LOCK_DEPTH:          48
[    0.000000] ... MAX_LOCKDEP_KEYS:        8191
[    0.000000] ... CLASSHASH_SIZE:          4096
[    0.000000] ... MAX_LOCKDEP_ENTRIES:     32768
[    0.000000] ... MAX_LOCKDEP_CHAINS:      65536
[    0.000000] ... CHAINHASH_SIZE:          32768
[    0.000000]  memory used by lock dependency info: 8127 kB
[    0.000000]  per task-struct memory footprint: 1920 bytes
[    0.000000] ODEBUG: selftest passed
[    0.000000] kmemleak: Early log buffer exceeded (3603), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE
[    0.000000] tsc: Fast TSC calibration failed
[    0.000000] tsc: Unable to calibrate against PIT
[    0.000000] tsc: using PMTIMER reference calibration
[    0.000000] tsc: Detected 2599.976 MHz processor
[    0.020055] Calibrating delay loop (skipped), value calculated using timer frequency.. 5199.95 BogoMIPS (lpj=10399904)
[    0.024022] pid_max: default: 65536 minimum: 512
[    0.028042] ACPI: Core revision 20160930
[    0.114692] ACPI: 1 ACPI AML tables successfully acquired and loaded
[    0.116880] Dentry cache hash table entries: 131072 (order: 8, 1048576 bytes)
[    0.124139] Inode-cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.128041] Mount-cache hash table entries: 2048 (order: 2, 16384 bytes)
[    0.131405] Mountpoint-cache hash table entries: 2048 (order: 2, 16384 bytes)
[    0.134762] CPU: Physical Processor ID: 0
[    0.136038] mce: CPU supports 1 MCE banks
[    0.138513] numa_add_cpu cpu 0 node 0: mask now 0
[    0.140026] Last level iTLB entries: 4KB 512, 2MB 8, 4MB 8
[    0.144021] Last level dTLB entries: 4KB 512, 2MB 0, 4MB 0, 1GB 4
[    0.155218] ftrace: allocating 24846 entries in 98 pages
[    0.176630] smpboot: Max logical packages: 64
[    0.180051] Switched APIC routing to physical flat.
[    0.201586] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.204119] smpboot: CPU0: Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz (family: 0x6, model: 0x3e, stepping: 0x4)
[    0.212134] Performance Events: unsupported p6 CPU model 62 no PMU driver, software events only.
[    0.228062] NMI watchdog: disabled (cpu0): hardware events not enabled
[    0.232013] NMI watchdog: Shutting down hard lockup detector on all cpus
[    0.237097] smp: Bringing up secondary CPUs ...
[    0.240012] smp: Brought up 1 node, 1 CPU
[    0.244008] smpboot: Total of 1 processors activated (5199.95 BogoMIPS)
[    0.252594] devtmpfs: initialized
[    0.265081] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
[    0.273529] pinctrl core: initialized pinctrl subsystem
[    0.280352] NET: Registered protocol family 16
[    0.294722] cpuidle: using governor ladder
[    0.296049] cpuidle: using governor menu
[    0.300034] PCCT header not found.
[    0.308069] ACPI: bus type PCI registered
[    0.318683] PCI: Using configuration type 1 for base access
[    0.373385] HugeTLB registered 1 GB page size, pre-allocated 0 pages
[    0.376043] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    0.397268] ACPI: Added _OSI(Module Device)
[    0.400032] ACPI: Added _OSI(Processor Device)
[    0.404036] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.408047] ACPI: Added _OSI(Processor Aggregator Device)
[    0.710343] ACPI: Interpreter enabled
[    0.716104] ACPI: (supports S0 S5)
[    0.720013] ACPI: Using IOAPIC for interrupt routing
[    0.725063] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[    1.167715] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    1.168109] acpi PNP0A03:00: _OSC: OS supports [Segments MSI]
[    1.172367] acpi PNP0A03:00: _OSC failed (AE_NOT_FOUND); disabling ASPM
[    1.179245] PCI host bridge to bus 0000:00
[    1.180095] pci_bus 0000:00: root bus resource [mem 0xfe0000000-0xfffffffff window]
[    1.184093] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7 window]
[    1.188098] pci_bus 0000:00: root bus resource [io  0x0d00-0xffff window]
[    1.192094] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window]
[    1.196095] pci_bus 0000:00: root bus resource [mem 0xf8000000-0xfffbffff window]
[    1.200097] pci_bus 0000:00: root bus resource [bus 00-ff]
[    1.204483] pci 0000:00:00.0: [8086:7192] type 00 class 0x060000
[    1.217123] pci 0000:00:07.0: [8086:7110] type 00 class 0x060100
[    1.231554] pci 0000:00:07.1: [8086:7111] type 00 class 0x010180
[    1.234213] pci 0000:00:07.1: reg 0x20: [io  0xffa0-0xffaf]
[    1.237776] pci 0000:00:07.1: legacy IDE quirk: reg 0x10: [io  0x01f0-0x01f7]
[    1.240053] pci 0000:00:07.1: legacy IDE quirk: reg 0x14: [io  0x03f6]
[    1.244050] pci 0000:00:07.1: legacy IDE quirk: reg 0x18: [io  0x0170-0x0177]
[    1.248050] pci 0000:00:07.1: legacy IDE quirk: reg 0x1c: [io  0x0376]
[    1.256759] pci 0000:00:07.3: [8086:7113] type 00 class 0x068000
[    1.260133] * Found PM-Timer Bug on the chipset. Due to workarounds for a bug,
[    1.260133] * this clock source is slow. Consider trying other clock sources
[    1.271103] pci 0000:00:07.3: quirk: [io  0x0400-0x043f] claimed by PIIX4 ACPI
[    1.278776] pci 0000:00:08.0: [1414:5353] type 00 class 0x030000
[    1.280837] pci 0000:00:08.0: reg 0x10: [mem 0xf8000000-0xfbffffff]
[    1.508378] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 7 9 10 *11 12 14 15)
[    1.518295] ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    1.528000] ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    1.536581] ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    1.545094] ACPI: Enabled 1 GPEs in block 00 to 0F
[    1.574583] pci 0000:00:08.0: vgaarb: setting as boot VGA device
[    1.576000] pci 0000:00:08.0: vgaarb: VGA device added: decodes=io+mem,owns=io+mem,locks=none
[    1.576054] pci 0000:00:08.0: vgaarb: bridge control possible
[    1.580026] vgaarb: loaded
[    1.590264] SCSI subsystem initialized
[    1.594244] libata version 3.00 loaded.
[    1.598341] PCI: Using ACPI for IRQ routing
[    1.600031] PCI: pci_cache_line_size set to 64 bytes
[    1.604446] e820: reserve RAM buffer [mem 0x0009fc00-0x0009ffff]
[    1.608096] e820: reserve RAM buffer [mem 0x3fff0000-0x3fffffff]
[    1.622412] clocksource: Switched to clocksource hyperv_clocksource
[    2.168473] VFS: Disk quotas dquot_6.6.0
[    2.199833] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    2.253611] pnp: PnP ACPI init
[    2.279244] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[    2.338261] pnp 00:01: Plug and Play ACPI device, IDs PNP0303 PNP030b (active)
[    2.400822] pnp 00:02: Plug and Play ACPI device, IDs PNP0f03 PNP0f13 (active)
[    2.474722] pnp 00:03: [dma 0 disabled]
[    2.504970] pnp 00:03: Plug and Play ACPI device, IDs PNP0501 (active)
[    2.564879] pnp 00:04: [dma 0 disabled]
[    2.596261] pnp 00:04: Plug and Play ACPI device, IDs PNP0501 (active)
[    2.668895] pnp 00:05: [dma 2]
[    2.695958] pnp 00:05: Plug and Play ACPI device, IDs PNP0700 (active)
[    2.751456] system 00:06: [io  0x01e0-0x01ef] has been reserved
[    2.810396] system 00:06: [io  0x0160-0x016f] has been reserved
[    2.859177] system 00:06: [io  0x0278-0x027f] has been reserved
[    2.947247] system 00:06: [io  0x0378-0x037f] has been reserved
[    2.997167] system 00:06: [io  0x0678-0x067f] has been reserved
[    3.050518] system 00:06: [io  0x0778-0x077f] has been reserved
[    3.104468] system 00:06: [io  0x04d0-0x04d1] has been reserved
[    3.156259] system 00:06: Plug and Play ACPI device, IDs PNP0c02 (active)
[    3.228073] system 00:07: [io  0x0400-0x043f] has been reserved
[    3.309146] system 00:07: [io  0x0370-0x0371] has been reserved
[    3.363529] system 00:07: [io  0x0440-0x044f] has been reserved
[    3.417407] system 00:07: [mem 0xfec00000-0xfec00fff] could not be reserved
[    3.470965] system 00:07: [mem 0xfee00000-0xfee00fff] has been reserved
[    3.535816] system 00:07: Plug and Play ACPI device, IDs PNP0c02 (active)
[    3.601401] system 00:08: [mem 0x00000000-0x0009ffff] could not be reserved
[    3.661929] system 00:08: [mem 0x000c0000-0x000dffff] could not be reserved
[    3.724843] system 00:08: [mem 0x000e0000-0x000fffff] could not be reserved
[    3.784780] system 00:08: [mem 0x00100000-0xf7ffffff] could not be reserved
[    3.836794] system 00:08: [mem 0xfffc0000-0xffffffff] has been reserved
[    3.896305] system 00:08: Plug and Play ACPI device, IDs PNP0c01 (active)
[    3.959068] pnp: PnP ACPI: found 9 devices
[    4.059377] clocksource: acpi_pm: mask: 0xffffff max_cycles: 0xffffff, max_idle_ns: 2085701024 ns
[    4.138966] pci_bus 0000:00: resource 4 [mem 0xfe0000000-0xfffffffff window]
[    4.203613] pci_bus 0000:00: resource 5 [io  0x0000-0x0cf7 window]
[    4.266754] pci_bus 0000:00: resource 6 [io  0x0d00-0xffff window]
[    4.323857] pci_bus 0000:00: resource 7 [mem 0x000a0000-0x000bffff window]
[    4.384073] pci_bus 0000:00: resource 8 [mem 0xf8000000-0xfffbffff window]
[    4.446382] NET: Registered protocol family 2
[    4.492042] TCP established hash table entries: 8192 (order: 4, 65536 bytes)
[    4.550963] TCP bind hash table entries: 8192 (order: 7, 524288 bytes)
[    4.606990] TCP: Hash tables configured (established 8192 bind 8192)
[    4.653913] UDP hash table entries: 512 (order: 4, 81920 bytes)
[    4.700511] UDP-Lite hash table entries: 512 (order: 4, 81920 bytes)
[    4.764170] NET: Registered protocol family 1
[    4.803997] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    4.851800] pci 0000:00:08.0: Video device with shadowed ROM at [mem 0x000c0000-0x000dffff]
[    4.916349] PCI: CLS 0 bytes, default 64
[    4.954665] Unpacking initramfs...
[    7.910742] debug: unmapping init [mem 0xffff88002df40000-0xffff880032f97fff]
[    7.939507] RAPL PMU: API unit is 2^-32 Joules, 3 fixed counters, 10737418240 ms ovfl timer
[    7.967877] RAPL PMU: hw unit of domain pp0-core 2^-0 Joules
[    7.989087] RAPL PMU: hw unit of domain package 2^-0 Joules
[    8.009919] RAPL PMU: hw unit of domain dram 2^-0 Joules
[    8.026505] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x257a25906f7, max_idle_ns: 440795319676 ns
[    8.060648] Scanning for low memory corruption every 60 seconds
[    8.084129] cryptomgr_test (31) used greatest stack depth: 29344 bytes left
[    8.119463] futex hash table entries: 16384 (order: 9, 2097152 bytes)
[    8.142358] audit: initializing netlink subsys (disabled)
[    8.170015] audit: type=2000 audit(1487081368.160:1): initialized
[    8.194098] Initialise system trusted keyrings
[    8.209823] workingset: timestamp_bits=40 max_order=18 bucket_order=0
[    8.393750] zbud: loaded
[    8.442056] SGI XFS with security attributes, no debug enabled
[    8.502466] cryptomgr_test (43) used greatest stack depth: 29120 bytes left
[    8.529500] cryptomgr_test (50) used greatest stack depth: 28616 bytes left
[    8.564780] Key type asymmetric registered
[    8.578682] Asymmetric key parser 'x509' registered
[    8.605328] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 251)
[    8.665619] random: fast init done
[    8.691699] io scheduler noop registered
[    8.725785] io scheduler deadline registered (default)
[    8.774396] io scheduler cfq registered
[    8.808405] io scheduler mq-deadline registered
[    8.878316] GHES: HEST is not enabled!
[    8.913808] Serial: 8250/16550 driver, 32 ports, IRQ sharing enabled
[    9.116686] 00:03: ttyS0 at I/O 0x3f8 (irq = 4, base_baud = 115200) is a 16550A
[    9.340733] 00:04: ttyS1 at I/O 0x2f8 (irq = 3, base_baud = 115200) is a 16550A
[    9.459918] Linux agpgart interface v0.103
[    9.507683] ata_piix 0000:00:07.1: version 2.13
[    9.546712] ata_piix 0000:00:07.1: Hyper-V Virtual Machine detected, ATA device ignore set
[    9.680172] scsi host0: scsi_eh_0: sleeping
[    9.715531] scsi host0: ata_piix
[    9.749703] scsi host1: scsi_eh_1: sleeping
[    9.791021] scsi host1: ata_piix
[    9.821329] ata1: PATA max UDMA/33 cmd 0x1f0 ctl 0x3f6 bmdma 0xffa0 irq 14
[    9.889581] ata2: PATA max UDMA/33 cmd 0x170 ctl 0x376 bmdma 0xffa8 irq 15
[    9.990112] scsi host0: Waking error handler thread
[   10.032269] scsi host0: scsi_eh_0: waking up 1/0/0
[   10.073002] scsi host1: Waking error handler thread
[   10.120455] scsi host1: scsi_eh_1: waking up 1/0/0
[   10.172770] i8042: PNP: PS/2 Controller [PNP0303:PS2K,PNP0f03:PS2M] at 0x60,0x64 irq 1,12
[   10.252613] serio: i8042 KBD port at 0x60,0x64 irq 1
[   10.293633] serio: i8042 AUX port at 0x60,0x64 irq 12
[   10.346090] scsi host1: waking up host to restart
[   10.386705] scsi host1: scsi_eh_1: sleeping
[   10.681782] rtc_cmos 00:00: RTC can wake from S4
[   10.745372] rtc_cmos 00:00: rtc core: registered rtc_cmos as rtc0
[   10.797256] rtc_cmos 00:00: alarms up to one month, 114 bytes nvram
[   10.851660] ata1.01: NODEV after polling detection
[   11.003810] ledtrig-cpu: registered to indicate activity on CPUs
[   11.073799] NET: Registered protocol family 10
[   11.140588] ata1.00: host indicates ignore ATA devices, ignored
[   11.190937] scsi host0: waking up host to restart
[   11.251567] scsi host0: scsi_eh_0: sleeping
[   11.295553] Segment Routing with IPv6
[   11.332968] NET: Registered protocol family 17
[   11.365656] Key type dns_resolver registered
[   11.404429] registered taskstats version 1
[   11.439716] Loading compiled-in X.509 certificates
[   11.487244] zswap: loaded using pool lzo/zbud
[   11.600878] rtc_cmos 00:00: setting system clock to 2017-02-14 14:09:35 UTC (1487081375)
[   11.704000] BIOS EDD facility v0.16 2004-Jun-25, 0 devices found
[   11.754153] EDD information not available.
[   11.813499] debug: unmapping init [mem 0xffffffffa0647000-0xffffffffa07ebfff]
[   11.868404] Write protecting the kernel read-only data: 18432k
[   11.912489] debug: unmapping init [mem 0xffff88000db18000-0xffff88000dbfffff]
[   12.008150] debug: unmapping init [mem 0xffff88000e0ea000-0xffff88000e1fffff]
[   12.105000] mount (68) used greatest stack depth: 28480 bytes left
Loading, please wait...
[   12.268549] all_generic_ide (78) used greatest stack depth: 27944 bytes left
starting version 229
[   13.184271] udevadm (85) used greatest stack depth: 27936 bytes left
[   13.314046] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input0
[   13.342220] clocksource: hyperv_clocksource_tsc_page: mask: 0xffffffffffffffff max_cycles: 0x24e6a1710, max_idle_ns: 440795202120 ns
[   13.533454] clocksource: Switched to clocksource hyperv_clocksource_tsc_page
[   13.559246] hv_vmbus: Hyper-V Host Build:14393-10.0-0-0.576; Vmbus version:4.0
[   13.677005] hv_vmbus: registering driver hyperv_fb
[   13.764010] hv_utils: Registering HyperV Utility Driver
[   13.805837] hv_vmbus: registering driver hv_util
[   13.842200] psmouse serio1: trackpoint: failed to get extended button data
[   13.903246] hv_vmbus: registering driver hyperv_keyboard
[   13.956938] hidraw: raw HID events driver (C) Jiri Kosina
[   14.006668] hv_vmbus: registering driver hv_storvsc
[   14.052442] hyperv_fb: Screen resolution: 1152x864, Color depth: 32
[   14.112021] hv_vmbus: registering driver hid_hyperv
[   14.190750] Console: switching to colour frame buffer device 144x54
[   14.281549] scsi host2: scsi_eh_2: sleeping
[   14.363114] scsi host2: storvsc_host_t
[   14.413729] hv_vmbus: registering driver hv_netvsc
[   14.500991] scsi 2:0:0:0: scsi scan: INQUIRY pass 1 length 36
[   14.552852] scsi 2:0:0:0: tag#0 Send: scmd 0xffff880019ff9448
[   14.612510] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 00 00 00 24 00
[   14.660636] scsi 2:0:0:0: tag#0 Done: SUCCESS Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   14.735249] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 00 00 00 24 00
[   14.739191] scsi 2:0:0:0: tag#0 scsi host busy 1 failed 0
[   14.739191] scsi 2:0:0:0: Notifying upper driver of completion (result 0)
[   14.739191] scsi 2:0:0:0: scsi scan: INQUIRY successful with code 0x0
[   14.964062] scsi 2:0:0:0: Direct-Access     Msft     Virtual Disk     1.0  PQ: 0 ANSI: 5
[   15.038762] scsi 2:0:0:0: tag#0 Send: scmd 0xffff880019ffd5e8
[   15.089616] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 01 00 00 ff 00
[   15.137673] scsi 2:0:0:0: tag#0 Done: SUCCESS Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   15.207744] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 01 00 00 ff 00
[   15.255839] scsi 2:0:0:0: tag#0 scsi host busy 1 failed 0
[   15.304291] scsi 2:0:0:0: Notifying upper driver of completion (result 0)
[   15.362157] scsi 2:0:0:0: tag#0 Send: scmd 0xffff880019ffa708
[   15.413281] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 01 83 00 ff 00
[   15.460944] scsi 2:0:0:0: tag#0 Done: SUCCESS Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   15.518901] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 01 83 00 ff 00
[   15.570146] scsi 2:0:0:0: tag#0 scsi host busy 1 failed 0
[   15.618008] scsi 2:0:0:0: Notifying upper driver of completion (result 0)
[   15.678847] sd 2:0:0:0: tag#0 Send: scmd 0xffff880019ffb068
[   15.723329] sd 2:0:0:0: tag#0 CDB: Test Unit Ready 00 00 00 00 00 00
[   15.775429] sd 2:0:0:0: sg_alloc: dev=0 
[   15.809186] sd 2:0:0:0: Attached scsi generic sg0 type 0
[   15.884302] input: Microsoft Vmbus HID-compliant Mouse as /devices/0006:045E:0621.0001/input/input3
[   15.976010] hid-generic 0006:045E:0621.0001: input: <UNKNOWN> HID v0.01 Mouse [Microsoft Vmbus HID-compliant Mouse] on 
[   16.198080] hv_utils: Using TimeSync version 4.0
[   18.949416] psmouse serio1: trackpoint: IBM TrackPoint firmware: 0x01, buttons: 0/0
[   19.013187] input: TPPS/2 IBM TrackPoint as /devices/platform/i8042/serio1/input/input2
[   19.088795] input: AT Translated Set 2 keyboard as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A03:00/device:07/VMBUS:01/d34b2567-b9b6-42b9-8778-0a4ec0b955bf/serio2/input/input4
[   46.048170] sd 2:0:0:0: tag#0 Done: TIMEOUT_ERROR Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   46.052054] sd 2:0:0:0: tag#0 CDB: Test Unit Ready 00 00 00 00 00 00
[   46.052054] sd 2:0:0:0: tag#0 scsi host busy 1 failed 0
[   78.816191] sd 2:0:0:0: tag#0 Done: TIMEOUT_ERROR Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   78.820056] sd 2:0:0:0: tag#0 CDB: Test Unit Ready 00 00 00 00 00 00
[   78.820056] sd 2:0:0:0: tag#0 scsi host busy 1 failed 0
[  111.584194] sd 2:0:0:0: tag#0 Done: TIMEOUT_ERROR Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[  111.588064] sd 2:0:0:0: tag#0 CDB: Test Unit Ready 00 00 00 00 00 00
[  111.588064] sd 2:0:0:0: tag#0 scsi host busy 1 failed 0
[  121.824225] INFO: task systemd-udevd:104 blocked for more than 60 seconds.
[  121.874720]       Not tainted 4.10.0-rc5+ #19
[  121.914743] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  121.990829] systemd-udevd   D25232   104     84 0x00000000
[  122.036193] Call Trace:
[  122.055827]  __schedule+0x48c/0xc90
[  122.086341]  ? do_raw_spin_lock+0x112/0x1a0
[  122.121098]  schedule+0x4d/0xd0
[  122.146919]  async_synchronize_cookie_domain+0x133/0x1e0
[  122.277712]  ? async_unregister_domain+0x70/0x70
[  122.326912]  ? remove_wait_queue+0xc0/0xc0
[  122.360589]  ? up_read+0x1f/0x40
[  122.391119]  ? __blocking_notifier_call_chain+0x6c/0x80
[  122.440005]  async_synchronize_full+0x17/0x20
[  122.478899]  do_init_module+0x174/0x311
[  122.517420]  load_module+0x4484/0x4c00
[  122.552531]  ? m_show+0x2b0/0x2b0
[  122.580499]  ? fsnotify+0x769/0x7e0
[  122.623893]  ? module_frob_arch_sections+0x20/0x20
[  122.671746]  ? __fsnotify_parent+0x2c/0x130
[  122.711308]  ? vfs_read+0x14b/0x1b0
[  122.742297]  ? kernel_read_file+0x12a/0x350
[  122.778480]  ? set_binfmt+0x90/0x90
[  122.826074]  ? kernel_read_file_from_fd+0x49/0x80
[  122.867793]  SYSC_finit_module+0x169/0x1a0
[  122.904224]  ? SYSC_init_module+0x1d0/0x1d0
[  122.947117]  ? vma_is_stack_for_current+0x60/0x60
[  122.988156]  ? __fget+0x10b/0x150
[  123.013801]  ? lockdep_sys_exit+0x1a/0xa0
[  123.050099]  ? lockdep_sys_exit_thunk+0x16/0x30
[  123.087355]  SyS_finit_module+0xe/0x10
[  123.115902]  entry_SYSCALL_64_fastpath+0x1e/0xb2
[  123.151843] RIP: 0033:0x7f6060499c19
[  123.179364] RSP: 002b:00007ffec634f808 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[  123.241863] RAX: ffffffffffffffda RBX: 000055f86ee6cbd0 RCX: 00007f6060499c19
[  123.301211] RDX: 0000000000000000 RSI: 00007f606098fe2a RDI: 000000000000000c
[  123.364226] RBP: 00007f606098fe2a R08: 0000000000000000 R09: 0000000000000000
[  123.427614] R10: 000000000000000c R11: 0000000000000246 R12: 0000000000000000
[  123.492838] R13: 000055f86ee70f30 R14: 0000000000020000 R15: 000000000aba9500
[  123.556355] 
[  123.556355] Showing all locks held in the system:
[  123.610348] 2 locks held by khungtaskd/17:
[  123.645766]  #0:  (rcu_read_lock){......}, at: [<ffffffff9f1abc56>] watchdog+0xc6/0x530
[  123.715830]  #1:  (tasklist_lock){......}, at: [<ffffffff9f111a89>] debug_show_all_locks+0x49/0x1e0
[  123.806486] 2 locks held by kworker/u128:1/61:
[  123.841210]  #0:  ("events_unbound"){......}, at: [<ffffffff9f0bf3c2>] process_one_work+0x3d2/0xa60
[  123.909420]  #1:  ((&entry->work)){......}, at: [<ffffffff9f0bf3c2>] process_one_work+0x3d2/0xa60
[  123.986273] 
[  123.999135] =============================================
[  123.999135] 
Begin: Loading essential drivers ... done.
Begin: Running /scripts/init-premount ... done.
Begin: Mounting root file system ... Begin: Running /scripts/local-top ... done.
Begin: Running /scripts/local-premount ... [  134.463258] random: crng init done

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

[-- Attachment #1: Type: text/plain, Size: 1107 bytes --]

> From: hch@lst.de [mailto:hch@lst.de]
> 
> Hi Dexuan,
> 
> can you try the hack below for now?  I disable the TUR call from
> sd_check_events, which I think your VM is hanging on.  The checks
> it does on the sense data look a bit fishy, but so far I've not
> identified a possible root cause.
> 
> diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
> index 40b4038c019e..1502e87c2be9 100644
> --- a/drivers/scsi/sd.c
> +++ b/drivers/scsi/sd.c
> @@ -1457,9 +1457,13 @@ static unsigned int sd_check_events(struct
> gendisk *disk, unsigned int clearing)
>  	retval = -ENODEV;
> 
>  	if (scsi_block_when_processing_errors(sdp)) {
> +#if 0
>  		sshdr  = kzalloc(sizeof(*sshdr), GFP_KERNEL);
>  		retval = scsi_test_unit_ready(sdp, SD_TIMEOUT,
> SD_MAX_RETRIES,
>  					      sshdr);
> +#else
> +		retval = 0;
> +#endif
>  	}
> 
>  	/* failed to execute TUR, assume media not present */

Unluckily, the issue is still there.
Please see the attachment for the new log.
It looks somewhere we're still sending the cmd "Test Unit Ready", which
somehow hangs.

Thanks,
-- Dexuan

[-- Attachment #2: putty.log --]
[-- Type: application/octet-stream, Size: 30844 bytes --]

[    0.000000] Linux version 4.10.0-rc5+ (root@decui-u1604) (gcc version 5.3.1 20160413 (Ubuntu 5.3.1-14ubuntu2.1) ) #19 SMP Tue Feb 14 21:59:36 CST 2017
[    0.000000] Command line: BOOT_IMAGE=/boot/hv/bzImage root=UUID=f07ac67a-0109-4738-a388-100218d2c5d2 ro ignore_loglevel scsi_mod.use_blk_mq=N scsi_mod.scsi_logging_level=0xFFFF console=ttyS0
[    0.000000] KERNEL supported cpus:
[    0.000000]   Intel GenuineIntel
[    0.000000] x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers'
[    0.000000] x86/fpu: xstate_offset[2]:  576, xstate_sizes[2]:  256
[    0.000000] x86/fpu: Enabled xstate features 0x7, context size is 832 bytes, using 'standard' format.
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009fbff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000e0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000003ffeffff] usable
[    0.000000] BIOS-e820: [mem 0x000000003fff0000-0x000000003fffefff] ACPI data
[    0.000000] BIOS-e820: [mem 0x000000003ffff000-0x000000003fffffff] ACPI NVS
[    0.000000] debug: ignoring loglevel setting.
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 2.3 present.
[    0.000000] DMI: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS 090006  04/28/2016
[    0.000000] Hypervisor detected: Microsoft HyperV
[    0.000000] HyperV: features 0x2e7f, hints 0xc2c
[    0.000000] HyperV: LAPIC Timer Frequency: 0xc3500
[    0.000000] clocksource: hyperv_clocksource: mask: 0xffffffffffffffff max_cycles: 0x24e6a1710, max_idle_ns: 440795202120 ns
[    0.000000] tsc: Marking TSC unstable due to running on Hyper-V
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x3fff0 max_arch_pfn = 0x400000000
[    0.000000] MTRR default type: uncachable
[    0.000000] MTRR fixed ranges enabled:
[    0.000000]   00000-9FFFF write-back
[    0.000000]   A0000-DFFFF uncachable
[    0.000000]   E0000-FFFFF write-back
[    0.000000] MTRR variable ranges enabled:
[    0.000000]   0 base 00000000000 mask FFF00000000 write-back
[    0.000000]   1 disabled
[    0.000000]   2 disabled
[    0.000000]   3 disabled
[    0.000000]   4 disabled
[    0.000000]   5 disabled
[    0.000000]   6 disabled
[    0.000000]   7 disabled
[    0.000000] x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WC  UC- WT  
[    0.000000] found SMP MP-table at [mem 0x000ff780-0x000ff78f] mapped at [ffff8800000ff780]
[    0.000000] Scanning 1 areas for low memory corruption
[    0.000000] Base memory trampoline at [ffff880000099000] 99000 size 24576
[    0.000000] BRK [0x0fb97000, 0x0fb97fff] PGTABLE
[    0.000000] BRK [0x0fb98000, 0x0fb98fff] PGTABLE
[    0.000000] BRK [0x0fb99000, 0x0fb99fff] PGTABLE
[    0.000000] BRK [0x0fb9a000, 0x0fb9afff] PGTABLE
[    0.000000] BRK [0x0fb9b000, 0x0fb9bfff] PGTABLE
[    0.000000] BRK [0x0fb9c000, 0x0fb9cfff] PGTABLE
[    0.000000] BRK [0x0fb9d000, 0x0fb9dfff] PGTABLE
[    0.000000] BRK [0x0fb9e000, 0x0fb9efff] PGTABLE
[    0.000000] BRK [0x0fb9f000, 0x0fb9ffff] PGTABLE
[    0.000000] BRK [0x0fba0000, 0x0fba0fff] PGTABLE
[    0.000000] BRK [0x0fba1000, 0x0fba1fff] PGTABLE
[    0.000000] BRK [0x0fba2000, 0x0fba2fff] PGTABLE
[    0.000000] RAMDISK: [mem 0x2df40000-0x32f97fff]
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000000F56F0 000014 (v00 ACPIAM)
[    0.000000] ACPI: RSDT 0x000000003FFF0000 000040 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: FACP 0x000000003FFF0200 000081 (v02 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: DSDT 0x000000003FFF1724 0033BE (v01 MSFTVM MSFTVM02 00000002 INTL 02002026)
[    0.000000] ACPI: FACS 0x000000003FFFF000 000040
[    0.000000] ACPI: WAET 0x000000003FFF1480 000028 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: SLIC 0x000000003FFF14C0 000176 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: OEM0 0x000000003FFF16C0 000064 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: SRAT 0x000000003FFF0600 0000E0 (v02 VRTUAL MICROSFT 00000001 MSFT 00000001)
[    0.000000] ACPI: APIC 0x000000003FFF0300 000252 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: OEMB 0x000000003FFFF040 000064 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] SRAT: PXM 0 -> APIC 0x00 -> Node 0
[    0.000000] No NUMA configuration found
[    0.000000] Faking a node at [mem 0x0000000000000000-0x000000003ffeffff]
[    0.000000] NODE_DATA(0) allocated [mem 0x3ffec000-0x3ffeffff]
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.000000]   DMA32    [mem 0x0000000001000000-0x000000003ffeffff]
[    0.000000]   Normal   empty
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x0000000000001000-0x000000000009efff]
[    0.000000]   node   0: [mem 0x0000000000100000-0x000000003ffeffff]
[    0.000000] Initmem setup node 0 [mem 0x0000000000001000-0x000000003ffeffff]
[    0.000000] On node 0 totalpages: 262030
[    0.000000]   DMA zone: 64 pages used for memmap
[    0.000000]   DMA zone: 21 pages reserved
[    0.000000]   DMA zone: 3998 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 4032 pages used for memmap
[    0.000000]   DMA32 zone: 258032 pages, LIFO batch:31
[    0.000000] kasan: KernelAddressSanitizer initialized
[    0.000000] ACPI: PM-Timer IO Port: 0x408
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
[    0.000000] IOAPIC[0]: apic_id 0, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] smpboot: Allowing 64 CPUs, 63 hotplug CPUs
[    0.000000] e820: [mem 0x40000000-0xffffffff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on bare hardware
[    0.000000] clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645519600211568 ns
[    0.000000] setup_percpu: NR_CPUS:256 nr_cpumask_bits:256 nr_cpu_ids:64 nr_node_ids:1
[    0.000000] percpu: Embedded 44 pages/cpu @ffff880035600000 s141000 r8192 d31032 u262144
[    0.000000] pcpu-alloc: s141000 r8192 d31032 u262144 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 00 01 02 03 04 05 06 07 [0] 08 09 10 11 12 13 14 15 
[    0.000000] pcpu-alloc: [0] 16 17 18 19 20 21 22 23 [0] 24 25 26 27 28 29 30 31 
[    0.000000] pcpu-alloc: [0] 32 33 34 35 36 37 38 39 [0] 40 41 42 43 44 45 46 47 
[    0.000000] pcpu-alloc: [0] 48 49 50 51 52 53 54 55 [0] 56 57 58 59 60 61 62 63 
[    0.000000] Built 1 zonelists in Node order, mobility grouping on.  Total pages: 257913
[    0.000000] Policy zone: DMA32
[    0.000000] Kernel command line: BOOT_IMAGE=/boot/hv/bzImage root=UUID=f07ac67a-0109-4738-a388-100218d2c5d2 ro ignore_loglevel scsi_mod.use_blk_mq=N scsi_mod.scsi_logging_level=0xFFFF console=ttyS0
[    0.000000] log_buf_len individual max cpu contribution: 4096 bytes
[    0.000000] log_buf_len total cpu_extra contributions: 258048 bytes
[    0.000000] log_buf_len min size: 262144 bytes
[    0.000000] log_buf_len: 524288 bytes
[    0.000000] early log buf free: 254188(96%)
[    0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    0.000000] Memory: 753612K/1048120K available (11344K kernel code, 4373K rwdata, 5032K rodata, 1684K init, 20112K bss, 294508K reserved, 0K cma-reserved)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=64, Nodes=1
[    0.000000] kmemleak: Kernel memory leak detector disabled
[    0.000000] Hierarchical RCU implementation.
[    0.000000] Build-time adjustment of leaf fanout to 64.
[    0.000000] RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=64.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=64
[    0.000000] NR_IRQS:16640 nr_irqs:936 16
[    0.000000] Offload RCU callbacks from all CPUs
[    0.000000] Offload RCU callbacks from CPUs: 0-63.
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [ttyS0] enabled
[    0.000000] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.000000] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.000000] ... MAX_LOCK_DEPTH:          48
[    0.000000] ... MAX_LOCKDEP_KEYS:        8191
[    0.000000] ... CLASSHASH_SIZE:          4096
[    0.000000] ... MAX_LOCKDEP_ENTRIES:     32768
[    0.000000] ... MAX_LOCKDEP_CHAINS:      65536
[    0.000000] ... CHAINHASH_SIZE:          32768
[    0.000000]  memory used by lock dependency info: 8127 kB
[    0.000000]  per task-struct memory footprint: 1920 bytes
[    0.000000] ODEBUG: selftest passed
[    0.000000] kmemleak: Early log buffer exceeded (3603), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE
[    0.000000] tsc: Fast TSC calibration failed
[    0.000000] tsc: Unable to calibrate against PIT
[    0.000000] tsc: using PMTIMER reference calibration
[    0.000000] tsc: Detected 2599.976 MHz processor
[    0.020055] Calibrating delay loop (skipped), value calculated using timer frequency.. 5199.95 BogoMIPS (lpj=10399904)
[    0.024022] pid_max: default: 65536 minimum: 512
[    0.028042] ACPI: Core revision 20160930
[    0.114692] ACPI: 1 ACPI AML tables successfully acquired and loaded
[    0.116880] Dentry cache hash table entries: 131072 (order: 8, 1048576 bytes)
[    0.124139] Inode-cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.128041] Mount-cache hash table entries: 2048 (order: 2, 16384 bytes)
[    0.131405] Mountpoint-cache hash table entries: 2048 (order: 2, 16384 bytes)
[    0.134762] CPU: Physical Processor ID: 0
[    0.136038] mce: CPU supports 1 MCE banks
[    0.138513] numa_add_cpu cpu 0 node 0: mask now 0
[    0.140026] Last level iTLB entries: 4KB 512, 2MB 8, 4MB 8
[    0.144021] Last level dTLB entries: 4KB 512, 2MB 0, 4MB 0, 1GB 4
[    0.155218] ftrace: allocating 24846 entries in 98 pages
[    0.176630] smpboot: Max logical packages: 64
[    0.180051] Switched APIC routing to physical flat.
[    0.201586] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.204119] smpboot: CPU0: Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz (family: 0x6, model: 0x3e, stepping: 0x4)
[    0.212134] Performance Events: unsupported p6 CPU model 62 no PMU driver, software events only.
[    0.228062] NMI watchdog: disabled (cpu0): hardware events not enabled
[    0.232013] NMI watchdog: Shutting down hard lockup detector on all cpus
[    0.237097] smp: Bringing up secondary CPUs ...
[    0.240012] smp: Brought up 1 node, 1 CPU
[    0.244008] smpboot: Total of 1 processors activated (5199.95 BogoMIPS)
[    0.252594] devtmpfs: initialized
[    0.265081] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
[    0.273529] pinctrl core: initialized pinctrl subsystem
[    0.280352] NET: Registered protocol family 16
[    0.294722] cpuidle: using governor ladder
[    0.296049] cpuidle: using governor menu
[    0.300034] PCCT header not found.
[    0.308069] ACPI: bus type PCI registered
[    0.318683] PCI: Using configuration type 1 for base access
[    0.373385] HugeTLB registered 1 GB page size, pre-allocated 0 pages
[    0.376043] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    0.397268] ACPI: Added _OSI(Module Device)
[    0.400032] ACPI: Added _OSI(Processor Device)
[    0.404036] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.408047] ACPI: Added _OSI(Processor Aggregator Device)
[    0.710343] ACPI: Interpreter enabled
[    0.716104] ACPI: (supports S0 S5)
[    0.720013] ACPI: Using IOAPIC for interrupt routing
[    0.725063] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[    1.167715] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    1.168109] acpi PNP0A03:00: _OSC: OS supports [Segments MSI]
[    1.172367] acpi PNP0A03:00: _OSC failed (AE_NOT_FOUND); disabling ASPM
[    1.179245] PCI host bridge to bus 0000:00
[    1.180095] pci_bus 0000:00: root bus resource [mem 0xfe0000000-0xfffffffff window]
[    1.184093] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7 window]
[    1.188098] pci_bus 0000:00: root bus resource [io  0x0d00-0xffff window]
[    1.192094] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window]
[    1.196095] pci_bus 0000:00: root bus resource [mem 0xf8000000-0xfffbffff window]
[    1.200097] pci_bus 0000:00: root bus resource [bus 00-ff]
[    1.204483] pci 0000:00:00.0: [8086:7192] type 00 class 0x060000
[    1.217123] pci 0000:00:07.0: [8086:7110] type 00 class 0x060100
[    1.231554] pci 0000:00:07.1: [8086:7111] type 00 class 0x010180
[    1.234213] pci 0000:00:07.1: reg 0x20: [io  0xffa0-0xffaf]
[    1.237776] pci 0000:00:07.1: legacy IDE quirk: reg 0x10: [io  0x01f0-0x01f7]
[    1.240053] pci 0000:00:07.1: legacy IDE quirk: reg 0x14: [io  0x03f6]
[    1.244050] pci 0000:00:07.1: legacy IDE quirk: reg 0x18: [io  0x0170-0x0177]
[    1.248050] pci 0000:00:07.1: legacy IDE quirk: reg 0x1c: [io  0x0376]
[    1.256759] pci 0000:00:07.3: [8086:7113] type 00 class 0x068000
[    1.260133] * Found PM-Timer Bug on the chipset. Due to workarounds for a bug,
[    1.260133] * this clock source is slow. Consider trying other clock sources
[    1.271103] pci 0000:00:07.3: quirk: [io  0x0400-0x043f] claimed by PIIX4 ACPI
[    1.278776] pci 0000:00:08.0: [1414:5353] type 00 class 0x030000
[    1.280837] pci 0000:00:08.0: reg 0x10: [mem 0xf8000000-0xfbffffff]
[    1.508378] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 7 9 10 *11 12 14 15)
[    1.518295] ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    1.528000] ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    1.536581] ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    1.545094] ACPI: Enabled 1 GPEs in block 00 to 0F
[    1.574583] pci 0000:00:08.0: vgaarb: setting as boot VGA device
[    1.576000] pci 0000:00:08.0: vgaarb: VGA device added: decodes=io+mem,owns=io+mem,locks=none
[    1.576054] pci 0000:00:08.0: vgaarb: bridge control possible
[    1.580026] vgaarb: loaded
[    1.590264] SCSI subsystem initialized
[    1.594244] libata version 3.00 loaded.
[    1.598341] PCI: Using ACPI for IRQ routing
[    1.600031] PCI: pci_cache_line_size set to 64 bytes
[    1.604446] e820: reserve RAM buffer [mem 0x0009fc00-0x0009ffff]
[    1.608096] e820: reserve RAM buffer [mem 0x3fff0000-0x3fffffff]
[    1.622412] clocksource: Switched to clocksource hyperv_clocksource
[    2.168473] VFS: Disk quotas dquot_6.6.0
[    2.199833] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    2.253611] pnp: PnP ACPI init
[    2.279244] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[    2.338261] pnp 00:01: Plug and Play ACPI device, IDs PNP0303 PNP030b (active)
[    2.400822] pnp 00:02: Plug and Play ACPI device, IDs PNP0f03 PNP0f13 (active)
[    2.474722] pnp 00:03: [dma 0 disabled]
[    2.504970] pnp 00:03: Plug and Play ACPI device, IDs PNP0501 (active)
[    2.564879] pnp 00:04: [dma 0 disabled]
[    2.596261] pnp 00:04: Plug and Play ACPI device, IDs PNP0501 (active)
[    2.668895] pnp 00:05: [dma 2]
[    2.695958] pnp 00:05: Plug and Play ACPI device, IDs PNP0700 (active)
[    2.751456] system 00:06: [io  0x01e0-0x01ef] has been reserved
[    2.810396] system 00:06: [io  0x0160-0x016f] has been reserved
[    2.859177] system 00:06: [io  0x0278-0x027f] has been reserved
[    2.947247] system 00:06: [io  0x0378-0x037f] has been reserved
[    2.997167] system 00:06: [io  0x0678-0x067f] has been reserved
[    3.050518] system 00:06: [io  0x0778-0x077f] has been reserved
[    3.104468] system 00:06: [io  0x04d0-0x04d1] has been reserved
[    3.156259] system 00:06: Plug and Play ACPI device, IDs PNP0c02 (active)
[    3.228073] system 00:07: [io  0x0400-0x043f] has been reserved
[    3.309146] system 00:07: [io  0x0370-0x0371] has been reserved
[    3.363529] system 00:07: [io  0x0440-0x044f] has been reserved
[    3.417407] system 00:07: [mem 0xfec00000-0xfec00fff] could not be reserved
[    3.470965] system 00:07: [mem 0xfee00000-0xfee00fff] has been reserved
[    3.535816] system 00:07: Plug and Play ACPI device, IDs PNP0c02 (active)
[    3.601401] system 00:08: [mem 0x00000000-0x0009ffff] could not be reserved
[    3.661929] system 00:08: [mem 0x000c0000-0x000dffff] could not be reserved
[    3.724843] system 00:08: [mem 0x000e0000-0x000fffff] could not be reserved
[    3.784780] system 00:08: [mem 0x00100000-0xf7ffffff] could not be reserved
[    3.836794] system 00:08: [mem 0xfffc0000-0xffffffff] has been reserved
[    3.896305] system 00:08: Plug and Play ACPI device, IDs PNP0c01 (active)
[    3.959068] pnp: PnP ACPI: found 9 devices
[    4.059377] clocksource: acpi_pm: mask: 0xffffff max_cycles: 0xffffff, max_idle_ns: 2085701024 ns
[    4.138966] pci_bus 0000:00: resource 4 [mem 0xfe0000000-0xfffffffff window]
[    4.203613] pci_bus 0000:00: resource 5 [io  0x0000-0x0cf7 window]
[    4.266754] pci_bus 0000:00: resource 6 [io  0x0d00-0xffff window]
[    4.323857] pci_bus 0000:00: resource 7 [mem 0x000a0000-0x000bffff window]
[    4.384073] pci_bus 0000:00: resource 8 [mem 0xf8000000-0xfffbffff window]
[    4.446382] NET: Registered protocol family 2
[    4.492042] TCP established hash table entries: 8192 (order: 4, 65536 bytes)
[    4.550963] TCP bind hash table entries: 8192 (order: 7, 524288 bytes)
[    4.606990] TCP: Hash tables configured (established 8192 bind 8192)
[    4.653913] UDP hash table entries: 512 (order: 4, 81920 bytes)
[    4.700511] UDP-Lite hash table entries: 512 (order: 4, 81920 bytes)
[    4.764170] NET: Registered protocol family 1
[    4.803997] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    4.851800] pci 0000:00:08.0: Video device with shadowed ROM at [mem 0x000c0000-0x000dffff]
[    4.916349] PCI: CLS 0 bytes, default 64
[    4.954665] Unpacking initramfs...
[    7.910742] debug: unmapping init [mem 0xffff88002df40000-0xffff880032f97fff]
[    7.939507] RAPL PMU: API unit is 2^-32 Joules, 3 fixed counters, 10737418240 ms ovfl timer
[    7.967877] RAPL PMU: hw unit of domain pp0-core 2^-0 Joules
[    7.989087] RAPL PMU: hw unit of domain package 2^-0 Joules
[    8.009919] RAPL PMU: hw unit of domain dram 2^-0 Joules
[    8.026505] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x257a25906f7, max_idle_ns: 440795319676 ns
[    8.060648] Scanning for low memory corruption every 60 seconds
[    8.084129] cryptomgr_test (31) used greatest stack depth: 29344 bytes left
[    8.119463] futex hash table entries: 16384 (order: 9, 2097152 bytes)
[    8.142358] audit: initializing netlink subsys (disabled)
[    8.170015] audit: type=2000 audit(1487081368.160:1): initialized
[    8.194098] Initialise system trusted keyrings
[    8.209823] workingset: timestamp_bits=40 max_order=18 bucket_order=0
[    8.393750] zbud: loaded
[    8.442056] SGI XFS with security attributes, no debug enabled
[    8.502466] cryptomgr_test (43) used greatest stack depth: 29120 bytes left
[    8.529500] cryptomgr_test (50) used greatest stack depth: 28616 bytes left
[    8.564780] Key type asymmetric registered
[    8.578682] Asymmetric key parser 'x509' registered
[    8.605328] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 251)
[    8.665619] random: fast init done
[    8.691699] io scheduler noop registered
[    8.725785] io scheduler deadline registered (default)
[    8.774396] io scheduler cfq registered
[    8.808405] io scheduler mq-deadline registered
[    8.878316] GHES: HEST is not enabled!
[    8.913808] Serial: 8250/16550 driver, 32 ports, IRQ sharing enabled
[    9.116686] 00:03: ttyS0 at I/O 0x3f8 (irq = 4, base_baud = 115200) is a 16550A
[    9.340733] 00:04: ttyS1 at I/O 0x2f8 (irq = 3, base_baud = 115200) is a 16550A
[    9.459918] Linux agpgart interface v0.103
[    9.507683] ata_piix 0000:00:07.1: version 2.13
[    9.546712] ata_piix 0000:00:07.1: Hyper-V Virtual Machine detected, ATA device ignore set
[    9.680172] scsi host0: scsi_eh_0: sleeping
[    9.715531] scsi host0: ata_piix
[    9.749703] scsi host1: scsi_eh_1: sleeping
[    9.791021] scsi host1: ata_piix
[    9.821329] ata1: PATA max UDMA/33 cmd 0x1f0 ctl 0x3f6 bmdma 0xffa0 irq 14
[    9.889581] ata2: PATA max UDMA/33 cmd 0x170 ctl 0x376 bmdma 0xffa8 irq 15
[    9.990112] scsi host0: Waking error handler thread
[   10.032269] scsi host0: scsi_eh_0: waking up 1/0/0
[   10.073002] scsi host1: Waking error handler thread
[   10.120455] scsi host1: scsi_eh_1: waking up 1/0/0
[   10.172770] i8042: PNP: PS/2 Controller [PNP0303:PS2K,PNP0f03:PS2M] at 0x60,0x64 irq 1,12
[   10.252613] serio: i8042 KBD port at 0x60,0x64 irq 1
[   10.293633] serio: i8042 AUX port at 0x60,0x64 irq 12
[   10.346090] scsi host1: waking up host to restart
[   10.386705] scsi host1: scsi_eh_1: sleeping
[   10.681782] rtc_cmos 00:00: RTC can wake from S4
[   10.745372] rtc_cmos 00:00: rtc core: registered rtc_cmos as rtc0
[   10.797256] rtc_cmos 00:00: alarms up to one month, 114 bytes nvram
[   10.851660] ata1.01: NODEV after polling detection
[   11.003810] ledtrig-cpu: registered to indicate activity on CPUs
[   11.073799] NET: Registered protocol family 10
[   11.140588] ata1.00: host indicates ignore ATA devices, ignored
[   11.190937] scsi host0: waking up host to restart
[   11.251567] scsi host0: scsi_eh_0: sleeping
[   11.295553] Segment Routing with IPv6
[   11.332968] NET: Registered protocol family 17
[   11.365656] Key type dns_resolver registered
[   11.404429] registered taskstats version 1
[   11.439716] Loading compiled-in X.509 certificates
[   11.487244] zswap: loaded using pool lzo/zbud
[   11.600878] rtc_cmos 00:00: setting system clock to 2017-02-14 14:09:35 UTC (1487081375)
[   11.704000] BIOS EDD facility v0.16 2004-Jun-25, 0 devices found
[   11.754153] EDD information not available.
[   11.813499] debug: unmapping init [mem 0xffffffffa0647000-0xffffffffa07ebfff]
[   11.868404] Write protecting the kernel read-only data: 18432k
[   11.912489] debug: unmapping init [mem 0xffff88000db18000-0xffff88000dbfffff]
[   12.008150] debug: unmapping init [mem 0xffff88000e0ea000-0xffff88000e1fffff]
[   12.105000] mount (68) used greatest stack depth: 28480 bytes left
Loading, please wait...
[   12.268549] all_generic_ide (78) used greatest stack depth: 27944 bytes left
starting version 229
[   13.184271] udevadm (85) used greatest stack depth: 27936 bytes left
[   13.314046] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input0
[   13.342220] clocksource: hyperv_clocksource_tsc_page: mask: 0xffffffffffffffff max_cycles: 0x24e6a1710, max_idle_ns: 440795202120 ns
[   13.533454] clocksource: Switched to clocksource hyperv_clocksource_tsc_page
[   13.559246] hv_vmbus: Hyper-V Host Build:14393-10.0-0-0.576; Vmbus version:4.0
[   13.677005] hv_vmbus: registering driver hyperv_fb
[   13.764010] hv_utils: Registering HyperV Utility Driver
[   13.805837] hv_vmbus: registering driver hv_util
[   13.842200] psmouse serio1: trackpoint: failed to get extended button data
[   13.903246] hv_vmbus: registering driver hyperv_keyboard
[   13.956938] hidraw: raw HID events driver (C) Jiri Kosina
[   14.006668] hv_vmbus: registering driver hv_storvsc
[   14.052442] hyperv_fb: Screen resolution: 1152x864, Color depth: 32
[   14.112021] hv_vmbus: registering driver hid_hyperv
[   14.190750] Console: switching to colour frame buffer device 144x54
[   14.281549] scsi host2: scsi_eh_2: sleeping
[   14.363114] scsi host2: storvsc_host_t
[   14.413729] hv_vmbus: registering driver hv_netvsc
[   14.500991] scsi 2:0:0:0: scsi scan: INQUIRY pass 1 length 36
[   14.552852] scsi 2:0:0:0: tag#0 Send: scmd 0xffff880019ff9448
[   14.612510] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 00 00 00 24 00
[   14.660636] scsi 2:0:0:0: tag#0 Done: SUCCESS Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   14.735249] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 00 00 00 24 00
[   14.739191] scsi 2:0:0:0: tag#0 scsi host busy 1 failed 0
[   14.739191] scsi 2:0:0:0: Notifying upper driver of completion (result 0)
[   14.739191] scsi 2:0:0:0: scsi scan: INQUIRY successful with code 0x0
[   14.964062] scsi 2:0:0:0: Direct-Access     Msft     Virtual Disk     1.0  PQ: 0 ANSI: 5
[   15.038762] scsi 2:0:0:0: tag#0 Send: scmd 0xffff880019ffd5e8
[   15.089616] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 01 00 00 ff 00
[   15.137673] scsi 2:0:0:0: tag#0 Done: SUCCESS Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   15.207744] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 01 00 00 ff 00
[   15.255839] scsi 2:0:0:0: tag#0 scsi host busy 1 failed 0
[   15.304291] scsi 2:0:0:0: Notifying upper driver of completion (result 0)
[   15.362157] scsi 2:0:0:0: tag#0 Send: scmd 0xffff880019ffa708
[   15.413281] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 01 83 00 ff 00
[   15.460944] scsi 2:0:0:0: tag#0 Done: SUCCESS Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   15.518901] scsi 2:0:0:0: tag#0 CDB: Inquiry 12 01 83 00 ff 00
[   15.570146] scsi 2:0:0:0: tag#0 scsi host busy 1 failed 0
[   15.618008] scsi 2:0:0:0: Notifying upper driver of completion (result 0)
[   15.678847] sd 2:0:0:0: tag#0 Send: scmd 0xffff880019ffb068
[   15.723329] sd 2:0:0:0: tag#0 CDB: Test Unit Ready 00 00 00 00 00 00
[   15.775429] sd 2:0:0:0: sg_alloc: dev=0 
[   15.809186] sd 2:0:0:0: Attached scsi generic sg0 type 0
[   15.884302] input: Microsoft Vmbus HID-compliant Mouse as /devices/0006:045E:0621.0001/input/input3
[   15.976010] hid-generic 0006:045E:0621.0001: input: <UNKNOWN> HID v0.01 Mouse [Microsoft Vmbus HID-compliant Mouse] on 
[   16.198080] hv_utils: Using TimeSync version 4.0
[   18.949416] psmouse serio1: trackpoint: IBM TrackPoint firmware: 0x01, buttons: 0/0
[   19.013187] input: TPPS/2 IBM TrackPoint as /devices/platform/i8042/serio1/input/input2
[   19.088795] input: AT Translated Set 2 keyboard as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A03:00/device:07/VMBUS:01/d34b2567-b9b6-42b9-8778-0a4ec0b955bf/serio2/input/input4
[   46.048170] sd 2:0:0:0: tag#0 Done: TIMEOUT_ERROR Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   46.052054] sd 2:0:0:0: tag#0 CDB: Test Unit Ready 00 00 00 00 00 00
[   46.052054] sd 2:0:0:0: tag#0 scsi host busy 1 failed 0
[   78.816191] sd 2:0:0:0: tag#0 Done: TIMEOUT_ERROR Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[   78.820056] sd 2:0:0:0: tag#0 CDB: Test Unit Ready 00 00 00 00 00 00
[   78.820056] sd 2:0:0:0: tag#0 scsi host busy 1 failed 0
[  111.584194] sd 2:0:0:0: tag#0 Done: TIMEOUT_ERROR Result: hostbyte=DID_OK driverbyte=DRIVER_OK
[  111.588064] sd 2:0:0:0: tag#0 CDB: Test Unit Ready 00 00 00 00 00 00
[  111.588064] sd 2:0:0:0: tag#0 scsi host busy 1 failed 0
[  121.824225] INFO: task systemd-udevd:104 blocked for more than 60 seconds.
[  121.874720]       Not tainted 4.10.0-rc5+ #19
[  121.914743] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  121.990829] systemd-udevd   D25232   104     84 0x00000000
[  122.036193] Call Trace:
[  122.055827]  __schedule+0x48c/0xc90
[  122.086341]  ? do_raw_spin_lock+0x112/0x1a0
[  122.121098]  schedule+0x4d/0xd0
[  122.146919]  async_synchronize_cookie_domain+0x133/0x1e0
[  122.277712]  ? async_unregister_domain+0x70/0x70
[  122.326912]  ? remove_wait_queue+0xc0/0xc0
[  122.360589]  ? up_read+0x1f/0x40
[  122.391119]  ? __blocking_notifier_call_chain+0x6c/0x80
[  122.440005]  async_synchronize_full+0x17/0x20
[  122.478899]  do_init_module+0x174/0x311
[  122.517420]  load_module+0x4484/0x4c00
[  122.552531]  ? m_show+0x2b0/0x2b0
[  122.580499]  ? fsnotify+0x769/0x7e0
[  122.623893]  ? module_frob_arch_sections+0x20/0x20
[  122.671746]  ? __fsnotify_parent+0x2c/0x130
[  122.711308]  ? vfs_read+0x14b/0x1b0
[  122.742297]  ? kernel_read_file+0x12a/0x350
[  122.778480]  ? set_binfmt+0x90/0x90
[  122.826074]  ? kernel_read_file_from_fd+0x49/0x80
[  122.867793]  SYSC_finit_module+0x169/0x1a0
[  122.904224]  ? SYSC_init_module+0x1d0/0x1d0
[  122.947117]  ? vma_is_stack_for_current+0x60/0x60
[  122.988156]  ? __fget+0x10b/0x150
[  123.013801]  ? lockdep_sys_exit+0x1a/0xa0
[  123.050099]  ? lockdep_sys_exit_thunk+0x16/0x30
[  123.087355]  SyS_finit_module+0xe/0x10
[  123.115902]  entry_SYSCALL_64_fastpath+0x1e/0xb2
[  123.151843] RIP: 0033:0x7f6060499c19
[  123.179364] RSP: 002b:00007ffec634f808 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[  123.241863] RAX: ffffffffffffffda RBX: 000055f86ee6cbd0 RCX: 00007f6060499c19
[  123.301211] RDX: 0000000000000000 RSI: 00007f606098fe2a RDI: 000000000000000c
[  123.364226] RBP: 00007f606098fe2a R08: 0000000000000000 R09: 0000000000000000
[  123.427614] R10: 000000000000000c R11: 0000000000000246 R12: 0000000000000000
[  123.492838] R13: 000055f86ee70f30 R14: 0000000000020000 R15: 000000000aba9500
[  123.556355] 
[  123.556355] Showing all locks held in the system:
[  123.610348] 2 locks held by khungtaskd/17:
[  123.645766]  #0:  (rcu_read_lock){......}, at: [<ffffffff9f1abc56>] watchdog+0xc6/0x530
[  123.715830]  #1:  (tasklist_lock){......}, at: [<ffffffff9f111a89>] debug_show_all_locks+0x49/0x1e0
[  123.806486] 2 locks held by kworker/u128:1/61:
[  123.841210]  #0:  ("events_unbound"){......}, at: [<ffffffff9f0bf3c2>] process_one_work+0x3d2/0xa60
[  123.909420]  #1:  ((&entry->work)){......}, at: [<ffffffff9f0bf3c2>] process_one_work+0x3d2/0xa60
[  123.986273] 
[  123.999135] =============================================
[  123.999135] 
Begin: Loading essential drivers ... done.
Begin: Running /scripts/init-premount ... done.
Begin: Mounting root file system ... Begin: Running /scripts/local-top ... done.
Begin: Running /scripts/local-premount ... [  134.463258] random: crng init done

Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[hch]

[-- Attachment #1: Type: text/plain, Size: 199 bytes --]

Ok, thanks for testing.  Can you try the patch below?  It fixes a
clear problem which was partially papered over before the commit
you bisected to, although it can't explain why blk-mq still works.


[-- Attachment #2: 0001-scsi-always-zero-sshdr-in-scsi_normalize_sense.patch --]
[-- Type: text/x-patch, Size: 999 bytes --]

>From e4a66856fa2d92c0298000de658365f31bea60cd Mon Sep 17 00:00:00 2001
From: Christoph Hellwig <hch@lst.de>
Date: Tue, 14 Feb 2017 15:08:39 +0100
Subject: scsi: always zero sshdr in scsi_normalize_sense

This gives us a clear state even if a command didn't return sense data.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 drivers/scsi/scsi_common.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/scsi/scsi_common.c b/drivers/scsi/scsi_common.c
index b1383a71400e..a75673bb82b3 100644
--- a/drivers/scsi/scsi_common.c
+++ b/drivers/scsi/scsi_common.c
@@ -137,11 +137,11 @@ EXPORT_SYMBOL(int_to_scsilun);
 bool scsi_normalize_sense(const u8 *sense_buffer, int sb_len,
 			  struct scsi_sense_hdr *sshdr)
 {
+	memset(sshdr, 0, sizeof(struct scsi_sense_hdr));
+
 	if (!sense_buffer || !sb_len)
 		return false;
 
-	memset(sshdr, 0, sizeof(struct scsi_sense_hdr));
-
 	sshdr->response_code = (sense_buffer[0] & 0x7f);
 
 	if (!scsi_sense_valid(sshdr))
-- 
2.11.0

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

> From: hch@lst.de [mailto:hch@lst.de]
> Sent: Tuesday, February 14, 2017 22:29
> To: Dexuan Cui <decui@microsoft.com>
> Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event l=
ock
> when scheduling workqueue elements")
>=20
> Ok, thanks for testing.  Can you try the patch below?  It fixes a
> clear problem which was partially papered over before the commit
> you bisected to, although it can't explain why blk-mq still works.

Still bad luck. :-(

BTW, I'm using the first "bad" commit (scsi: allocate scsi_cmnd structures =
as
part of struct request) + the 2 patches you provided today.

I suppose I don't need to test the 2 patches on the latest linux-next repo.

Thanks,
-- Dexuan

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

> From: hch@lst.de [mailto:hch@lst.de]
> Sent: Tuesday, February 14, 2017 22:29
> To: Dexuan Cui <decui@microsoft.com>
> Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock
> when scheduling workqueue elements")
> 
> Ok, thanks for testing.  Can you try the patch below?  It fixes a
> clear problem which was partially papered over before the commit
> you bisected to, although it can't explain why blk-mq still works.

Still bad luck. :-(

BTW, I'm using the first "bad" commit (scsi: allocate scsi_cmnd structures as
part of struct request) + the 2 patches you provided today.

I suppose I don't need to test the 2 patches on the latest linux-next repo.

Thanks,
-- Dexuan

Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[hch]

On Tue, Feb 14, 2017 at 02:46:41PM +0000, Dexuan Cui wrote:
> > From: hch@lst.de [mailto:hch@lst.de]
> > Sent: Tuesday, February 14, 2017 22:29
> > To: Dexuan Cui <decui@microsoft.com>
> > Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock
> > when scheduling workqueue elements")
> > 
> > Ok, thanks for testing.  Can you try the patch below?  It fixes a
> > clear problem which was partially papered over before the commit
> > you bisected to, although it can't explain why blk-mq still works.
> 
> Still bad luck. :-(
> 
> BTW, I'm using the first "bad" commit (scsi: allocate scsi_cmnd structures as
> part of struct request) + the 2 patches you provided today.
> 
> I suppose I don't need to test the 2 patches on the latest linux-next repo.

I'd love a test on that repo actually.  We had a few other for sense
handling since then I think.

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

[-- Attachment #1: Type: text/plain, Size: 2451 bytes --]

> From: hch@lst.de [mailto:hch@lst.de]
> Sent: Tuesday, February 14, 2017 22:51
> To: Dexuan Cui <decui@microsoft.com>
> Cc: hch@lst.de; Jens Axboe <axboe@kernel.dk>; Bart Van Assche
> <Bart.VanAssche@sandisk.com>; hare@suse.com; hare@suse.de; Martin K.
> Petersen <martin.petersen@oracle.com>; linux-kernel@vger.kernel.org;
> linux-block@vger.kernel.org; jth@kernel.org; Nick Meier
> <Nick.Meier@microsoft.com>; Alex Ng (LIS) <alexng@microsoft.com>; Long Li
> <longli@microsoft.com>; Adrian Suhov (Cloudbase Solutions SRL) <v-
> adsuho@microsoft.com>; Chris Valean (Cloudbase Solutions SRL) <v-
> chvale@microsoft.com>
> Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock
> when scheduling workqueue elements")
> 
> On Tue, Feb 14, 2017 at 02:46:41PM +0000, Dexuan Cui wrote:
> > > From: hch@lst.de [mailto:hch@lst.de]
> > > Sent: Tuesday, February 14, 2017 22:29
> > > To: Dexuan Cui <decui@microsoft.com>
> > > Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event
> lock
> > > when scheduling workqueue elements")
> > >
> > > Ok, thanks for testing.  Can you try the patch below?  It fixes a
> > > clear problem which was partially papered over before the commit
> > > you bisected to, although it can't explain why blk-mq still works.
> >
> > Still bad luck. :-(
> >
> > BTW, I'm using the first "bad" commit (scsi: allocate scsi_cmnd structures
> as
> > part of struct request) + the 2 patches you provided today.
> >
> > I suppose I don't need to test the 2 patches on the latest linux-next repo.
> 
> I'd love a test on that repo actually.  We had a few other for sense
> handling since then I think.

I tested today's linux-next (next-20170214) + the 2 patches just now and got
a weird result: 
sometimes the VM stills hung with a new calltrace (BUG: spinlock bad
magic) , but sometimes the VM did boot up despite the new calltrace!

Attached is the log of a "good" boot.

It looks we have a memory corruption issue somewhere...

Actually previously I saw the "BUG: spinlock bad magic" message once, but I
couldn't repro it later, so I didn't mention it to you.

The good news is that now I can repro the "spinlock bad magic" message
every time. 
I tried to dig into this by enabling Kernel hacking -> Memory debugging,
but didn't find anything abnormal. 
Is it possible that the SCSI layer passes a wrong memory address?

Thanks,
-- Dexuan

[-- Attachment #2: dmesg.log --]
[-- Type: application/octet-stream, Size: 31337 bytes --]

[    0.000000] Linux version 4.10.0-rc8-next-20170214+ (root@decui-u1604) (gcc version 5.3.1 20160413 (Ubuntu 5.3.1-14ubuntu2.1) ) #1 SMP Tue Feb 14 23:10:48 CST 2017
[    0.000000] Command line: BOOT_IMAGE=/boot/hv/bzImage root=UUID=f07ac67a-0109-4738-a388-100218d2c5d2 ro ignore_loglevel scsi_mod.use_blk_mq=N
[    0.000000] KERNEL supported cpus:
[    0.000000]   Intel GenuineIntel
[    0.000000] x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers'
[    0.000000] x86/fpu: xstate_offset[2]:  576, xstate_sizes[2]:  256
[    0.000000] x86/fpu: Enabled xstate features 0x7, context size is 832 bytes, using 'standard' format.
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009fbff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000e0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000003ffeffff] usable
[    0.000000] BIOS-e820: [mem 0x000000003fff0000-0x000000003fffefff] ACPI data
[    0.000000] BIOS-e820: [mem 0x000000003ffff000-0x000000003fffffff] ACPI NVS
[    0.000000] debug: ignoring loglevel setting.
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 2.3 present.
[    0.000000] DMI: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS 090006  04/28/2016
[    0.000000] Hypervisor detected: Microsoft HyperV
[    0.000000] HyperV: features 0x2e7f, hints 0xc2c
[    0.000000] Hyper-V Host Build:14393-10.0-0-0.576
[    0.000000] HyperV: LAPIC Timer Frequency: 0xc3500
[    0.000000] tsc: Marking TSC unstable due to running on Hyper-V
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x3fff0 max_arch_pfn = 0x400000000
[    0.000000] MTRR default type: uncachable
[    0.000000] MTRR fixed ranges enabled:
[    0.000000]   00000-9FFFF write-back
[    0.000000]   A0000-DFFFF uncachable
[    0.000000]   E0000-FFFFF write-back
[    0.000000] MTRR variable ranges enabled:
[    0.000000]   0 base 00000000000 mask FFF00000000 write-back
[    0.000000]   1 disabled
[    0.000000]   2 disabled
[    0.000000]   3 disabled
[    0.000000]   4 disabled
[    0.000000]   5 disabled
[    0.000000]   6 disabled
[    0.000000]   7 disabled
[    0.000000] x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WC  UC- WT  
[    0.000000] found SMP MP-table at [mem 0x000ff780-0x000ff78f] mapped at [ffff9539400ff780]
[    0.000000] Scanning 1 areas for low memory corruption
[    0.000000] Base memory trampoline at [ffff953940099000] 99000 size 24576
[    0.000000] BRK [0x0e91c000, 0x0e91cfff] PGTABLE
[    0.000000] BRK [0x0e91d000, 0x0e91dfff] PGTABLE
[    0.000000] BRK [0x0e91e000, 0x0e91efff] PGTABLE
[    0.000000] BRK [0x0e91f000, 0x0e91ffff] PGTABLE
[    0.000000] BRK [0x0e920000, 0x0e920fff] PGTABLE
[    0.000000] RAMDISK: [mem 0x2fde8000-0x33eebfff]
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000000F56F0 000014 (v00 ACPIAM)
[    0.000000] ACPI: RSDT 0x000000003FFF0000 000040 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: FACP 0x000000003FFF0200 000081 (v02 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: DSDT 0x000000003FFF1724 0033BE (v01 MSFTVM MSFTVM02 00000002 INTL 02002026)
[    0.000000] ACPI: FACS 0x000000003FFFF000 000040
[    0.000000] ACPI: WAET 0x000000003FFF1480 000028 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: SLIC 0x000000003FFF14C0 000176 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: OEM0 0x000000003FFF16C0 000064 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: SRAT 0x000000003FFF0600 0000E0 (v02 VRTUAL MICROSFT 00000001 MSFT 00000001)
[    0.000000] ACPI: APIC 0x000000003FFF0300 000252 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: OEMB 0x000000003FFFF040 000064 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] SRAT: PXM 0 -> APIC 0x00 -> Node 0
[    0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x00000000-0x3fffffff] hotplug
[    0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x40200000-0xf7ffffff] hotplug
[    0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x100000000-0xfdfffffff] hotplug
[    0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x1000000000-0xffffffffff] hotplug
[    0.000000] NODE_DATA(0) allocated [mem 0x3ffec000-0x3ffeffff]
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.000000]   DMA32    [mem 0x0000000001000000-0x000000003ffeffff]
[    0.000000]   Normal   empty
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x0000000000001000-0x000000000009efff]
[    0.000000]   node   0: [mem 0x0000000000100000-0x000000003ffeffff]
[    0.000000] Initmem setup node 0 [mem 0x0000000000001000-0x000000003ffeffff]
[    0.000000] On node 0 totalpages: 262030
[    0.000000]   DMA zone: 64 pages used for memmap
[    0.000000]   DMA zone: 21 pages reserved
[    0.000000]   DMA zone: 3998 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 4032 pages used for memmap
[    0.000000]   DMA32 zone: 258032 pages, LIFO batch:31
[    0.000000] ACPI: PM-Timer IO Port: 0x408
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
[    0.000000] IOAPIC[0]: apic_id 0, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] smpboot: Allowing 64 CPUs, 63 hotplug CPUs
[    0.000000] e820: [mem 0x40000000-0xffffffff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on bare hardware
[    0.000000] clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645519600211568 ns
[    0.000000] setup_percpu: NR_CPUS:256 nr_cpumask_bits:256 nr_cpu_ids:64 nr_node_ids:1
[    0.000000] percpu: Embedded 37 pages/cpu @ffff95397d600000 s112328 r8192 d31032 u262144
[    0.000000] pcpu-alloc: s112328 r8192 d31032 u262144 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 00 01 02 03 04 05 06 07 [0] 08 09 10 11 12 13 14 15 
[    0.000000] pcpu-alloc: [0] 16 17 18 19 20 21 22 23 [0] 24 25 26 27 28 29 30 31 
[    0.000000] pcpu-alloc: [0] 32 33 34 35 36 37 38 39 [0] 40 41 42 43 44 45 46 47 
[    0.000000] pcpu-alloc: [0] 48 49 50 51 52 53 54 55 [0] 56 57 58 59 60 61 62 63 
[    0.000000] Built 1 zonelists in Node order, mobility grouping on.  Total pages: 257913
[    0.000000] Policy zone: DMA32
[    0.000000] Kernel command line: BOOT_IMAGE=/boot/hv/bzImage root=UUID=f07ac67a-0109-4738-a388-100218d2c5d2 ro ignore_loglevel scsi_mod.use_blk_mq=N
[    0.000000] log_buf_len individual max cpu contribution: 4096 bytes
[    0.000000] log_buf_len total cpu_extra contributions: 258048 bytes
[    0.000000] log_buf_len min size: 262144 bytes
[    0.000000] log_buf_len: 524288 bytes
[    0.000000] early log buf free: 254580(97%)
[    0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    0.000000] Memory: 863304K/1048120K available (6896K kernel code, 1440K rwdata, 2784K rodata, 1304K init, 10644K bss, 184816K reserved, 0K cma-reserved)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=64, Nodes=1
[    0.000000] Hierarchical RCU implementation.
[    0.000000] 	Build-time adjustment of leaf fanout to 64.
[    0.000000] 	RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=64.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=64
[    0.000000] NR_IRQS:16640 nr_irqs:936 16
[    0.000000] 	Offload RCU callbacks from all CPUs
[    0.000000] 	Offload RCU callbacks from CPUs: 0-63.
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [tty0] enabled
[    0.000000] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.000000] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.000000] ... MAX_LOCK_DEPTH:          48
[    0.000000] ... MAX_LOCKDEP_KEYS:        8191
[    0.000000] ... CLASSHASH_SIZE:          4096
[    0.000000] ... MAX_LOCKDEP_ENTRIES:     32768
[    0.000000] ... MAX_LOCKDEP_CHAINS:      65536
[    0.000000] ... CHAINHASH_SIZE:          32768
[    0.000000]  memory used by lock dependency info: 8127 kB
[    0.000000]  per task-struct memory footprint: 1920 bytes
[    0.000000] tsc: Fast TSC calibration failed
[    0.012000] tsc: Unable to calibrate against PIT
[    0.016000] tsc: using PMTIMER reference calibration
[    0.016000] tsc: Detected 2600.008 MHz processor
[    0.016000] Calibrating delay loop (skipped), value calculated using timer frequency.. 5200.01 BogoMIPS (lpj=10400032)
[    0.016000] pid_max: default: 65536 minimum: 512
[    0.020112] ACPI: Core revision 20170119
[    0.025555] ACPI: 1 ACPI AML tables successfully acquired and loaded
[    0.026775] Dentry cache hash table entries: 131072 (order: 8, 1048576 bytes)
[    0.028419] Inode-cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.030051] Mount-cache hash table entries: 2048 (order: 2, 16384 bytes)
[    0.030981] Mountpoint-cache hash table entries: 2048 (order: 2, 16384 bytes)
[    0.032912] CPU: Physical Processor ID: 0
[    0.033859] mce: CPU supports 1 MCE banks
[    0.034848] Last level iTLB entries: 4KB 512, 2MB 8, 4MB 8
[    0.036046] Last level dTLB entries: 4KB 512, 2MB 0, 4MB 0, 1GB 4
[    0.054815] ftrace: allocating 24941 entries in 98 pages
[    0.069829] smpboot: Max logical packages: 64
[    0.072084] Switched APIC routing to physical flat.
[    0.072963] clocksource: hyperv_clocksource_tsc_page: mask: 0xffffffffffffffff max_cycles: 0x24e6a1710, max_idle_ns: 440795202120 ns
[    0.111150] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.112225] smpboot: CPU0: Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz (family: 0x6, model: 0x3e, stepping: 0x4)
[    0.113384] Performance Events: unsupported p6 CPU model 62 no PMU driver, software events only.
[    0.116871] NMI watchdog: disabled (cpu0): hardware events not enabled
[    0.117812] NMI watchdog: Shutting down hard lockup detector on all cpus
[    0.118714] smp: Bringing up secondary CPUs ...
[    0.119568] smp: Brought up 1 node, 1 CPU
[    0.120023] smpboot: Total of 1 processors activated (5200.01 BogoMIPS)
[    0.129001] devtmpfs: initialized
[    0.130063] x86/mm: Memory block size: 128MB
[    0.136138] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
[    0.137510] pinctrl core: initialized pinctrl subsystem
[    0.138840] NET: Registered protocol family 16
[    0.144098] cpuidle: using governor ladder
[    0.148010] cpuidle: using governor menu
[    0.148845] PCCT header not found.
[    0.149763] ACPI: bus type PCI registered
[    0.152433] PCI: Using configuration type 1 for base access
[    0.158484] HugeTLB registered 1 GB page size, pre-allocated 0 pages
[    0.160011] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    0.162411] ACPI: Added _OSI(Module Device)
[    0.164009] ACPI: Added _OSI(Processor Device)
[    0.164843] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.165672] ACPI: Added _OSI(Processor Aggregator Device)
[    0.176058] ACPI: Interpreter enabled
[    0.176988] ACPI: (supports S0 S5)
[    0.177811] ACPI: Using IOAPIC for interrupt routing
[    0.178691] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[    0.205665] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.206736] acpi PNP0A03:00: _OSC: OS supports [Segments MSI]
[    0.207608] acpi PNP0A03:00: _OSC failed (AE_NOT_FOUND); disabling ASPM
[    0.208181] PCI host bridge to bus 0000:00
[    0.209050] pci_bus 0000:00: root bus resource [mem 0xfe0000000-0xfffffffff window]
[    0.209979] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7 window]
[    0.210833] pci_bus 0000:00: root bus resource [io  0x0d00-0xffff window]
[    0.212009] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window]
[    0.212945] pci_bus 0000:00: root bus resource [mem 0xf8000000-0xfffbffff window]
[    0.213875] pci_bus 0000:00: root bus resource [bus 00-ff]
[    0.215031] pci 0000:00:00.0: [8086:7192] type 00 class 0x060000
[    0.220504] pci 0000:00:07.0: [8086:7110] type 00 class 0x060100
[    0.225876] pci 0000:00:07.1: [8086:7111] type 00 class 0x010180
[    0.230354] pci 0000:00:07.1: reg 0x20: [io  0xffa0-0xffaf]
[    0.232661] pci 0000:00:07.1: legacy IDE quirk: reg 0x10: [io  0x01f0-0x01f7]
[    0.234841] pci 0000:00:07.1: legacy IDE quirk: reg 0x14: [io  0x03f6]
[    0.235712] pci 0000:00:07.1: legacy IDE quirk: reg 0x18: [io  0x0170-0x0177]
[    0.236010] pci 0000:00:07.1: legacy IDE quirk: reg 0x1c: [io  0x0376]
[    0.239084] pci 0000:00:07.3: [8086:7113] type 00 class 0x068000
[    0.240088] * Found PM-Timer Bug on the chipset. Due to workarounds for a bug,
               * this clock source is slow. Consider trying other clock sources
[    0.245925] pci 0000:00:07.3: quirk: [io  0x0400-0x043f] claimed by PIIX4 ACPI
[    0.248454] pci 0000:00:08.0: [1414:5353] type 00 class 0x030000
[    0.250097] pci 0000:00:08.0: reg 0x10: [mem 0xf8000000-0xfbffffff]
[    0.269769] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 7 9 10 *11 12 14 15)
[    0.271040] ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    0.272376] ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    0.273715] ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    0.274964] ACPI: Enabled 1 GPEs in block 00 to 0F
[    0.277122] pci 0000:00:08.0: vgaarb: setting as boot VGA device
[    0.277911] pci 0000:00:08.0: vgaarb: VGA device added: decodes=io+mem,owns=io+mem,locks=none
[    0.278775] pci 0000:00:08.0: vgaarb: bridge control possible
[    0.279559] vgaarb: loaded
[    0.280170] SCSI subsystem initialized
[    0.281217] libata version 3.00 loaded.
[    0.284051] PCI: Using ACPI for IRQ routing
[    0.286144] PCI: pci_cache_line_size set to 64 bytes
[    0.287653] e820: reserve RAM buffer [mem 0x0009fc00-0x0009ffff]
[    0.288023] e820: reserve RAM buffer [mem 0x3fff0000-0x3fffffff]
[    0.289406] clocksource: Switched to clocksource hyperv_clocksource_tsc_page
[    0.342146] VFS: Disk quotas dquot_6.6.0
[    0.343311] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    0.345144] pnp: PnP ACPI init
[    0.346201] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[    0.347171] pnp 00:01: Plug and Play ACPI device, IDs PNP0303 PNP030b (active)
[    0.348231] pnp 00:02: Plug and Play ACPI device, IDs PNP0f03 PNP0f13 (active)
[    0.350505] pnp 00:03: [dma 0 disabled]
[    0.351389] pnp 00:03: Plug and Play ACPI device, IDs PNP0501 (active)
[    0.353580] pnp 00:04: [dma 0 disabled]
[    0.354429] pnp 00:04: Plug and Play ACPI device, IDs PNP0501 (active)
[    0.356734] pnp 00:05: [dma 2]
[    0.357670] pnp 00:05: Plug and Play ACPI device, IDs PNP0700 (active)
[    0.358706] system 00:06: [io  0x01e0-0x01ef] has been reserved
[    0.359537] system 00:06: [io  0x0160-0x016f] has been reserved
[    0.360393] system 00:06: [io  0x0278-0x027f] has been reserved
[    0.361216] system 00:06: [io  0x0378-0x037f] has been reserved
[    0.362032] system 00:06: [io  0x0678-0x067f] has been reserved
[    0.362848] system 00:06: [io  0x0778-0x077f] has been reserved
[    0.363661] system 00:06: [io  0x04d0-0x04d1] has been reserved
[    0.364519] system 00:06: Plug and Play ACPI device, IDs PNP0c02 (active)
[    0.365670] system 00:07: [io  0x0400-0x043f] has been reserved
[    0.366579] system 00:07: [io  0x0370-0x0371] has been reserved
[    0.367403] system 00:07: [io  0x0440-0x044f] has been reserved
[    0.368261] system 00:07: [mem 0xfec00000-0xfec00fff] could not be reserved
[    0.369087] system 00:07: [mem 0xfee00000-0xfee00fff] has been reserved
[    0.369875] system 00:07: Plug and Play ACPI device, IDs PNP0c02 (active)
[    0.370930] system 00:08: [mem 0x00000000-0x0009ffff] could not be reserved
[    0.371843] system 00:08: [mem 0x000c0000-0x000dffff] could not be reserved
[    0.374012] system 00:08: [mem 0x000e0000-0x000fffff] could not be reserved
[    0.374858] system 00:08: [mem 0x00100000-0xf7ffffff] could not be reserved
[    0.375703] system 00:08: [mem 0xfffc0000-0xffffffff] has been reserved
[    0.376579] system 00:08: Plug and Play ACPI device, IDs PNP0c01 (active)
[    0.379161] pnp: PnP ACPI: found 9 devices
[    0.387621] clocksource: acpi_pm: mask: 0xffffff max_cycles: 0xffffff, max_idle_ns: 2085701024 ns
[    0.388618] pci_bus 0000:00: resource 4 [mem 0xfe0000000-0xfffffffff window]
[    0.389456] pci_bus 0000:00: resource 5 [io  0x0000-0x0cf7 window]
[    0.390281] pci_bus 0000:00: resource 6 [io  0x0d00-0xffff window]
[    0.391105] pci_bus 0000:00: resource 7 [mem 0x000a0000-0x000bffff window]
[    0.391931] pci_bus 0000:00: resource 8 [mem 0xf8000000-0xfffbffff window]
[    0.392959] NET: Registered protocol family 2
[    0.394453] TCP established hash table entries: 8192 (order: 4, 65536 bytes)
[    0.395457] TCP bind hash table entries: 8192 (order: 7, 524288 bytes)
[    0.396936] TCP: Hash tables configured (established 8192 bind 8192)
[    0.398239] UDP hash table entries: 512 (order: 4, 81920 bytes)
[    0.399295] UDP-Lite hash table entries: 512 (order: 4, 81920 bytes)
[    0.400881] NET: Registered protocol family 1
[    0.401792] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    0.403210] pci 0000:00:08.0: Video device with shadowed ROM at [mem 0x000c0000-0x000dffff]
[    0.404276] PCI: CLS 0 bytes, default 64
[    0.405282] Unpacking initramfs...
[    1.706148] Freeing initrd memory: 66576K
[    1.706919] PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
[    1.707234] software IO TLB [mem 0x39600000-0x3d600000] (64MB) mapped at [ffff953979600000-ffff95397d5fffff]
[    1.707690] RAPL PMU: API unit is 2^-32 Joules, 3 fixed counters, 10737418240 ms ovfl timer
[    1.708045] RAPL PMU: hw unit of domain pp0-core 2^-0 Joules
[    1.708358] RAPL PMU: hw unit of domain package 2^-0 Joules
[    1.708669] RAPL PMU: hw unit of domain dram 2^-0 Joules
[    1.709048] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x257a4365b0e, max_idle_ns: 440795224253 ns
[    1.709427] Scanning for low memory corruption every 60 seconds
[    1.710395] futex hash table entries: 16384 (order: 9, 2097152 bytes)
[    1.711254] audit: initializing netlink subsys (disabled)
[    1.711684] cryptomgr_test (31) used greatest stack depth: 14528 bytes left
[    1.712237] Initialise system trusted keyrings
[    1.712598] audit: type=2000 audit(1487085689.712:1): state=initialized audit_enabled=0 res=1
[    1.713063] workingset: timestamp_bits=40 max_order=18 bucket_order=0
[    1.714737] zbud: loaded
[    1.715782] SGI XFS with security attributes, no debug enabled
[    1.718151] Key type asymmetric registered
[    1.718518] Asymmetric key parser 'x509' registered
[    1.718887] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 251)
[    1.719379] io scheduler noop registered
[    1.719720] io scheduler deadline registered (default)
[    1.720518] io scheduler cfq registered
[    1.720877] io scheduler mq-deadline registered
[    1.721393] GHES: HEST is not enabled!
[    1.721770] Serial: 8250/16550 driver, 32 ports, IRQ sharing enabled
[    1.769785] 00:03: ttyS0 at I/O 0x3f8 (irq = 4, base_baud = 115200) is a 16550A
[    1.818962] 00:04: ttyS1 at I/O 0x2f8 (irq = 3, base_baud = 115200) is a 16550A
[    1.820459] Linux agpgart interface v0.103
[    1.820970] ata_piix 0000:00:07.1: version 2.13
[    1.821431] ata_piix 0000:00:07.1: Hyper-V Virtual Machine detected, ATA device ignore set
[    1.823794] scsi host0: ata_piix
[    1.824253] scsi host1: ata_piix
[    1.824589] ata1: PATA max UDMA/33 cmd 0x1f0 ctl 0x3f6 bmdma 0xffa0 irq 14
[    1.824922] ata2: PATA max UDMA/33 cmd 0x170 ctl 0x376 bmdma 0xffa8 irq 15
[    1.825372] i8042: PNP: PS/2 Controller [PNP0303:PS2K,PNP0f03:PS2M] at 0x60,0x64 irq 1,12
[    1.828787] serio: i8042 KBD port at 0x60,0x64 irq 1
[    1.829127] serio: i8042 AUX port at 0x60,0x64 irq 12
[    1.830457] rtc_cmos 00:00: RTC can wake from S4
[    1.852390] rtc_cmos 00:00: rtc core: registered rtc_cmos as rtc0
[    1.853241] rtc_cmos 00:00: alarms up to one month, 114 bytes nvram
[    1.853629] ledtrig-cpu: registered to indicate activity on CPUs
[    1.854189] NET: Registered protocol family 10
[    1.855119] Segment Routing with IPv6
[    1.855495] NET: Registered protocol family 17
[    1.855813] Key type dns_resolver registered
[    1.856411] registered taskstats version 1
[    1.857175] Loading compiled-in X.509 certificates
[    1.857529] zswap: loaded using pool lzo/zbud
[    1.858415] rtc_cmos 00:00: setting system clock to 2017-02-14 15:21:30 UTC (1487085690)
[    1.858766] BIOS EDD facility v0.16 2004-Jun-25, 0 devices found
[    1.859087] EDD information not available.
[    2.093148] ata1.01: NODEV after polling detection
[    2.143359] ata1.00: host indicates ignore ATA devices, ignored
[    2.144884] Freeing unused kernel memory: 1304K
[    2.145283] Write protecting the kernel read-only data: 12288k
[    2.146252] Freeing unused kernel memory: 1280K
[    2.148804] Freeing unused kernel memory: 1312K
[    2.150963] mount (67) used greatest stack depth: 14512 bytes left
[    2.152646] exe (70) used greatest stack depth: 14504 bytes left
[    2.155475] exe (73) used greatest stack depth: 14064 bytes left
[    2.163564] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.163978] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.164923] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    2.165384] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    2.165716] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    2.166047] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    2.171533] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.171994] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.172758] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.173192] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.235888] udevadm (84) used greatest stack depth: 13904 bytes left
[    2.279298] hv_vmbus: Vmbus version:4.0
[    2.291556] hv_vmbus: registering driver hyperv_fb
[    2.301275] hyperv_fb: Screen resolution: 1152x864, Color depth: 32
[    2.310220] Console: switching to colour frame buffer device 144x54
[    2.342210] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input0
[    2.454348] hidraw: raw HID events driver (C) Jiri Kosina
[    2.455357] hv_vmbus: registering driver hid_hyperv
[    2.456632] hv_vmbus: registering driver hyperv_keyboard
[    2.459289] hv_utils: Registering HyperV Utility Driver
[    2.459346] hv_vmbus: registering driver hv_util
[    2.468340] hv_vmbus: registering driver hv_netvsc
[    2.472734] hv_vmbus: registering driver hv_storvsc
[    2.477142] input: Microsoft Vmbus HID-compliant Mouse as /devices/0006:045E:0621.0001/input/input3
[    2.477250] hid 0006:045E:0621.0001: input: <UNKNOWN> HID v0.01 Mouse [Microsoft Vmbus HID-compliant Mouse] on 
[    2.478576] hv_utils: Heartbeat IC version 3.0
[    2.524966] scsi host2: storvsc_host_t
[    2.525973] scsi 2:0:0:0: Direct-Access     Msft     Virtual Disk     1.0  PQ: 0 ANSI: 5
[    2.527299] sd 2:0:0:0: Attached scsi generic sg0 type 0
[    2.531966] hv_utils: cannot register PTP clock: 0
[    2.532187] hv_utils: Shutdown IC version 3.0
[    2.532790] hv_utils: TimeSync IC version 4.0
[    2.534166] sd 2:0:0:0: [sda] 83886080 512-byte logical blocks: (42.9 GB/40.0 GiB)
[    2.534244] sd 2:0:0:0: [sda] 4096-byte physical blocks
[    2.534394] sd 2:0:0:0: [sda] Write Protect is off
[    2.534448] sd 2:0:0:0: [sda] Mode Sense: 0f 00 00 00
[    2.534952] sd 2:0:0:0: [sda] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
[    2.552432]  sda: sda1 sda2 < sda5 >
[    2.566464] sd 2:0:0:0: [sda] Attached SCSI disk
[    2.570541] random: fast init done
[    2.582194] scsi host3: storvsc_host_t
[    2.584084] hv_utils: VSS IC version 5.0
[    2.593413] scsi 3:0:0:0: Direct-Access     Msft     Virtual Disk     1.0  PQ: 0 ANSI: 5
[    2.618615] sd 3:0:0:0: Attached scsi generic sg1 type 0
[    2.623955] sd 3:0:0:0: [sdb] 266338304 512-byte logical blocks: (136 GB/127 GiB)
[    2.625517] sd 3:0:0:0: [sdb] 4096-byte physical blocks
[    2.627181] sd 3:0:0:0: [sdb] Write Protect is off
[    2.628748] sd 3:0:0:0: [sdb] Mode Sense: 0f 00 00 00
[    2.630468] sd 3:0:0:0: [sdb] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
[    2.634044]  sdb: sdb1
[    2.643154] sd 3:0:0:0: [sdb] Attached SCSI disk
[    2.704593] systemd-udevd (96) used greatest stack depth: 13392 bytes left
[    2.706902] systemd-udevd (90) used greatest stack depth: 12816 bytes left
[    2.787475] EXT4-fs (sda1): mounted filesystem with ordered data mode. Opts: (null)
[    2.804051] psmouse serio1: trackpoint: failed to get extended button data
[    3.004174] systemd[1]: systemd 229 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ -LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN)
[    3.007863] systemd[1]: Detected virtualization microsoft.
[    3.010207] systemd[1]: Detected architecture x86-64.
[    3.017969] systemd[1]: Set hostname to <decui-u1604>.
[    3.239480] systemd[1]: Listening on Journal Audit Socket.
[    3.240013] systemd[1]: Reached target Encrypted Volumes.
[    3.248937] systemd[1]: Created slice User and Session Slice.
[    3.253336] systemd[1]: Created slice System Slice.
[    3.257596] systemd[1]: Listening on Device-mapper event daemon FIFOs.
[    3.261790] systemd[1]: Listening on LVM2 poll daemon socket.
[    3.731378] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro
[    3.798352] systemd-journald[212]: Received request to flush runtime journal from PID 1
[    4.406082] hv_vmbus: registering driver hv_pci
[    4.518904] piix4_smbus 0000:00:07.3: SMBus base address uninitialized - upgrade BIOS or use force_addr=0xaddr
[    4.599516] ACPI: \: failed to evaluate _DSM (0x1001)
[    4.601025] hv_pci 2d985bf6-6e3e-453e-bca5-7e5f5930df64: PCI host bridge to bus bca5:00
[    4.668864] pci_bus bca5:00: root bus resource [mem 0xfe0000000-0xfe07fffff window]
[    4.732549] pci bca5:00:02.0: [15b3:1004] type 00 class 0x020000
[    4.742880] pci bca5:00:02.0: reg 0x18: [mem 0xfe0000000-0xfe07fffff 64bit pref]
[    4.804954] AVX version of gcm_enc/dec engaged.
[    4.805602] AES CTR mode by8 optimization enabled
[    4.926729] pci bca5:00:02.0: BAR 2: assigned [mem 0xfe0000000-0xfe07fffff 64bit pref]
[    5.055412] alg: No test for pcbc(aes) (pcbc-aes-aesni)
[    5.526597] random: crng init done
[    5.573076] hv_vmbus: registering driver hv_balloon
[    5.666867] hv_balloon: Using Dynamic Memory protocol version 2.0
[    6.182094] IPv6: ADDRCONF(NETDEV_UP): eth0: link is not ready
[    6.208794] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.209447] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.210043] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.210618] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.212272] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.212897] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.213474] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.214051] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.358405] XFS (sdb1): Mounting V5 Filesystem
[    6.404478] XFS (sdb1): Ending clean mount
[    7.535174] BUG: spinlock bad magic on CPU#0, swapper/0/0
[    7.536807]  lock: host_ts+0x30/0xffffffffffffe1a0 [hv_utils], .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0
[    7.538436] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.10.0-rc8-next-20170214+ #1
[    7.539142] Hardware name: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS 090006  04/28/2016
[    7.539142] Call Trace:
[    7.539142]  <IRQ>
[    7.539142]  dump_stack+0x63/0x82
[    7.539142]  spin_dump+0x78/0xc0
[    7.539142]  do_raw_spin_lock+0xfd/0x160
[    7.539142]  _raw_spin_lock_irqsave+0x4c/0x60
[    7.539142]  ? timesync_onchannelcallback+0x153/0x220 [hv_utils]
[    7.539142]  timesync_onchannelcallback+0x153/0x220 [hv_utils]
[    7.539142]  vmbus_on_event+0x101/0x170 [hv_vmbus]
[    7.539142]  tasklet_action+0x118/0x130
[    7.539142]  __do_softirq+0x10a/0x2da
[    7.539142]  irq_exit+0xf3/0x100
[    7.539142]  hyperv_vector_handler+0x39/0x50
[    7.539142]  hyperv_callback_vector+0x93/0xa0
[    7.539142] RIP: 0010:native_safe_halt+0x6/0x10
[    7.539142] RSP: 0018:ffffffff81c03e08 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff0c
[    7.539142] RAX: 0000000000000000 RBX: ffffffff81c14500 RCX: 0000000000000000
[    7.539142] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[    7.539142] RBP: ffffffff81c03e08 R08: 0100000000000000 R09: 0000000000000000
[    7.539142] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000000
[    7.539142] R13: ffffffff81c14500 R14: 0000000000000000 R15: 0000000000000000
[    7.539142]  </IRQ>
[    7.539142]  default_idle+0x1e/0xd0
[    7.539142]  arch_cpu_idle+0xf/0x20
[    7.539142]  default_idle_call+0x23/0x30
[    7.539142]  do_idle+0x179/0x1c0
[    7.539142]  cpu_startup_entry+0x62/0x70
[    7.539142]  rest_init+0xb3/0xc0
[    7.539142]  start_kernel+0x45e/0x46b
[    7.539142]  ? early_idt_handler_array+0x120/0x120
[    7.539142]  x86_64_start_reservations+0x24/0x26
[    7.539142]  x86_64_start_kernel+0x134/0x141
[    7.539142]  start_cpu+0x14/0x14
[    7.704113] psmouse serio1: trackpoint: IBM TrackPoint firmware: 0x01, buttons: 0/0
[    7.711357] input: TPPS/2 IBM TrackPoint as /devices/platform/i8042/serio1/input/input2
[    7.723446] input: AT Translated Set 2 keyboard as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A03:00/device:07/VMBUS:01/d34b2567-b9b6-42b9-8778-0a4ec0b955bf/serio2/input/input4
[   53.756564] hv_balloon: INFO_TYPE_MAX_PAGE_CNT = 262144
[   58.336111] hv_utils: KVP IC version 4.0

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

[-- Attachment #1: Type: text/plain, Size: 2451 bytes --]

> From: hch@lst.de [mailto:hch@lst.de]
> Sent: Tuesday, February 14, 2017 22:51
> To: Dexuan Cui <decui@microsoft.com>
> Cc: hch@lst.de; Jens Axboe <axboe@kernel.dk>; Bart Van Assche
> <Bart.VanAssche@sandisk.com>; hare@suse.com; hare@suse.de; Martin K.
> Petersen <martin.petersen@oracle.com>; linux-kernel@vger.kernel.org;
> linux-block@vger.kernel.org; jth@kernel.org; Nick Meier
> <Nick.Meier@microsoft.com>; Alex Ng (LIS) <alexng@microsoft.com>; Long Li
> <longli@microsoft.com>; Adrian Suhov (Cloudbase Solutions SRL) <v-
> adsuho@microsoft.com>; Chris Valean (Cloudbase Solutions SRL) <v-
> chvale@microsoft.com>
> Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock
> when scheduling workqueue elements")
> 
> On Tue, Feb 14, 2017 at 02:46:41PM +0000, Dexuan Cui wrote:
> > > From: hch@lst.de [mailto:hch@lst.de]
> > > Sent: Tuesday, February 14, 2017 22:29
> > > To: Dexuan Cui <decui@microsoft.com>
> > > Subject: Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event
> lock
> > > when scheduling workqueue elements")
> > >
> > > Ok, thanks for testing.  Can you try the patch below?  It fixes a
> > > clear problem which was partially papered over before the commit
> > > you bisected to, although it can't explain why blk-mq still works.
> >
> > Still bad luck. :-(
> >
> > BTW, I'm using the first "bad" commit (scsi: allocate scsi_cmnd structures
> as
> > part of struct request) + the 2 patches you provided today.
> >
> > I suppose I don't need to test the 2 patches on the latest linux-next repo.
> 
> I'd love a test on that repo actually.  We had a few other for sense
> handling since then I think.

I tested today's linux-next (next-20170214) + the 2 patches just now and got
a weird result: 
sometimes the VM stills hung with a new calltrace (BUG: spinlock bad
magic) , but sometimes the VM did boot up despite the new calltrace!

Attached is the log of a "good" boot.

It looks we have a memory corruption issue somewhere...

Actually previously I saw the "BUG: spinlock bad magic" message once, but I
couldn't repro it later, so I didn't mention it to you.

The good news is that now I can repro the "spinlock bad magic" message
every time. 
I tried to dig into this by enabling Kernel hacking -> Memory debugging,
but didn't find anything abnormal. 
Is it possible that the SCSI layer passes a wrong memory address?

Thanks,
-- Dexuan

[-- Attachment #2: dmesg.log --]
[-- Type: application/octet-stream, Size: 31337 bytes --]

[    0.000000] Linux version 4.10.0-rc8-next-20170214+ (root@decui-u1604) (gcc version 5.3.1 20160413 (Ubuntu 5.3.1-14ubuntu2.1) ) #1 SMP Tue Feb 14 23:10:48 CST 2017
[    0.000000] Command line: BOOT_IMAGE=/boot/hv/bzImage root=UUID=f07ac67a-0109-4738-a388-100218d2c5d2 ro ignore_loglevel scsi_mod.use_blk_mq=N
[    0.000000] KERNEL supported cpus:
[    0.000000]   Intel GenuineIntel
[    0.000000] x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers'
[    0.000000] x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers'
[    0.000000] x86/fpu: xstate_offset[2]:  576, xstate_sizes[2]:  256
[    0.000000] x86/fpu: Enabled xstate features 0x7, context size is 832 bytes, using 'standard' format.
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009fbff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000e0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000003ffeffff] usable
[    0.000000] BIOS-e820: [mem 0x000000003fff0000-0x000000003fffefff] ACPI data
[    0.000000] BIOS-e820: [mem 0x000000003ffff000-0x000000003fffffff] ACPI NVS
[    0.000000] debug: ignoring loglevel setting.
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 2.3 present.
[    0.000000] DMI: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS 090006  04/28/2016
[    0.000000] Hypervisor detected: Microsoft HyperV
[    0.000000] HyperV: features 0x2e7f, hints 0xc2c
[    0.000000] Hyper-V Host Build:14393-10.0-0-0.576
[    0.000000] HyperV: LAPIC Timer Frequency: 0xc3500
[    0.000000] tsc: Marking TSC unstable due to running on Hyper-V
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x3fff0 max_arch_pfn = 0x400000000
[    0.000000] MTRR default type: uncachable
[    0.000000] MTRR fixed ranges enabled:
[    0.000000]   00000-9FFFF write-back
[    0.000000]   A0000-DFFFF uncachable
[    0.000000]   E0000-FFFFF write-back
[    0.000000] MTRR variable ranges enabled:
[    0.000000]   0 base 00000000000 mask FFF00000000 write-back
[    0.000000]   1 disabled
[    0.000000]   2 disabled
[    0.000000]   3 disabled
[    0.000000]   4 disabled
[    0.000000]   5 disabled
[    0.000000]   6 disabled
[    0.000000]   7 disabled
[    0.000000] x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WC  UC- WT  
[    0.000000] found SMP MP-table at [mem 0x000ff780-0x000ff78f] mapped at [ffff9539400ff780]
[    0.000000] Scanning 1 areas for low memory corruption
[    0.000000] Base memory trampoline at [ffff953940099000] 99000 size 24576
[    0.000000] BRK [0x0e91c000, 0x0e91cfff] PGTABLE
[    0.000000] BRK [0x0e91d000, 0x0e91dfff] PGTABLE
[    0.000000] BRK [0x0e91e000, 0x0e91efff] PGTABLE
[    0.000000] BRK [0x0e91f000, 0x0e91ffff] PGTABLE
[    0.000000] BRK [0x0e920000, 0x0e920fff] PGTABLE
[    0.000000] RAMDISK: [mem 0x2fde8000-0x33eebfff]
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000000F56F0 000014 (v00 ACPIAM)
[    0.000000] ACPI: RSDT 0x000000003FFF0000 000040 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: FACP 0x000000003FFF0200 000081 (v02 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: DSDT 0x000000003FFF1724 0033BE (v01 MSFTVM MSFTVM02 00000002 INTL 02002026)
[    0.000000] ACPI: FACS 0x000000003FFFF000 000040
[    0.000000] ACPI: WAET 0x000000003FFF1480 000028 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: SLIC 0x000000003FFF14C0 000176 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: OEM0 0x000000003FFF16C0 000064 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: SRAT 0x000000003FFF0600 0000E0 (v02 VRTUAL MICROSFT 00000001 MSFT 00000001)
[    0.000000] ACPI: APIC 0x000000003FFF0300 000252 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: OEMB 0x000000003FFFF040 000064 (v01 VRTUAL MICROSFT 04001628 MSFT 00000097)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] SRAT: PXM 0 -> APIC 0x00 -> Node 0
[    0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x00000000-0x3fffffff] hotplug
[    0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x40200000-0xf7ffffff] hotplug
[    0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x100000000-0xfdfffffff] hotplug
[    0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x1000000000-0xffffffffff] hotplug
[    0.000000] NODE_DATA(0) allocated [mem 0x3ffec000-0x3ffeffff]
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.000000]   DMA32    [mem 0x0000000001000000-0x000000003ffeffff]
[    0.000000]   Normal   empty
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x0000000000001000-0x000000000009efff]
[    0.000000]   node   0: [mem 0x0000000000100000-0x000000003ffeffff]
[    0.000000] Initmem setup node 0 [mem 0x0000000000001000-0x000000003ffeffff]
[    0.000000] On node 0 totalpages: 262030
[    0.000000]   DMA zone: 64 pages used for memmap
[    0.000000]   DMA zone: 21 pages reserved
[    0.000000]   DMA zone: 3998 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 4032 pages used for memmap
[    0.000000]   DMA32 zone: 258032 pages, LIFO batch:31
[    0.000000] ACPI: PM-Timer IO Port: 0x408
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
[    0.000000] IOAPIC[0]: apic_id 0, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] smpboot: Allowing 64 CPUs, 63 hotplug CPUs
[    0.000000] e820: [mem 0x40000000-0xffffffff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on bare hardware
[    0.000000] clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645519600211568 ns
[    0.000000] setup_percpu: NR_CPUS:256 nr_cpumask_bits:256 nr_cpu_ids:64 nr_node_ids:1
[    0.000000] percpu: Embedded 37 pages/cpu @ffff95397d600000 s112328 r8192 d31032 u262144
[    0.000000] pcpu-alloc: s112328 r8192 d31032 u262144 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 00 01 02 03 04 05 06 07 [0] 08 09 10 11 12 13 14 15 
[    0.000000] pcpu-alloc: [0] 16 17 18 19 20 21 22 23 [0] 24 25 26 27 28 29 30 31 
[    0.000000] pcpu-alloc: [0] 32 33 34 35 36 37 38 39 [0] 40 41 42 43 44 45 46 47 
[    0.000000] pcpu-alloc: [0] 48 49 50 51 52 53 54 55 [0] 56 57 58 59 60 61 62 63 
[    0.000000] Built 1 zonelists in Node order, mobility grouping on.  Total pages: 257913
[    0.000000] Policy zone: DMA32
[    0.000000] Kernel command line: BOOT_IMAGE=/boot/hv/bzImage root=UUID=f07ac67a-0109-4738-a388-100218d2c5d2 ro ignore_loglevel scsi_mod.use_blk_mq=N
[    0.000000] log_buf_len individual max cpu contribution: 4096 bytes
[    0.000000] log_buf_len total cpu_extra contributions: 258048 bytes
[    0.000000] log_buf_len min size: 262144 bytes
[    0.000000] log_buf_len: 524288 bytes
[    0.000000] early log buf free: 254580(97%)
[    0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    0.000000] Memory: 863304K/1048120K available (6896K kernel code, 1440K rwdata, 2784K rodata, 1304K init, 10644K bss, 184816K reserved, 0K cma-reserved)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=64, Nodes=1
[    0.000000] Hierarchical RCU implementation.
[    0.000000] 	Build-time adjustment of leaf fanout to 64.
[    0.000000] 	RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=64.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=64
[    0.000000] NR_IRQS:16640 nr_irqs:936 16
[    0.000000] 	Offload RCU callbacks from all CPUs
[    0.000000] 	Offload RCU callbacks from CPUs: 0-63.
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [tty0] enabled
[    0.000000] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.000000] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.000000] ... MAX_LOCK_DEPTH:          48
[    0.000000] ... MAX_LOCKDEP_KEYS:        8191
[    0.000000] ... CLASSHASH_SIZE:          4096
[    0.000000] ... MAX_LOCKDEP_ENTRIES:     32768
[    0.000000] ... MAX_LOCKDEP_CHAINS:      65536
[    0.000000] ... CHAINHASH_SIZE:          32768
[    0.000000]  memory used by lock dependency info: 8127 kB
[    0.000000]  per task-struct memory footprint: 1920 bytes
[    0.000000] tsc: Fast TSC calibration failed
[    0.012000] tsc: Unable to calibrate against PIT
[    0.016000] tsc: using PMTIMER reference calibration
[    0.016000] tsc: Detected 2600.008 MHz processor
[    0.016000] Calibrating delay loop (skipped), value calculated using timer frequency.. 5200.01 BogoMIPS (lpj=10400032)
[    0.016000] pid_max: default: 65536 minimum: 512
[    0.020112] ACPI: Core revision 20170119
[    0.025555] ACPI: 1 ACPI AML tables successfully acquired and loaded
[    0.026775] Dentry cache hash table entries: 131072 (order: 8, 1048576 bytes)
[    0.028419] Inode-cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.030051] Mount-cache hash table entries: 2048 (order: 2, 16384 bytes)
[    0.030981] Mountpoint-cache hash table entries: 2048 (order: 2, 16384 bytes)
[    0.032912] CPU: Physical Processor ID: 0
[    0.033859] mce: CPU supports 1 MCE banks
[    0.034848] Last level iTLB entries: 4KB 512, 2MB 8, 4MB 8
[    0.036046] Last level dTLB entries: 4KB 512, 2MB 0, 4MB 0, 1GB 4
[    0.054815] ftrace: allocating 24941 entries in 98 pages
[    0.069829] smpboot: Max logical packages: 64
[    0.072084] Switched APIC routing to physical flat.
[    0.072963] clocksource: hyperv_clocksource_tsc_page: mask: 0xffffffffffffffff max_cycles: 0x24e6a1710, max_idle_ns: 440795202120 ns
[    0.111150] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.112225] smpboot: CPU0: Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz (family: 0x6, model: 0x3e, stepping: 0x4)
[    0.113384] Performance Events: unsupported p6 CPU model 62 no PMU driver, software events only.
[    0.116871] NMI watchdog: disabled (cpu0): hardware events not enabled
[    0.117812] NMI watchdog: Shutting down hard lockup detector on all cpus
[    0.118714] smp: Bringing up secondary CPUs ...
[    0.119568] smp: Brought up 1 node, 1 CPU
[    0.120023] smpboot: Total of 1 processors activated (5200.01 BogoMIPS)
[    0.129001] devtmpfs: initialized
[    0.130063] x86/mm: Memory block size: 128MB
[    0.136138] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
[    0.137510] pinctrl core: initialized pinctrl subsystem
[    0.138840] NET: Registered protocol family 16
[    0.144098] cpuidle: using governor ladder
[    0.148010] cpuidle: using governor menu
[    0.148845] PCCT header not found.
[    0.149763] ACPI: bus type PCI registered
[    0.152433] PCI: Using configuration type 1 for base access
[    0.158484] HugeTLB registered 1 GB page size, pre-allocated 0 pages
[    0.160011] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    0.162411] ACPI: Added _OSI(Module Device)
[    0.164009] ACPI: Added _OSI(Processor Device)
[    0.164843] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.165672] ACPI: Added _OSI(Processor Aggregator Device)
[    0.176058] ACPI: Interpreter enabled
[    0.176988] ACPI: (supports S0 S5)
[    0.177811] ACPI: Using IOAPIC for interrupt routing
[    0.178691] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[    0.205665] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.206736] acpi PNP0A03:00: _OSC: OS supports [Segments MSI]
[    0.207608] acpi PNP0A03:00: _OSC failed (AE_NOT_FOUND); disabling ASPM
[    0.208181] PCI host bridge to bus 0000:00
[    0.209050] pci_bus 0000:00: root bus resource [mem 0xfe0000000-0xfffffffff window]
[    0.209979] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7 window]
[    0.210833] pci_bus 0000:00: root bus resource [io  0x0d00-0xffff window]
[    0.212009] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window]
[    0.212945] pci_bus 0000:00: root bus resource [mem 0xf8000000-0xfffbffff window]
[    0.213875] pci_bus 0000:00: root bus resource [bus 00-ff]
[    0.215031] pci 0000:00:00.0: [8086:7192] type 00 class 0x060000
[    0.220504] pci 0000:00:07.0: [8086:7110] type 00 class 0x060100
[    0.225876] pci 0000:00:07.1: [8086:7111] type 00 class 0x010180
[    0.230354] pci 0000:00:07.1: reg 0x20: [io  0xffa0-0xffaf]
[    0.232661] pci 0000:00:07.1: legacy IDE quirk: reg 0x10: [io  0x01f0-0x01f7]
[    0.234841] pci 0000:00:07.1: legacy IDE quirk: reg 0x14: [io  0x03f6]
[    0.235712] pci 0000:00:07.1: legacy IDE quirk: reg 0x18: [io  0x0170-0x0177]
[    0.236010] pci 0000:00:07.1: legacy IDE quirk: reg 0x1c: [io  0x0376]
[    0.239084] pci 0000:00:07.3: [8086:7113] type 00 class 0x068000
[    0.240088] * Found PM-Timer Bug on the chipset. Due to workarounds for a bug,
               * this clock source is slow. Consider trying other clock sources
[    0.245925] pci 0000:00:07.3: quirk: [io  0x0400-0x043f] claimed by PIIX4 ACPI
[    0.248454] pci 0000:00:08.0: [1414:5353] type 00 class 0x030000
[    0.250097] pci 0000:00:08.0: reg 0x10: [mem 0xf8000000-0xfbffffff]
[    0.269769] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 7 9 10 *11 12 14 15)
[    0.271040] ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    0.272376] ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    0.273715] ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 5 7 9 10 11 12 14 15) *0, disabled.
[    0.274964] ACPI: Enabled 1 GPEs in block 00 to 0F
[    0.277122] pci 0000:00:08.0: vgaarb: setting as boot VGA device
[    0.277911] pci 0000:00:08.0: vgaarb: VGA device added: decodes=io+mem,owns=io+mem,locks=none
[    0.278775] pci 0000:00:08.0: vgaarb: bridge control possible
[    0.279559] vgaarb: loaded
[    0.280170] SCSI subsystem initialized
[    0.281217] libata version 3.00 loaded.
[    0.284051] PCI: Using ACPI for IRQ routing
[    0.286144] PCI: pci_cache_line_size set to 64 bytes
[    0.287653] e820: reserve RAM buffer [mem 0x0009fc00-0x0009ffff]
[    0.288023] e820: reserve RAM buffer [mem 0x3fff0000-0x3fffffff]
[    0.289406] clocksource: Switched to clocksource hyperv_clocksource_tsc_page
[    0.342146] VFS: Disk quotas dquot_6.6.0
[    0.343311] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    0.345144] pnp: PnP ACPI init
[    0.346201] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[    0.347171] pnp 00:01: Plug and Play ACPI device, IDs PNP0303 PNP030b (active)
[    0.348231] pnp 00:02: Plug and Play ACPI device, IDs PNP0f03 PNP0f13 (active)
[    0.350505] pnp 00:03: [dma 0 disabled]
[    0.351389] pnp 00:03: Plug and Play ACPI device, IDs PNP0501 (active)
[    0.353580] pnp 00:04: [dma 0 disabled]
[    0.354429] pnp 00:04: Plug and Play ACPI device, IDs PNP0501 (active)
[    0.356734] pnp 00:05: [dma 2]
[    0.357670] pnp 00:05: Plug and Play ACPI device, IDs PNP0700 (active)
[    0.358706] system 00:06: [io  0x01e0-0x01ef] has been reserved
[    0.359537] system 00:06: [io  0x0160-0x016f] has been reserved
[    0.360393] system 00:06: [io  0x0278-0x027f] has been reserved
[    0.361216] system 00:06: [io  0x0378-0x037f] has been reserved
[    0.362032] system 00:06: [io  0x0678-0x067f] has been reserved
[    0.362848] system 00:06: [io  0x0778-0x077f] has been reserved
[    0.363661] system 00:06: [io  0x04d0-0x04d1] has been reserved
[    0.364519] system 00:06: Plug and Play ACPI device, IDs PNP0c02 (active)
[    0.365670] system 00:07: [io  0x0400-0x043f] has been reserved
[    0.366579] system 00:07: [io  0x0370-0x0371] has been reserved
[    0.367403] system 00:07: [io  0x0440-0x044f] has been reserved
[    0.368261] system 00:07: [mem 0xfec00000-0xfec00fff] could not be reserved
[    0.369087] system 00:07: [mem 0xfee00000-0xfee00fff] has been reserved
[    0.369875] system 00:07: Plug and Play ACPI device, IDs PNP0c02 (active)
[    0.370930] system 00:08: [mem 0x00000000-0x0009ffff] could not be reserved
[    0.371843] system 00:08: [mem 0x000c0000-0x000dffff] could not be reserved
[    0.374012] system 00:08: [mem 0x000e0000-0x000fffff] could not be reserved
[    0.374858] system 00:08: [mem 0x00100000-0xf7ffffff] could not be reserved
[    0.375703] system 00:08: [mem 0xfffc0000-0xffffffff] has been reserved
[    0.376579] system 00:08: Plug and Play ACPI device, IDs PNP0c01 (active)
[    0.379161] pnp: PnP ACPI: found 9 devices
[    0.387621] clocksource: acpi_pm: mask: 0xffffff max_cycles: 0xffffff, max_idle_ns: 2085701024 ns
[    0.388618] pci_bus 0000:00: resource 4 [mem 0xfe0000000-0xfffffffff window]
[    0.389456] pci_bus 0000:00: resource 5 [io  0x0000-0x0cf7 window]
[    0.390281] pci_bus 0000:00: resource 6 [io  0x0d00-0xffff window]
[    0.391105] pci_bus 0000:00: resource 7 [mem 0x000a0000-0x000bffff window]
[    0.391931] pci_bus 0000:00: resource 8 [mem 0xf8000000-0xfffbffff window]
[    0.392959] NET: Registered protocol family 2
[    0.394453] TCP established hash table entries: 8192 (order: 4, 65536 bytes)
[    0.395457] TCP bind hash table entries: 8192 (order: 7, 524288 bytes)
[    0.396936] TCP: Hash tables configured (established 8192 bind 8192)
[    0.398239] UDP hash table entries: 512 (order: 4, 81920 bytes)
[    0.399295] UDP-Lite hash table entries: 512 (order: 4, 81920 bytes)
[    0.400881] NET: Registered protocol family 1
[    0.401792] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    0.403210] pci 0000:00:08.0: Video device with shadowed ROM at [mem 0x000c0000-0x000dffff]
[    0.404276] PCI: CLS 0 bytes, default 64
[    0.405282] Unpacking initramfs...
[    1.706148] Freeing initrd memory: 66576K
[    1.706919] PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
[    1.707234] software IO TLB [mem 0x39600000-0x3d600000] (64MB) mapped at [ffff953979600000-ffff95397d5fffff]
[    1.707690] RAPL PMU: API unit is 2^-32 Joules, 3 fixed counters, 10737418240 ms ovfl timer
[    1.708045] RAPL PMU: hw unit of domain pp0-core 2^-0 Joules
[    1.708358] RAPL PMU: hw unit of domain package 2^-0 Joules
[    1.708669] RAPL PMU: hw unit of domain dram 2^-0 Joules
[    1.709048] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x257a4365b0e, max_idle_ns: 440795224253 ns
[    1.709427] Scanning for low memory corruption every 60 seconds
[    1.710395] futex hash table entries: 16384 (order: 9, 2097152 bytes)
[    1.711254] audit: initializing netlink subsys (disabled)
[    1.711684] cryptomgr_test (31) used greatest stack depth: 14528 bytes left
[    1.712237] Initialise system trusted keyrings
[    1.712598] audit: type=2000 audit(1487085689.712:1): state=initialized audit_enabled=0 res=1
[    1.713063] workingset: timestamp_bits=40 max_order=18 bucket_order=0
[    1.714737] zbud: loaded
[    1.715782] SGI XFS with security attributes, no debug enabled
[    1.718151] Key type asymmetric registered
[    1.718518] Asymmetric key parser 'x509' registered
[    1.718887] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 251)
[    1.719379] io scheduler noop registered
[    1.719720] io scheduler deadline registered (default)
[    1.720518] io scheduler cfq registered
[    1.720877] io scheduler mq-deadline registered
[    1.721393] GHES: HEST is not enabled!
[    1.721770] Serial: 8250/16550 driver, 32 ports, IRQ sharing enabled
[    1.769785] 00:03: ttyS0 at I/O 0x3f8 (irq = 4, base_baud = 115200) is a 16550A
[    1.818962] 00:04: ttyS1 at I/O 0x2f8 (irq = 3, base_baud = 115200) is a 16550A
[    1.820459] Linux agpgart interface v0.103
[    1.820970] ata_piix 0000:00:07.1: version 2.13
[    1.821431] ata_piix 0000:00:07.1: Hyper-V Virtual Machine detected, ATA device ignore set
[    1.823794] scsi host0: ata_piix
[    1.824253] scsi host1: ata_piix
[    1.824589] ata1: PATA max UDMA/33 cmd 0x1f0 ctl 0x3f6 bmdma 0xffa0 irq 14
[    1.824922] ata2: PATA max UDMA/33 cmd 0x170 ctl 0x376 bmdma 0xffa8 irq 15
[    1.825372] i8042: PNP: PS/2 Controller [PNP0303:PS2K,PNP0f03:PS2M] at 0x60,0x64 irq 1,12
[    1.828787] serio: i8042 KBD port at 0x60,0x64 irq 1
[    1.829127] serio: i8042 AUX port at 0x60,0x64 irq 12
[    1.830457] rtc_cmos 00:00: RTC can wake from S4
[    1.852390] rtc_cmos 00:00: rtc core: registered rtc_cmos as rtc0
[    1.853241] rtc_cmos 00:00: alarms up to one month, 114 bytes nvram
[    1.853629] ledtrig-cpu: registered to indicate activity on CPUs
[    1.854189] NET: Registered protocol family 10
[    1.855119] Segment Routing with IPv6
[    1.855495] NET: Registered protocol family 17
[    1.855813] Key type dns_resolver registered
[    1.856411] registered taskstats version 1
[    1.857175] Loading compiled-in X.509 certificates
[    1.857529] zswap: loaded using pool lzo/zbud
[    1.858415] rtc_cmos 00:00: setting system clock to 2017-02-14 15:21:30 UTC (1487085690)
[    1.858766] BIOS EDD facility v0.16 2004-Jun-25, 0 devices found
[    1.859087] EDD information not available.
[    2.093148] ata1.01: NODEV after polling detection
[    2.143359] ata1.00: host indicates ignore ATA devices, ignored
[    2.144884] Freeing unused kernel memory: 1304K
[    2.145283] Write protecting the kernel read-only data: 12288k
[    2.146252] Freeing unused kernel memory: 1280K
[    2.148804] Freeing unused kernel memory: 1312K
[    2.150963] mount (67) used greatest stack depth: 14512 bytes left
[    2.152646] exe (70) used greatest stack depth: 14504 bytes left
[    2.155475] exe (73) used greatest stack depth: 14064 bytes left
[    2.163564] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.163978] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.164923] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    2.165384] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    2.165716] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    2.166047] random: systemd-udevd: uninitialized urandom read (16 bytes read)
[    2.171533] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.171994] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.172758] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.173192] random: udevadm: uninitialized urandom read (16 bytes read)
[    2.235888] udevadm (84) used greatest stack depth: 13904 bytes left
[    2.279298] hv_vmbus: Vmbus version:4.0
[    2.291556] hv_vmbus: registering driver hyperv_fb
[    2.301275] hyperv_fb: Screen resolution: 1152x864, Color depth: 32
[    2.310220] Console: switching to colour frame buffer device 144x54
[    2.342210] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input0
[    2.454348] hidraw: raw HID events driver (C) Jiri Kosina
[    2.455357] hv_vmbus: registering driver hid_hyperv
[    2.456632] hv_vmbus: registering driver hyperv_keyboard
[    2.459289] hv_utils: Registering HyperV Utility Driver
[    2.459346] hv_vmbus: registering driver hv_util
[    2.468340] hv_vmbus: registering driver hv_netvsc
[    2.472734] hv_vmbus: registering driver hv_storvsc
[    2.477142] input: Microsoft Vmbus HID-compliant Mouse as /devices/0006:045E:0621.0001/input/input3
[    2.477250] hid 0006:045E:0621.0001: input: <UNKNOWN> HID v0.01 Mouse [Microsoft Vmbus HID-compliant Mouse] on 
[    2.478576] hv_utils: Heartbeat IC version 3.0
[    2.524966] scsi host2: storvsc_host_t
[    2.525973] scsi 2:0:0:0: Direct-Access     Msft     Virtual Disk     1.0  PQ: 0 ANSI: 5
[    2.527299] sd 2:0:0:0: Attached scsi generic sg0 type 0
[    2.531966] hv_utils: cannot register PTP clock: 0
[    2.532187] hv_utils: Shutdown IC version 3.0
[    2.532790] hv_utils: TimeSync IC version 4.0
[    2.534166] sd 2:0:0:0: [sda] 83886080 512-byte logical blocks: (42.9 GB/40.0 GiB)
[    2.534244] sd 2:0:0:0: [sda] 4096-byte physical blocks
[    2.534394] sd 2:0:0:0: [sda] Write Protect is off
[    2.534448] sd 2:0:0:0: [sda] Mode Sense: 0f 00 00 00
[    2.534952] sd 2:0:0:0: [sda] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
[    2.552432]  sda: sda1 sda2 < sda5 >
[    2.566464] sd 2:0:0:0: [sda] Attached SCSI disk
[    2.570541] random: fast init done
[    2.582194] scsi host3: storvsc_host_t
[    2.584084] hv_utils: VSS IC version 5.0
[    2.593413] scsi 3:0:0:0: Direct-Access     Msft     Virtual Disk     1.0  PQ: 0 ANSI: 5
[    2.618615] sd 3:0:0:0: Attached scsi generic sg1 type 0
[    2.623955] sd 3:0:0:0: [sdb] 266338304 512-byte logical blocks: (136 GB/127 GiB)
[    2.625517] sd 3:0:0:0: [sdb] 4096-byte physical blocks
[    2.627181] sd 3:0:0:0: [sdb] Write Protect is off
[    2.628748] sd 3:0:0:0: [sdb] Mode Sense: 0f 00 00 00
[    2.630468] sd 3:0:0:0: [sdb] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
[    2.634044]  sdb: sdb1
[    2.643154] sd 3:0:0:0: [sdb] Attached SCSI disk
[    2.704593] systemd-udevd (96) used greatest stack depth: 13392 bytes left
[    2.706902] systemd-udevd (90) used greatest stack depth: 12816 bytes left
[    2.787475] EXT4-fs (sda1): mounted filesystem with ordered data mode. Opts: (null)
[    2.804051] psmouse serio1: trackpoint: failed to get extended button data
[    3.004174] systemd[1]: systemd 229 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ -LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN)
[    3.007863] systemd[1]: Detected virtualization microsoft.
[    3.010207] systemd[1]: Detected architecture x86-64.
[    3.017969] systemd[1]: Set hostname to <decui-u1604>.
[    3.239480] systemd[1]: Listening on Journal Audit Socket.
[    3.240013] systemd[1]: Reached target Encrypted Volumes.
[    3.248937] systemd[1]: Created slice User and Session Slice.
[    3.253336] systemd[1]: Created slice System Slice.
[    3.257596] systemd[1]: Listening on Device-mapper event daemon FIFOs.
[    3.261790] systemd[1]: Listening on LVM2 poll daemon socket.
[    3.731378] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro
[    3.798352] systemd-journald[212]: Received request to flush runtime journal from PID 1
[    4.406082] hv_vmbus: registering driver hv_pci
[    4.518904] piix4_smbus 0000:00:07.3: SMBus base address uninitialized - upgrade BIOS or use force_addr=0xaddr
[    4.599516] ACPI: \: failed to evaluate _DSM (0x1001)
[    4.601025] hv_pci 2d985bf6-6e3e-453e-bca5-7e5f5930df64: PCI host bridge to bus bca5:00
[    4.668864] pci_bus bca5:00: root bus resource [mem 0xfe0000000-0xfe07fffff window]
[    4.732549] pci bca5:00:02.0: [15b3:1004] type 00 class 0x020000
[    4.742880] pci bca5:00:02.0: reg 0x18: [mem 0xfe0000000-0xfe07fffff 64bit pref]
[    4.804954] AVX version of gcm_enc/dec engaged.
[    4.805602] AES CTR mode by8 optimization enabled
[    4.926729] pci bca5:00:02.0: BAR 2: assigned [mem 0xfe0000000-0xfe07fffff 64bit pref]
[    5.055412] alg: No test for pcbc(aes) (pcbc-aes-aesni)
[    5.526597] random: crng init done
[    5.573076] hv_vmbus: registering driver hv_balloon
[    5.666867] hv_balloon: Using Dynamic Memory protocol version 2.0
[    6.182094] IPv6: ADDRCONF(NETDEV_UP): eth0: link is not ready
[    6.208794] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.209447] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.210043] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.210618] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.212272] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.212897] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.213474] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.214051] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.358405] XFS (sdb1): Mounting V5 Filesystem
[    6.404478] XFS (sdb1): Ending clean mount
[    7.535174] BUG: spinlock bad magic on CPU#0, swapper/0/0
[    7.536807]  lock: host_ts+0x30/0xffffffffffffe1a0 [hv_utils], .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0
[    7.538436] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.10.0-rc8-next-20170214+ #1
[    7.539142] Hardware name: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS 090006  04/28/2016
[    7.539142] Call Trace:
[    7.539142]  <IRQ>
[    7.539142]  dump_stack+0x63/0x82
[    7.539142]  spin_dump+0x78/0xc0
[    7.539142]  do_raw_spin_lock+0xfd/0x160
[    7.539142]  _raw_spin_lock_irqsave+0x4c/0x60
[    7.539142]  ? timesync_onchannelcallback+0x153/0x220 [hv_utils]
[    7.539142]  timesync_onchannelcallback+0x153/0x220 [hv_utils]
[    7.539142]  vmbus_on_event+0x101/0x170 [hv_vmbus]
[    7.539142]  tasklet_action+0x118/0x130
[    7.539142]  __do_softirq+0x10a/0x2da
[    7.539142]  irq_exit+0xf3/0x100
[    7.539142]  hyperv_vector_handler+0x39/0x50
[    7.539142]  hyperv_callback_vector+0x93/0xa0
[    7.539142] RIP: 0010:native_safe_halt+0x6/0x10
[    7.539142] RSP: 0018:ffffffff81c03e08 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff0c
[    7.539142] RAX: 0000000000000000 RBX: ffffffff81c14500 RCX: 0000000000000000
[    7.539142] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[    7.539142] RBP: ffffffff81c03e08 R08: 0100000000000000 R09: 0000000000000000
[    7.539142] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000000
[    7.539142] R13: ffffffff81c14500 R14: 0000000000000000 R15: 0000000000000000
[    7.539142]  </IRQ>
[    7.539142]  default_idle+0x1e/0xd0
[    7.539142]  arch_cpu_idle+0xf/0x20
[    7.539142]  default_idle_call+0x23/0x30
[    7.539142]  do_idle+0x179/0x1c0
[    7.539142]  cpu_startup_entry+0x62/0x70
[    7.539142]  rest_init+0xb3/0xc0
[    7.539142]  start_kernel+0x45e/0x46b
[    7.539142]  ? early_idt_handler_array+0x120/0x120
[    7.539142]  x86_64_start_reservations+0x24/0x26
[    7.539142]  x86_64_start_kernel+0x134/0x141
[    7.539142]  start_cpu+0x14/0x14
[    7.704113] psmouse serio1: trackpoint: IBM TrackPoint firmware: 0x01, buttons: 0/0
[    7.711357] input: TPPS/2 IBM TrackPoint as /devices/platform/i8042/serio1/input/input2
[    7.723446] input: AT Translated Set 2 keyboard as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A03:00/device:07/VMBUS:01/d34b2567-b9b6-42b9-8778-0a4ec0b955bf/serio2/input/input4
[   53.756564] hv_balloon: INFO_TYPE_MAX_PAGE_CNT = 262144
[   58.336111] hv_utils: KVP IC version 4.0

Re: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[hch]

> I tested today's linux-next (next-20170214) + the 2 patches just now and got
> a weird result: 
> sometimes the VM stills hung with a new calltrace (BUG: spinlock bad
> magic) , but sometimes the VM did boot up despite the new calltrace!
> 
> Attached is the log of a "good" boot.
> 
> It looks we have a memory corruption issue somewhere...

Yes.

> Actually previously I saw the "BUG: spinlock bad magic" message once, but I
> couldn't repro it later, so I didn't mention it to you.

Interesting.

> 
> The good news is that now I can repro the "spinlock bad magic" message
> every time. 
> I tried to dig into this by enabling Kernel hacking -> Memory debugging,
> but didn't find anything abnormal. 
> Is it possible that the SCSI layer passes a wrong memory address?

It's possible, but this looks like it might be a different issue.

A few questions on the dmesg:

[    6.208794] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.209447] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.210043] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.210618] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.212272] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.212897] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operation code
[    6.213474] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current] 
[    6.214051] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operation code

I didn't see anything like this in the other logs.  Are these messages
something usual on HyperV VMs?

[    6.358405] XFS (sdb1): Mounting V5 Filesystem
[    6.404478] XFS (sdb1): Ending clean mount
[    7.535174] BUG: spinlock bad magic on CPU#0, swapper/0/0
[    7.536807]  lock: host_ts+0x30/0xffffffffffffe1a0 [hv_utils], .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0
[    7.538436] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.10.0-rc8-next-20170214+ #1
[    7.539142] Hardware name: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS 090006  04/28/2016
[    7.539142] Call Trace:
[    7.539142]  <IRQ>
[    7.539142]  dump_stack+0x63/0x82
[    7.539142]  spin_dump+0x78/0xc0
[    7.539142]  do_raw_spin_lock+0xfd/0x160
[    7.539142]  _raw_spin_lock_irqsave+0x4c/0x60
[    7.539142]  ? timesync_onchannelcallback+0x153/0x220 [hv_utils]
[    7.539142]  timesync_onchannelcallback+0x153/0x220 [hv_utils]

Can you resolve this address using gdb to a line of code?  Once inside
gdb do:

l *(timesync_onchannelcallback+0x153)

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

> From: hch@lst.de [mailto:hch@lst.de]
> Sent: Wednesday, February 15, 2017 00:35
> > I tested today's linux-next (next-20170214) + the 2 patches just now an=
d
> got
> > a weird result:
> > sometimes the VM stills hung with a new calltrace (BUG: spinlock bad
> > magic) , but sometimes the VM did boot up despite the new calltrace!
> >
> > Attached is the log of a "good" boot.
> >
> > It looks we have a memory corruption issue somewhere...
>
> Yes.
It's due to an uninitialized spinlock. Please see the below.

> > Actually previously I saw the "BUG: spinlock bad magic" message once, b=
ut
> I
> > couldn't repro it later, so I didn't mention it to you.
>
> Interesting.
Ditto.
And probably my memory was inaccurate due to the long period of bisecting.
We should always see the message.

>
> A few questions on the dmesg:
>
> [    6.208794] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current=
]
> [    6.209447] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operatio=
n
> code
> [    6.210043] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current=
]
> [    6.210618] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operatio=
n
> code
> [    6.212272] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current=
]
> [    6.212897] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operatio=
n
> code
> [    6.213474] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current=
]
> [    6.214051] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operatio=
n
> code
>
> I didn't see anything like this in the other logs.  Are these messages
> something usual on HyperV VMs?

The messages should be normal, and I have always seen them for years
at least.

I think they're triggered by "/sbin/hdparm /dev/sda" or
"/lib/udev/ata_id /dev/sda"  in my Ubuntu 16.04 VM: during the VM
boot-up, the 2 programs are executed by the initrd's systemd-udev or other
system init scripts and the 2 programs try to run these 2 SCSI commands,
which are not supported by Hyper-V:

ATA PASS THROUGH (12) -- 0xa1
ATA PASS THROUGH (16) -- 0x85

IMO the commands are used when PATA/SATA devices are connected to
SCSI-to-ATA bridge, so it's understandable that Hyper-V doesn't support
them.

In the case of boot failure (hang), the 2 programs don't have a chance to
run, so we can't see the messages.

IMO we should ignore the messages, which should be harmless.

> [    6.358405] XFS (sdb1): Mounting V5 Filesystem
> [    6.404478] XFS (sdb1): Ending clean mount
> [    7.535174] BUG: spinlock bad magic on CPU#0, swapper/0/0
> [    7.536807]  lock: host_ts+0x30/0xffffffffffffe1a0 [hv_utils], .magic:
> 00000000, .owner: <none>/-1, .owner_cpu: 0
> [    7.538436] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.10.0-rc8-next-
> 20170214+ #1
> [    7.539142] Hardware name: Microsoft Corporation Virtual Machine/Virtu=
al
> Machine, BIOS 090006  04/28/2016
> [    7.539142] Call Trace:
> [    7.539142]  <IRQ>
> [    7.539142]  dump_stack+0x63/0x82
> [    7.539142]  spin_dump+0x78/0xc0
> [    7.539142]  do_raw_spin_lock+0xfd/0x160
> [    7.539142]  _raw_spin_lock_irqsave+0x4c/0x60
> [    7.539142]  ? timesync_onchannelcallback+0x153/0x220 [hv_utils]
> [    7.539142]  timesync_onchannelcallback+0x153/0x220 [hv_utils]
>
> Can you resolve this address using gdb to a line of code?  Once inside
> gdb do:
>
> l *(timesync_onchannelcallback+0x153)

(gdb) l *(timesync_onchannelcallback+0x153)
0xffffffffc0104593 is in timesync_onchannelcallback (drivers/hv/hv_util.c:2=
79).
274             } else {
275                     /*
276                      * Save the adjusted time sample from the host and =
the snapshot
277                      * of the current system time for PTP device.
278                      */
279                     spin_lock_irqsave(&host_ts.lock, flags);
280
281                     cur_reftime =3D hyperv_cs->read(hyperv_cs);
282                     host_ts.host_time =3D hosttime;
283                     host_ts.ref_time =3D cur_reftime;

It turns out the "host_ts.lock" isn't initialized with spin_lock_init().
I'll submit a patch for this.

However, the SCSI issue (i.e. *sometimes* the VM fails to boot) is still th=
ere, I think.
We need to continue to debug...

Thanks,
-- Dexuan

RE: Boot regression (was "Re: [PATCH] genhd: Do not hold event lock when scheduling workqueue elements")

[Dexuan Cui]

> From: hch@lst.de [mailto:hch@lst.de]
> Sent: Wednesday, February 15, 2017 00:35
> > I tested today's linux-next (next-20170214) + the 2 patches just now and
> got
> > a weird result:
> > sometimes the VM stills hung with a new calltrace (BUG: spinlock bad
> > magic) , but sometimes the VM did boot up despite the new calltrace!
> >
> > Attached is the log of a "good" boot.
> >
> > It looks we have a memory corruption issue somewhere...
>
> Yes.
It's due to an uninitialized spinlock. Please see the below.

> > Actually previously I saw the "BUG: spinlock bad magic" message once, but
> I
> > couldn't repro it later, so I didn't mention it to you.
>
> Interesting.
Ditto.
And probably my memory was inaccurate due to the long period of bisecting.
We should always see the message.

>
> A few questions on the dmesg:
>
> [    6.208794] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current]
> [    6.209447] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operation
> code
> [    6.210043] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current]
> [    6.210618] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operation
> code
> [    6.212272] sd 2:0:0:0: [storvsc] Sense Key : Illegal Request [current]
> [    6.212897] sd 2:0:0:0: [storvsc] Add. Sense: Invalid command operation
> code
> [    6.213474] sd 3:0:0:0: [storvsc] Sense Key : Illegal Request [current]
> [    6.214051] sd 3:0:0:0: [storvsc] Add. Sense: Invalid command operation
> code
>
> I didn't see anything like this in the other logs.  Are these messages
> something usual on HyperV VMs?

The messages should be normal, and I have always seen them for years
at least.

I think they're triggered by "/sbin/hdparm /dev/sda" or
"/lib/udev/ata_id /dev/sda"  in my Ubuntu 16.04 VM: during the VM
boot-up, the 2 programs are executed by the initrd's systemd-udev or other
system init scripts and the 2 programs try to run these 2 SCSI commands,
which are not supported by Hyper-V:

ATA PASS THROUGH (12) -- 0xa1
ATA PASS THROUGH (16) -- 0x85

IMO the commands are used when PATA/SATA devices are connected to
SCSI-to-ATA bridge, so it's understandable that Hyper-V doesn't support
them.

In the case of boot failure (hang), the 2 programs don't have a chance to
run, so we can't see the messages.

IMO we should ignore the messages, which should be harmless.

> [    6.358405] XFS (sdb1): Mounting V5 Filesystem
> [    6.404478] XFS (sdb1): Ending clean mount
> [    7.535174] BUG: spinlock bad magic on CPU#0, swapper/0/0
> [    7.536807]  lock: host_ts+0x30/0xffffffffffffe1a0 [hv_utils], .magic:
> 00000000, .owner: <none>/-1, .owner_cpu: 0
> [    7.538436] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.10.0-rc8-next-
> 20170214+ #1
> [    7.539142] Hardware name: Microsoft Corporation Virtual Machine/Virtual
> Machine, BIOS 090006  04/28/2016
> [    7.539142] Call Trace:
> [    7.539142]  <IRQ>
> [    7.539142]  dump_stack+0x63/0x82
> [    7.539142]  spin_dump+0x78/0xc0
> [    7.539142]  do_raw_spin_lock+0xfd/0x160
> [    7.539142]  _raw_spin_lock_irqsave+0x4c/0x60
> [    7.539142]  ? timesync_onchannelcallback+0x153/0x220 [hv_utils]
> [    7.539142]  timesync_onchannelcallback+0x153/0x220 [hv_utils]
>
> Can you resolve this address using gdb to a line of code?  Once inside
> gdb do:
>
> l *(timesync_onchannelcallback+0x153)

(gdb) l *(timesync_onchannelcallback+0x153)
0xffffffffc0104593 is in timesync_onchannelcallback (drivers/hv/hv_util.c:279).
274             } else {
275                     /*
276                      * Save the adjusted time sample from the host and the snapshot
277                      * of the current system time for PTP device.
278                      */
279                     spin_lock_irqsave(&host_ts.lock, flags);
280
281                     cur_reftime = hyperv_cs->read(hyperv_cs);
282                     host_ts.host_time = hosttime;
283                     host_ts.ref_time = cur_reftime;

It turns out the "host_ts.lock" isn't initialized with spin_lock_init().
I'll submit a patch for this.

However, the SCSI issue (i.e. *sometimes* the VM fails to boot) is still there, I think.
We need to continue to debug...

Thanks,
-- Dexuan