Greetings,

0day kernel testing robot got the below dmesg and the first bad commit is

git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git revert-c418b8035fac0cc7d242e5de126cec1006a34bed-dd2b39be8eee9d175c7842c30e405a5cbe50095a

commit dd2b39be8eee9d175c7842c30e405a5cbe50095a
Author:     Paul E. McKenney <paulmck@linux.vnet.ibm.com>
AuthorDate: Wed Jan 28 14:42:09 2015 -0800
Commit:     Paul E. McKenney <paulmck@linux.vnet.ibm.com>
CommitDate: Fri Jan 30 12:59:22 2015 -0800

    rcu: Handle outgoing CPUs on exit from idle loop
    
    This commit informs RCU of an outgoing CPU just before that CPU invokes
    arch_cpu_idle_dead() during its last pass through the idle loop (via a
    new CPU_DYING_IDLE notifier value).  This change means that RCU need not
    deal with outgoing CPUs passing through the scheduler after informing
    RCU that they are no longer online.  Note that removing the CPU from
    the rcu_node ->qsmaskinit bit masks is done at CPU_DYING_IDLE time,
    and orphaning callbacks is still done at CPU_DEAD time, the reason being
    that at CPU_DEAD time we have another CPU that can adopt them.
    
    Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

+-------------------------------------+------------+------------+------------+
|                                     | 8c50d7da91 | dd2b39be8e | d586642522 |
+-------------------------------------+------------+------------+------------+
| boot_successes                      | 198        | 11         | 51         |
| boot_failures                       | 0          | 55         | 15         |
| INFO:suspicious_RCU_usage           | 0          | 55         | 15         |
| RCU_used_illegally_from_offline_CPU | 0          | 55         | 15         |
| backtrace:cpu_startup_entry         | 0          | 55         | 15         |
| BUG:kernel_test_hang                | 0          | 0          | 4          |
+-------------------------------------+------------+------------+------------+

[   15.244825] numa_remove_cpu cpu 0 node 0: mask now 1
[   15.246713] 
[   15.246917] ===============================
[   15.247424] [ INFO: suspicious RCU usage. ]
[   15.247964] 3.19.0-rc1-gdd2b39b #10 Not tainted
[   15.248531] -------------------------------
[   15.248586] include/trace/events/rcu.h:35 suspicious rcu_dereference_check() usage!
[   15.248586] 
[   15.248586] other info that might help us debug this:
[   15.248586] 
[   15.248586] 
[   15.248586] RCU used illegally from offline CPU!
[   15.248586] rcu_scheduler_active = 1, debug_locks = 0
[   15.248586] no locks held by swapper/0/0.
[   15.248586] 
[   15.248586] stack backtrace:
[   15.248586] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 3.19.0-rc1-gdd2b39b #10
[   15.248586]  0000000000000001 ffffffff81e03e08 ffffffff8171b89b 0000000000000000
[   15.248586]  ffffffff81e0e580 ffffffff81e03e38 ffffffff810efec2 ffffffff81e4b140
[   15.248586]  ffffffff81c77ba0 0000000000000002 ffffffff81e11e98 ffffffff81e03e58
[   15.248586] Call Trace:
[   15.248586]  [<ffffffff8171b89b>] dump_stack+0x7f/0xa7
[   15.248586]  [<ffffffff810efec2>] lockdep_rcu_suspicious+0x107/0x110
[   15.248586]  [<ffffffff81111363>] trace_rcu_utilization+0x127/0x133
[   15.248586]  [<ffffffff8111291e>] rcu_cpu_notify+0x527/0x53b
[   15.248586]  [<ffffffff810e9722>] cpu_startup_entry+0x1dc/0x4ea
[   15.248586]  [<ffffffff8170eb5d>] rest_init+0x159/0x15f
[   15.248586]  [<ffffffff8237b2da>] start_kernel+0x565/0x572
[   15.248586]  [<ffffffff8237a120>] ? early_idt_handlers+0x120/0x120
[   15.248586]  [<ffffffff8237a4e4>] x86_64_start_reservations+0x41/0x43
[   15.248586]  [<ffffffff8237a623>] x86_64_start_kernel+0x13d/0x14c
[   15.265151] CPU 0 is now offline
[   15.265941] debug: unmapping init [mem 0xffffffff82365000-0xffffffff82539fff]
[   15.266726] Write protecting the kernel read-only data: 14336k

git bisect start d58664252218cfefb19709d597ff0c5d93688203 26bc420b59a38e4e6685a73345a0def461136dce --
git bisect  bad 19f7d9c2f948a4c5c7742adb16fe00920f35f302  # 13:29     23-      6  Merge 'jtkirshe-net-next/i40e-queue' into devel-roam-smoke-201501311226
git bisect  bad 2c86978183cc365003e2d6949052a30865ef8b89  # 13:33     34-     32  Merge 'wsa/i2c/for-next' into devel-roam-smoke-201501311226
git bisect good 1ffdd3662d27b1d4d59d51bbcc104b200be63d6a  # 13:37     66+      0  Merge 'pci/pci/virtualization' into devel-roam-smoke-201501311226
git bisect  bad 0ce6ea6707a3d5ae5bfdbdc4d16ebc86cff77f5f  # 13:43     32-     22  Merge 'rcu/rcu/next' into devel-roam-smoke-201501311226
git bisect good 53805a9f2fa76294af534fb7e9f96d43f1d820eb  # 13:52     66+      0  Merge 'iio/testing' into devel-roam-smoke-201501311226
git bisect good 78e691f4ae2d5edea0199ca802bb505b9cdced88  # 14:01     66+      0  Merge branches 'doc.2015.01.07a', 'fixes.2015.01.15a', 'preempt.2015.01.06a', 'srcu.2015.01.06a', 'stall.2015.01.16a' and 'torture.2015.01.11a' into HEAD
git bisect good 17366dc8dc49858ba931c4120d8de494e388d93e  # 14:05     66+      0  documentation: Update rcutree.kthread_prio for grace-period kthread use
git bisect good 569c1500e44189136c8a9f4b5e39f0055e422b0d  # 14:14     66+      0  documentation: Update based on on-demand vmstat workers
git bisect good 14fefdb410cf48327c972ce91deb5e98edc8671f  # 14:18     66+      0  rcu: Eliminate ->onoff_mutex from rcu_node structure
git bisect  bad dd2b39be8eee9d175c7842c30e405a5cbe50095a  # 14:29     11-     55  rcu: Handle outgoing CPUs on exit from idle loop
git bisect good 8c50d7da9124a9f1e92e13996a0a148b2431390d  # 14:34     66+      0  cpu: Make CPU-offline idle-loop transition point more precise
# first bad commit: [dd2b39be8eee9d175c7842c30e405a5cbe50095a] rcu: Handle outgoing CPUs on exit from idle loop
git bisect good 8c50d7da9124a9f1e92e13996a0a148b2431390d  # 14:37    198+      0  cpu: Make CPU-offline idle-loop transition point more precise
# extra tests with DEBUG_INFO
git bisect good dd2b39be8eee9d175c7842c30e405a5cbe50095a  # 15:35    198+    198  rcu: Handle outgoing CPUs on exit from idle loop
# extra tests on HEAD of linux-devel/devel-roam-smoke-201501311226
git bisect  bad d58664252218cfefb19709d597ff0c5d93688203  # 15:35      0-     15  0day head guard for 'devel-roam-smoke-201501311226'
# extra tests on tree/branch rcu/rcu/next
git bisect  bad c418b8035fac0cc7d242e5de126cec1006a34bed  # 15:52     47-     21  cpu: Stop newly offlined CPU from using RCU readers
# extra tests with first bad commit reverted
# extra tests on tree/branch linus/master
git bisect good 2141fd018156db0f29efb384f4d99ead23b48f18  # 16:04    198+      0  Merge tag 'char-misc-3.19-rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc
# extra tests on tree/branch next/master
git bisect good 827e3bdf1bb2401c1a1e5586eb3977d228d298b2  # 16:12    198+      0  Add linux-next specific files for 20150130


This script may reproduce the error.

----------------------------------------------------------------------------
#!/bin/bash

kernel=$1
initrd=quantal-core-x86_64.cgz

wget --no-clobber https://github.com/fengguang/reproduce-kernel-bug/raw/master/initrd/$initrd

kvm=(
	qemu-system-x86_64
	-cpu kvm64
	-enable-kvm
	-kernel $kernel
	-initrd $initrd
	-m 320
	-smp 2
	-net nic,vlan=1,model=e1000
	-net user,vlan=1
	-boot order=nc
	-no-reboot
	-watchdog i6300esb
	-rtc base=localtime
	-serial stdio
	-display none
	-monitor null 
)

append=(
	hung_task_panic=1
	earlyprintk=ttyS0,115200
	debug
	apic=debug
	sysrq_always_enabled
	rcupdate.rcu_cpu_stall_timeout=100
	panic=-1
	softlockup_panic=1
	nmi_watchdog=panic
	oops=panic
	load_ramdisk=2
	prompt_ramdisk=0
	console=ttyS0,115200
	console=tty0
	vga=normal
	root=/dev/ram0
	rw
	drbd.minor_count=8
)

"${kvm[@]}" --append "${append[*]}"
----------------------------------------------------------------------------

Thanks,
Fengguang