stack unwinder warning.

stack unwinder warning.

[Dave Jones]

I'm not sure what to make of this. Josh ?  (4.10-rc1)

WARNING: kernel stack frame pointer at ffffc900003e7858 in trinity-c6:29122 has bad value ffffffff82103a80
unwind stack type:0 next_sp:          (null) mask:2 graph_idx:0
ffffc900003e7808: ffffffff811a02e5 (ring_buffer_lock_reserve+0x1d5/0x580)
ffffc900003e7810: ffffffff8119adc3 (rb_commit+0x93/0x350)
ffffc900003e7818: ffffffff811b31d4 (function_trace_call+0x104/0x1f0)
ffffc900003e7820: ffff8804f10ec000 (0xffff8804f10ec000)
ffffc900003e7828: 0000000000000000 ...
ffffc900003e7830: ffffffff8119b3ae (ring_buffer_unlock_commit+0x8e/0x120)
ffffc900003e7838: 0000000000000001 (0x1)
ffffc900003e7840: ffffea0002854e00 (0xffffea0002854e00)
ffffc900003e7848: 000000000000000a (0xa)
ffffc900003e7850: ffffea0002854ec0 (0xffffea0002854ec0)
ffffc900003e7858: ffffea000287c480 (0xffffea000287c480)
ffffc900003e7860: ffffc900003e7918 (0xffffc900003e7918)
ffffc900003e7868: 00000000794ac01f (0x794ac01f)
ffffc900003e7870: ffffc900003e7898 (0xffffc900003e7898)
ffffc900003e7878: ffffffff8119adc3 (rb_commit+0x93/0x350)
ffffc900003e7880: ffff8804f10eb454 (0xffff8804f10eb454)
ffffc900003e7888: ffff88017fc10d48 (0xffff88017fc10d48)
ffffc900003e7890: ffffffff82103a80 (contig_page_data+0x640/0x1d40)
ffffc900003e7898: ffff8804f10eb000 (0xffff8804f10eb000)
ffffc900003e78a0: ffffc900003e7940 (0xffffc900003e7940)
ffffc900003e78a8: 0000000000000080 (0x80)
ffffc900003e78b0: ffffffff8119adc3 (rb_commit+0x93/0x350)
ffffc900003e78b8: 0000000000000000 ...
ffffc900003e78c0: ffff88017fc10d48 (0xffff88017fc10d48)
ffffc900003e78c8: ffff8804f10ea208 (0xffff8804f10ea208)
ffffc900003e78d0: 00000000000003d4 (0x3d4)
ffffc900003e78d8: 0000000000000000 ...
ffffc900003e78e0: ffffffff8119ca13 (__rb_reserve_next+0x153/0x210)
ffffc900003e78e8: ffff8804f10ea000 (0xffff8804f10ea000)
ffffc900003e78f0: ffffffff8127072f (page_mkclean+0x7f/0xf0)
ffffc900003e78f8: 00000000000003e8 (0x3e8)
ffffc900003e7900: ffff8804f10eadd8 (0xffff8804f10eadd8)
ffffc900003e7908: ffff88017fc23528 (0xffff88017fc23528)
ffffc900003e7910: 000004fb2fa02c8a (0x4fb2fa02c8a)
ffffc900003e7918: ffff8804f10e9298 (0xffff8804f10e9298)
ffffc900003e7920: ffffffff8205c460 (ftrace_trace_arrays+0x10/0x10)
ffffc900003e7928: 0000000000000001 (0x1)
ffffc900003e7930: 0000000000000080 (0x80)
ffffc900003e7938: 0000000000000007 (0x7)
ffffc900003e7940: ffff8804f8c55440 (0xffff8804f8c55440)
ffffc900003e7948: ffffc900003e7998 (0xffffc900003e7998)
ffffc900003e7950: ffffffff811b325a (function_trace_call+0x18a/0x1f0)
ffffc900003e7958: 80000001003e7998 (0x80000001003e7998)
ffffc900003e7960: ffffffff812bf326 (__check_object_size+0x66/0x412)
ffffc900003e7968: ffffffff81068aa0 (pmd_clear_huge+0x70/0x70)
ffffc900003e7970: ffff8800a1403000 (0xffff8800a1403000)
ffffc900003e7978: 0000000000001000 (0x1000)
ffffc900003e7980: ffffffff81000000 (0xffffffff81000000)
ffffc900003e7988: 0000000000000000 ...
ffffc900003e7990: ffffea0002850240 (0xffffea0002850240)
ffffc900003e7998: ffffc900003e7a50 (0xffffc900003e7a50)
ffffc900003e79a0: ffffffffa026c077 (0xffffffffa026c077)
ffffc900003e79a8: 80000001000003e8 (0x80000001000003e8)
ffffc900003e79b0: ffffffffa00b3c6d (alloc_extent_state+0x2d/0x260 [btrfs])
ffffc900003e79b8: ffffea0002840040 (0xffffea0002840040)
ffffc900003e79c0: 0000000000000000 ...
ffffc900003e79c8: ffffea0002840080 (0xffffea0002840080)
ffffc900003e79d0: ffff8804f8c55440 (0xffff8804f8c55440)
ffffc900003e79d8: ffffffffa026c077 (0xffffffffa026c077)
ffffc900003e79e0: 0000000000000000 ...
ffffc900003e79e8: ffffea0002840140 (0xffffea0002840140)
ffffc900003e79f0: ffffffff814e2b5c (debug_smp_processor_id+0x1c/0x20)
ffffc900003e79f8: ffffea00028401c0 (0xffffea00028401c0)
ffffc900003e7a00: ffffc900003e7af0 (0xffffc900003e7af0)
ffffc900003e7a08: 0000000000000000 ...
ffffc900003e7a10: ffffc900003e7ac8 (0xffffc900003e7ac8)
ffffc900003e7a18: ffffc900003e7aa0 (0xffffc900003e7aa0)
ffffc900003e7a20: 0000000000000001 (0x1)
ffffc900003e7a28: ffffffff81241085 (__inc_node_page_state+0x5/0x20)
ffffc900003e7a30: 000000000000000f (0xf)
ffffc900003e7a38: ffffffff81240f37 (__inc_zone_state+0x47/0xb0)
ffffc900003e7a40: ffffc900003e7a50 (0xffffc900003e7a50)
ffffc900003e7a48: ffff8804ee0d8798 (0xffff8804ee0d8798)
ffffc900003e7a50: ffffea0002850000 (0xffffea0002850000)
ffffc900003e7a58: ffffc900003e7b38 (0xffffc900003e7b38)
ffffc900003e7a60: ffff8804ef2354b8 (0xffff8804ef2354b8)
ffffc900003e7a68: ffffffff81240fd5 (__inc_node_state+0x5/0xb0)
ffffc900003e7a70: ffffc900003e7ab0 (0xffffc900003e7ab0)
ffffc900003e7a78: ffffc900003e7ab0 (0xffffc900003e7ab0)
ffffc900003e7a80: 0000000000000000 ...
ffffc900003e7a88: ffffea0002850140 (0xffffea0002850140)
ffffc900003e7a90: ffffffff8121ee85 (account_page_dirtied+0x5/0x3c0)
ffffc900003e7a98: ffff8804ef2356f8 (0xffff8804ef2356f8)
ffffc900003e7aa0: ffffffff8121f296 (__set_page_dirty_nobuffers+0x56/0x210)
ffffc900003e7aa8: ffffc900003e7af0 (0xffffc900003e7af0)
ffffc900003e7ab0: 0000000000000000 ...
ffffc900003e7ab8: ffffffff8121f311 (__set_page_dirty_nobuffers+0xd1/0x210)
ffffc900003e7ac0: 0000000000000003 (0x3)
ffffc900003e7ac8: ffffea0002850300 (0xffffea0002850300)
ffffc900003e7ad0: ffffffff8121f245 (__set_page_dirty_nobuffers+0x5/0x210)
ffffc900003e7ad8: ffffea0002850380 (0xffffea0002850380)
ffffc900003e7ae0: dead000000000100 (0xdead000000000100)
ffffc900003e7ae8: 0000000000000020 (0x20)
ffffc900003e7af0: ffffea0002850240 (0xffffea0002850240)
ffffc900003e7af8: ffff8804ef2354b8 (0xffff8804ef2354b8)
ffffc900003e7b00: 0000000000000000 ...
ffffc900003e7b08: ffffc900003e7b20 (0xffffc900003e7b20)
ffffc900003e7b10: 00000000829af000 (0x829af000)
ffffc900003e7b18: 000000000008298f (0x8298f)
ffffc900003e7b20: ffff8804f374a508 (0xffff8804f374a508)
ffffc900003e7b28: 00000000ef2354b8 (0xef2354b8)
ffffc900003e7b30: 0000000000000020 (0x20)
ffffc900003e7b38: 000000008298f000 (0x8298f000)
ffffc900003e7b40: 000000000000001f (0x1f)
ffffc900003e7b48: 0000000000000002 (0x2)
ffffc900003e7b50: 0000000000082991 (0x82991)
ffffc900003e7b58: 000000030342004a (0x30342004a)
ffffc900003e7b60: ffffc900003e7c28 (0xffffc900003e7c28)
ffffc900003e7b68: 000000008298f000 (0x8298f000)
ffffc900003e7b70: ffff8804ef2354b8 (0xffff8804ef2354b8)
ffffc900003e7b78: 0000000000000020 (0x20)
ffffc900003e7b80: ffff8804f374a508 (0xffff8804f374a508)
ffffc900003e7b88: ffffc900003e7c68 (0xffffc900003e7c68)
ffffc900003e7b90: ffffffffa00a5ae8 (__btrfs_buffered_write+0x258/0x820 [btrfs])
ffffc900003e7b98: ffffc900003e7c28 (0xffffc900003e7c28)
ffffc900003e7ba0: ffffc900003e7c18 (0xffffc900003e7c18)
ffffc900003e7ba8: 000000000001ffff (0x1ffff)
ffffc900003e7bb0: 0000000c810fd3d9 (0xc810fd3d9)
ffffc900003e7bb8: ffff8804ef2350d0 (0xffff8804ef2350d0)
ffffc900003e7bc0: ffff8804ed1e1158 (0xffff8804ed1e1158)
ffffc900003e7bc8: 0000000000000fff (0xfff)
ffffc900003e7bd0: 0000000000000020 (0x20)
ffffc900003e7bd8: 0000000000000000 ...
ffffc900003e7be0: 0000000043640000 (0x43640000)
ffffc900003e7be8: ffffc900003e7d88 (0xffffc900003e7d88)
ffffc900003e7bf0: 0000000000020000 (0x20000)
ffffc900003e7bf8: ffff8804f374a508 (0xffff8804f374a508)
ffffc900003e7c00: ffff8804ee0d8000 (0xffff8804ee0d8000)
ffffc900003e7c08: 0000000000000000 ...
ffffc900003e7c20: 00000004ef235538 (0x4ef235538)
ffffc900003e7c28: ffff8804ef235538 (0xffff8804ef235538)
ffffc900003e7c30: 0000000000020000 (0x20000)
ffffc900003e7c38: 00000000794ac01f (0x794ac01f)
ffffc900003e7c40: ffff8804ef2354b8 (0xffff8804ef2354b8)
ffffc900003e7c48: 0000000000000fff (0xfff)
ffffc900003e7c50: ffff8804fcfd2dc0 (0xffff8804fcfd2dc0)
ffffc900003e7c58: ffffc900003e7d08 (0xffffc900003e7d08)
ffffc900003e7c60: 000000003f34f000 (0x3f34f000)
ffffc900003e7c68: ffffc900003e7ce0 (0xffffc900003e7ce0)
ffffc900003e7c70: ffffffffa00aac3b (btrfs_file_write_iter+0x1fb/0x640 [btrfs])
ffffc900003e7c78: ffffc900003e7d88 (0xffffc900003e7d88)
ffffc900003e7c80: 000000003f34f000 (0x3f34f000)
ffffc900003e7c88: ffffffff00000000 (0xffffffff00000000)
ffffc900003e7c90: ffff8804ed1e1158 (0xffff8804ed1e1158)
ffffc900003e7c98: 0000000000000001 (0x1)
ffffc900003e7ca0: 000000003f34f000 (0x3f34f000)
ffffc900003e7ca8: ffff8804ef235598 (0xffff8804ef235598)
ffffc900003e7cb0: ffffc900003e7d88 (0xffffc900003e7d88)
ffffc900003e7cb8: 0000000000000002 (0x2)
ffffc900003e7cc0: 0000000000000000 ...
ffffc900003e7cd0: ffff8804fcfd2dc0 (0xffff8804fcfd2dc0)
ffffc900003e7cd8: ffffc900003e7ec0 (0xffffc900003e7ec0)
ffffc900003e7ce0: ffffc900003e7d60 (0xffffc900003e7d60)
ffffc900003e7ce8: ffffffff812c5985 (do_iter_readv_writev+0x125/0x1c0)
ffffc900003e7cf0: ffff8804ef2354b8 (0xffff8804ef2354b8)
ffffc900003e7cf8: ffffffffa00aaa40 (btrfs_sync_file+0x5e0/0x5e0 [btrfs])
ffffc900003e7d00: ffffc900003e7d88 (0xffffc900003e7d88)
ffffc900003e7d08: ffff8804fcfd2dc0 (0xffff8804fcfd2dc0)
ffffc900003e7d10: 000000003f34f000 (0x3f34f000)
ffffc900003e7d18: 0000000000000000 ...
ffffc900003e7d28: 0000000000000002 (0x2)
ffffc900003e7d30: 00000000794ac01f (0x794ac01f)
ffffc900003e7d38: ffff8804fcfd2dc0 (0xffff8804fcfd2dc0)
ffffc900003e7d40: 0000000000000001 (0x1)
ffffc900003e7d48: ffffc900003e7ec0 (0xffffc900003e7ec0)
ffffc900003e7d50: 0000000000000000 ...
ffffc900003e7d58: ffffffffa00aaa40 (btrfs_sync_file+0x5e0/0x5e0 [btrfs])
ffffc900003e7d60: ffffc900003e7e70 (0xffffc900003e7e70)
ffffc900003e7d68: ffffffff812c6a99 (do_readv_writev+0x209/0x2f0)
ffffc900003e7d70: ffff8804ef2354b8 (0xffff8804ef2354b8)
ffffc900003e7d78: 000000000000006c (0x6c)
ffffc900003e7d80: ffff8802d95c0008 (0xffff8802d95c0008)
ffffc900003e7d88: 0000000000000001 (0x1)
ffffc900003e7d90: 0000000007fea7a5 (0x7fea7a5)
ffffc900003e7d98: 000000003c9bf000 (0x3c9bf000)
ffffc900003e7da0: ffff8802d95c0278 (0xffff8802d95c0278)
ffffc900003e7da8: 0000000000000045 (0x45)
ffffc900003e7db0: 000055f0eb100390 (0x55f0eb100390)
ffffc900003e7db8: ffff8804fcfd2dc0 (0xffff8804fcfd2dc0)
ffffc900003e7dc0: ffffc900003e7eb0 (0xffffc900003e7eb0)
ffffc900003e7dc8: 0000000000000000 ...
ffffc900003e7dd0: ffffc900003e7e88 (0xffffc900003e7e88)
ffffc900003e7dd8: ffffffffa026c077 (0xffffffffa026c077)
ffffc900003e7de0: 0000000000000000 ...
ffffc900003e7de8: ffffc900003e7ec0 (0xffffc900003e7ec0)
ffffc900003e7df0: ffff8804f8c55440 (0xffff8804f8c55440)
ffffc900003e7df8: 000000000000006c (0x6c)
ffffc900003e7e00: 000055f0eb100390 (0x55f0eb100390)
ffffc900003e7e08: ffff8804fcfd2dc0 (0xffff8804fcfd2dc0)
ffffc900003e7e10: 0000000000000001 (0x1)
ffffc900003e7e18: 0000000000000000 ...
ffffc900003e7e20: ffffffff812c6895 (do_readv_writev+0x5/0x2f0)
ffffc900003e7e28: 0000000000000000 ...
ffffc900003e7e30: 0000000000000088 (0x88)
ffffc900003e7e38: 00000000794ac01f (0x794ac01f)
ffffc900003e7e40: 000000000000006c (0x6c)
ffffc900003e7e48: ffff8804fcfd2dc0 (0xffff8804fcfd2dc0)
ffffc900003e7e50: 0000000000040000 (0x40000)
ffffc900003e7e58: 000000000000006c (0x6c)
ffffc900003e7e60: ffffc900003e7ec0 (0xffffc900003e7ec0)
ffffc900003e7e68: 0000000000000000 ...
ffffc900003e7e70: ffffc900003e7eb0 (0xffffc900003e7eb0)
ffffc900003e7e78: ffffffff812c6e9c (vfs_writev+0x6c/0xa0)
ffffc900003e7e80: 000055f0eb100390 (0x55f0eb100390)
ffffc900003e7e88: ffff8804fcfd2dc2 (0xffff8804fcfd2dc2)
ffffc900003e7e90: 000055f0eb100390 (0x55f0eb100390)
ffffc900003e7e98: ffff8804fcfd2dc0 (0xffff8804fcfd2dc0)
ffffc900003e7ea0: 000000000000006c (0x6c)
ffffc900003e7ea8: 0000000000000000 ...
ffffc900003e7eb0: ffffc900003e7ef8 (0xffffc900003e7ef8)
ffffc900003e7eb8: ffffffff812c6f42 (do_writev+0x72/0x120)
ffffc900003e7ec0: 0000000000000088 (0x88)
ffffc900003e7ec8: 00000000794ac01f (0x794ac01f)
ffffc900003e7ed0: 000000000000000e (0xe)
ffffc900003e7ed8: 000055f0eb100390 (0x55f0eb100390)
ffffc900003e7ee0: 000000000000006c (0x6c)
ffffc900003e7ee8: 00007f2320b17ad8 (0x7f2320b17ad8)
ffffc900003e7ef0: 00007f2320ae6000 (0x7f2320ae6000)
ffffc900003e7ef8: ffffc900003e7f20 (0xffffc900003e7f20)
ffffc900003e7f00: ffffffff812c86dc (SyS_writev+0x2c/0x40)
ffffc900003e7f08: ffffc900003e7f58 (0xffffc900003e7f58)
ffffc900003e7f10: 0000000000000014 (0x14)
ffffc900003e7f18: ffff8804f8c55440 (0xffff8804f8c55440)
ffffc900003e7f20: ffffc900003e7f48 (0xffffc900003e7f48)
ffffc900003e7f28: ffffffff810031b4 (do_syscall_64+0x84/0x210)
ffffc900003e7f30: 0000000000000014 (0x14)
ffffc900003e7f38: 0000000000000002 (0x2)
ffffc900003e7f40: 00007f2320ae6048 (0x7f2320ae6048)
ffffc900003e7f48: 00007f2320ae6000 (0x7f2320ae6000)
ffffc900003e7f50: ffffffff81b1eacb (entry_SYSCALL64_slow_path+0x25/0x25)
ffffc900003e7f58: 00007f2320ae6000 (0x7f2320ae6000)
ffffc900003e7f60: 00007f2320b17ad8 (0x7f2320b17ad8)
ffffc900003e7f68: 00007f2320ae6048 (0x7f2320ae6048)
ffffc900003e7f70: 0000000000000002 (0x2)
ffffc900003e7f78: 00007f2320ae6000 (0x7f2320ae6000)
ffffc900003e7f80: 0000000000000014 (0x14)
ffffc900003e7f88: 0000000000000246 (0x246)
ffffc900003e7f90: 0000000008000000 (0x8000000)
ffffc900003e7f98: 00c5193a2b3c602e (0xc5193a2b3c602e)
ffffc900003e7fa0: 00a7a7a7a7a7a7a7 (0xa7a7a7a7a7a7a7)
ffffc900003e7fa8: ffffffffffffffda (0xffffffffffffffda)
ffffc900003e7fb0: 00007f2320442119 (0x7f2320442119)
ffffc900003e7fb8: 000000000000006c (0x6c)
ffffc900003e7fc0: 000055f0eb100390 (0x55f0eb100390)
ffffc900003e7fc8: 000000000000000e (0xe)
ffffc900003e7fd0: 0000000000000014 (0x14)
ffffc900003e7fd8: 00007f2320442119 (0x7f2320442119)
ffffc900003e7fe0: 0000000000000033 (0x33)
ffffc900003e7fe8: 0000000000000246 (0x246)
ffffc900003e7ff0: 00007fff152602d8 (0x7fff152602d8)
ffffc900003e7ff8: 000000000000002b (0x2b)

Re: stack unwinder warning.

[Josh Poimboeuf]

On Tue, Dec 27, 2016 at 02:00:30PM -0500, Dave Jones wrote:
> I'm not sure what to make of this. Josh ?  (4.10-rc1)
> 
> WARNING: kernel stack frame pointer at ffffc900003e7858 in trinity-c6:29122 has bad value ffffffff82103a80
> unwind stack type:0 next_sp:          (null) mask:2 graph_idx:0
> ffffc900003e7808: ffffffff811a02e5 (ring_buffer_lock_reserve+0x1d5/0x580)
> ffffc900003e7810: ffffffff8119adc3 (rb_commit+0x93/0x350)
> ffffc900003e7818: ffffffff811b31d4 (function_trace_call+0x104/0x1f0)
> ffffc900003e7820: ffff8804f10ec000 (0xffff8804f10ec000)
> ffffc900003e7828: 0000000000000000 ...
> ffffc900003e7830: ffffffff8119b3ae (ring_buffer_unlock_commit+0x8e/0x120)
> ffffc900003e7838: 0000000000000001 (0x1)
> ffffc900003e7840: ffffea0002854e00 (0xffffea0002854e00)
> ffffc900003e7848: 000000000000000a (0xa)
> ffffc900003e7850: ffffea0002854ec0 (0xffffea0002854ec0)
> ffffc900003e7858: ffffea000287c480 (0xffffea000287c480)

The value reported by the warning contradicts the value reported by the
dump.  So this seems to have been caused by dumping the stack of a task
which is running on another CPU.  There are still some places in the
code where that's possible.  So I'm going to need to remove these
unwinder warnings for now.

-- 
Josh

Re: stack unwinder warning.

[Josh Poimboeuf]

On Thu, Jan 05, 2017 at 08:52:49AM -0600, Josh Poimboeuf wrote:
> On Tue, Dec 27, 2016 at 02:00:30PM -0500, Dave Jones wrote:
> > I'm not sure what to make of this. Josh ?  (4.10-rc1)
> > 
> > WARNING: kernel stack frame pointer at ffffc900003e7858 in trinity-c6:29122 has bad value ffffffff82103a80
> > unwind stack type:0 next_sp:          (null) mask:2 graph_idx:0
> > ffffc900003e7808: ffffffff811a02e5 (ring_buffer_lock_reserve+0x1d5/0x580)
> > ffffc900003e7810: ffffffff8119adc3 (rb_commit+0x93/0x350)
> > ffffc900003e7818: ffffffff811b31d4 (function_trace_call+0x104/0x1f0)
> > ffffc900003e7820: ffff8804f10ec000 (0xffff8804f10ec000)
> > ffffc900003e7828: 0000000000000000 ...
> > ffffc900003e7830: ffffffff8119b3ae (ring_buffer_unlock_commit+0x8e/0x120)
> > ffffc900003e7838: 0000000000000001 (0x1)
> > ffffc900003e7840: ffffea0002854e00 (0xffffea0002854e00)
> > ffffc900003e7848: 000000000000000a (0xa)
> > ffffc900003e7850: ffffea0002854ec0 (0xffffea0002854ec0)
> > ffffc900003e7858: ffffea000287c480 (0xffffea000287c480)
> 
> The value reported by the warning contradicts the value reported by the
> dump.  So this seems to have been caused by dumping the stack of a task
> which is running on another CPU.  There are still some places in the
> code where that's possible.  So I'm going to need to remove these
> unwinder warnings for now.

I'll be submitting the following patch soon, which I think should
silence the warning.  If the warning is recreatable, would you mind
testing it?


diff --git a/arch/x86/kernel/unwind_frame.c b/arch/x86/kernel/unwind_frame.c
index 4443e49..6fda186 100644
--- a/arch/x86/kernel/unwind_frame.c
+++ b/arch/x86/kernel/unwind_frame.c
@@ -207,6 +207,16 @@ bool unwind_next_frame(struct unwind_state *state)
 	return true;
 
 bad_address:
+	/*
+	 * When dumping a task other than current, the task might actually be
+	 * running on another CPU, in which case it could be modifying its
+	 * stack while we're reading it.  This is generally not a problem and
+	 * can be ignored as long as the caller understands that unwinding
+	 * another task will not always succeed.
+	 */
+	if (state->task != current)
+		goto the_end;
+
 	if (state->regs) {
 		printk_deferred_once(KERN_WARNING
 			"WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n",
-- 
2.7.4

Re: stack unwinder warning.

[Dave Jones]

On Thu, Jan 05, 2017 at 11:02:14AM -0600, Josh Poimboeuf wrote:
 > On Thu, Jan 05, 2017 at 08:52:49AM -0600, Josh Poimboeuf wrote:
 > > On Tue, Dec 27, 2016 at 02:00:30PM -0500, Dave Jones wrote:
 > > > I'm not sure what to make of this. Josh ?  (4.10-rc1)
 > > > 
 > > > WARNING: kernel stack frame pointer at ffffc900003e7858 in trinity-c6:29122 has bad value ffffffff82103a80
 > > > unwind stack type:0 next_sp:          (null) mask:2 graph_idx:0
 > > > ffffc900003e7808: ffffffff811a02e5 (ring_buffer_lock_reserve+0x1d5/0x580)
 > > > ffffc900003e7810: ffffffff8119adc3 (rb_commit+0x93/0x350)
 > > > ffffc900003e7818: ffffffff811b31d4 (function_trace_call+0x104/0x1f0)
 > > > ffffc900003e7820: ffff8804f10ec000 (0xffff8804f10ec000)
 > > > ffffc900003e7828: 0000000000000000 ...
 > > > ffffc900003e7830: ffffffff8119b3ae (ring_buffer_unlock_commit+0x8e/0x120)
 > > > ffffc900003e7838: 0000000000000001 (0x1)
 > > > ffffc900003e7840: ffffea0002854e00 (0xffffea0002854e00)
 > > > ffffc900003e7848: 000000000000000a (0xa)
 > > > ffffc900003e7850: ffffea0002854ec0 (0xffffea0002854ec0)
 > > > ffffc900003e7858: ffffea000287c480 (0xffffea000287c480)
 > > 
 > > The value reported by the warning contradicts the value reported by the
 > > dump.  So this seems to have been caused by dumping the stack of a task
 > > which is running on another CPU.  There are still some places in the
 > > code where that's possible.  So I'm going to need to remove these
 > > unwinder warnings for now.
 > 
 > I'll be submitting the following patch soon, which I think should
 > silence the warning.  If the warning is recreatable, would you mind
 > testing it?

Sure. It usually spits it out pretty quickly. I'll give it a run..

	Dave

Re: stack unwinder warning.

[Dave Jones]

On Thu, Jan 05, 2017 at 11:02:14AM -0600, Josh Poimboeuf wrote:
 > On Thu, Jan 05, 2017 at 08:52:49AM -0600, Josh Poimboeuf wrote:
 > > On Tue, Dec 27, 2016 at 02:00:30PM -0500, Dave Jones wrote:
 > > > I'm not sure what to make of this. Josh ?  (4.10-rc1)
 > > > 
 > > > WARNING: kernel stack frame pointer at ffffc900003e7858 in trinity-c6:29122 has bad value ffffffff82103a80
 > > > unwind stack type:0 next_sp:          (null) mask:2 graph_idx:0
 > > > ffffc900003e7808: ffffffff811a02e5 (ring_buffer_lock_reserve+0x1d5/0x580)
 > > > ffffc900003e7810: ffffffff8119adc3 (rb_commit+0x93/0x350)
 > > > ffffc900003e7818: ffffffff811b31d4 (function_trace_call+0x104/0x1f0)
 > > > ffffc900003e7820: ffff8804f10ec000 (0xffff8804f10ec000)
 > > > ffffc900003e7828: 0000000000000000 ...
 > > > ffffc900003e7830: ffffffff8119b3ae (ring_buffer_unlock_commit+0x8e/0x120)
 > > > ffffc900003e7838: 0000000000000001 (0x1)
 > > > ffffc900003e7840: ffffea0002854e00 (0xffffea0002854e00)
 > > > ffffc900003e7848: 000000000000000a (0xa)
 > > > ffffc900003e7850: ffffea0002854ec0 (0xffffea0002854ec0)
 > > > ffffc900003e7858: ffffea000287c480 (0xffffea000287c480)
 > > 
 > > The value reported by the warning contradicts the value reported by the
 > > dump.  So this seems to have been caused by dumping the stack of a task
 > > which is running on another CPU.  There are still some places in the
 > > code where that's possible.  So I'm going to need to remove these
 > > unwinder warnings for now.
 > 
 > I'll be submitting the following patch soon, which I think should
 > silence the warning.  If the warning is recreatable, would you mind
 > testing it?

Ok, haven't seen anything fall out today while running with this, I'd
call it good. It usually showed up pretty quickly.

	Dave

Re: stack unwinder warning.

[Dave Jones]

On Thu, Jan 05, 2017 at 11:02:14AM -0600, Josh Poimboeuf wrote:
 > On Thu, Jan 05, 2017 at 08:52:49AM -0600, Josh Poimboeuf wrote:
 > > On Tue, Dec 27, 2016 at 02:00:30PM -0500, Dave Jones wrote:
 > > > I'm not sure what to make of this. Josh ?  (4.10-rc1)
 > > > 
 > > > WARNING: kernel stack frame pointer at ffffc900003e7858 in trinity-c6:29122 has bad value ffffffff82103a80
 > > > unwind stack type:0 next_sp:          (null) mask:2 graph_idx:0
 > > > ffffc900003e7808: ffffffff811a02e5 (ring_buffer_lock_reserve+0x1d5/0x580)
 > > > ffffc900003e7810: ffffffff8119adc3 (rb_commit+0x93/0x350)
 > > > ffffc900003e7818: ffffffff811b31d4 (function_trace_call+0x104/0x1f0)
 > > > ffffc900003e7820: ffff8804f10ec000 (0xffff8804f10ec000)
 > > > ffffc900003e7828: 0000000000000000 ...
 > > > ffffc900003e7830: ffffffff8119b3ae (ring_buffer_unlock_commit+0x8e/0x120)
 > > > ffffc900003e7838: 0000000000000001 (0x1)
 > > > ffffc900003e7840: ffffea0002854e00 (0xffffea0002854e00)
 > > > ffffc900003e7848: 000000000000000a (0xa)
 > > > ffffc900003e7850: ffffea0002854ec0 (0xffffea0002854ec0)
 > > > ffffc900003e7858: ffffea000287c480 (0xffffea000287c480)
 > > 
 > > The value reported by the warning contradicts the value reported by the
 > > dump.  So this seems to have been caused by dumping the stack of a task
 > > which is running on another CPU.  There are still some places in the
 > > code where that's possible.  So I'm going to need to remove these
 > > unwinder warnings for now.
 > 
 > I'll be submitting the following patch soon, which I think should
 > silence the warning.  If the warning is recreatable, would you mind
 > testing it?

I didn't wait long enough..

WARNING: kernel stack frame pointer at ffffc90001443f30 in kworker/u8:8:30468 has bad value           (null)
unwind stack type:0 next_sp:          (null) mask:6 graph_idx:0
ffffc90001443d88: d0d0d0d0d0d0d0d0 (0xd0d0d0d0d0d0d0d0)
ffffc90001443d90: ffffffff81e5c420 (ftrace_trace_arrays+0x10/0x10)
ffffc90001443d98: d0d0d0d0d0d0d0d0 (0xd0d0d0d0d0d0d0d0)
ffffc90001443da0: ffff880253ff0848 (0xffff880253ff0848)
ffffc90001443da8: ffffc90001443e68 (0xffffc90001443e68)
ffffc90001443db0: ffffffff810a1430 (umh_complete+0x40/0x40)
ffffc90001443db8: 00000000000002a0 (0x2a0)
ffffc90001443dc0: 0000000000000001 (0x1)
ffffc90001443dc8: 0000000000000000 ...
ffffc90001443dd0: 0000000000000001 (0x1)
ffffc90001443dd8: 0000000000000001 (0x1)
ffffc90001443de0: 0000000000000000 ...
ffffc90001443de8: ffffffff81e5bcc0 (fgraph_sleep_time+0x20/0x20)
ffffc90001443df0: ffffffff818cb022 (ret_from_fork+0x22/0x30)
ffffc90001443df8: ffffffff810a1430 (umh_complete+0x40/0x40)
ffffc90001443e00: ffffffffffffff10 (0xffffffffffffff10)
ffffc90001443e08: ffffffff8117140c (function_trace_call+0x1c/0x190)
ffffc90001443e10: 0000000000000010 (0x10)
ffffc90001443e18: 0000000000000292 (0x292)
ffffc90001443e20: ffffc90001443e38 (0xffffc90001443e38)
ffffc90001443e28: 0000000000000000 ...
ffffc90001443e30: 0000000000000020 (0x20)
ffffc90001443e38: ffff8804f254a468 (0xffff8804f254a468)
ffffc90001443e40: ffffffff810a1430 (umh_complete+0x40/0x40)
ffffc90001443e48: ffff880253ff0848 (0xffff880253ff0848)
ffffc90001443e50: d0d0d0d0d0d0d0d0 (0xd0d0d0d0d0d0d0d0)
ffffc90001443e58: d0d0d0d0d0d0d0d0 (0xd0d0d0d0d0d0d0d0)
ffffc90001443e60: d0d0d0d0d0d0d0d0 (0xd0d0d0d0d0d0d0d0)
ffffc90001443e68: ffffc90001443f20 (0xffffc90001443f20)
ffffc90001443e70: ffffffffa0333077 (0xffffffffa0333077)
ffffc90001443e78: ffffc90001443ee0 (0xffffc90001443ee0)
ffffc90001443e80: 0000000000000086 (0x86)
ffffc90001443e88: ffffffff810b4618 (finish_task_switch+0x78/0x280)
ffffc90001443e90: ffff8804f2549c00 (0xffff8804f2549c00)
ffffc90001443e98: 0000000000000000 ...
ffffc90001443ea0: ffff880507bd8780 (0xffff880507bd8780)
ffffc90001443ea8: ffff880507bd8780 (0xffff880507bd8780)
ffffc90001443eb0: ffff8803211b5380 (0xffff8803211b5380)
ffffc90001443eb8: 0000000000000000 ...
ffffc90001443ec0: 0000000000000001 (0x1)
ffffc90001443ec8: 0000000080000000 (0x80000000)
ffffc90001443ed0: 0000000000000001 (0x1)
ffffc90001443ed8: 0000000080000001 (0x80000001)
ffffc90001443ee0: 0000000000000000 ...
ffffc90001443ee8: ffff880253ff0848 (0xffff880253ff0848)
ffffc90001443ef0: ffffffff810b4618 (finish_task_switch+0x78/0x280)
ffffc90001443ef8: ffffffff810a1435 (call_usermodehelper_exec_async+0x5/0x140)
ffffc90001443f00: ffff8803211b6a00 (0xffff8803211b6a00)
ffffc90001443f08: ffffffff810a1430 (umh_complete+0x40/0x40)
ffffc90001443f10: ffff880253ff0848 (0xffff880253ff0848)
ffffc90001443f18: d0d0d0d0d0d0d0d0 (0xd0d0d0d0d0d0d0d0)
ffffc90001443f20: ffffc90001443f30 (0xffffc90001443f30)
ffffc90001443f28: ffffffff810a1435 (call_usermodehelper_exec_async+0x5/0x140)
ffffc90001443f30: 0000000000000000 ...
ffffc90001443f38: ffffffff818cb022 (ret_from_fork+0x22/0x30)
ffffc90001443f40: 0000000000000000 ...
ffffc90001443f48: ffffffff810a1435 (call_usermodehelper_exec_async+0x5/0x140)
ffffc90001443f50: ffffffff818cb022 (ret_from_fork+0x22/0x30)
ffffc90001443f58: 0000000000000000 ...

Re: stack unwinder warning.

[Josh Poimboeuf]

On Fri, Jan 06, 2017 at 12:09:47AM -0500, Dave Jones wrote:
> On Thu, Jan 05, 2017 at 11:02:14AM -0600, Josh Poimboeuf wrote:
>  > On Thu, Jan 05, 2017 at 08:52:49AM -0600, Josh Poimboeuf wrote:
>  > > On Tue, Dec 27, 2016 at 02:00:30PM -0500, Dave Jones wrote:
>  > > > I'm not sure what to make of this. Josh ?  (4.10-rc1)
>  > > > 
>  > > > WARNING: kernel stack frame pointer at ffffc900003e7858 in trinity-c6:29122 has bad value ffffffff82103a80
>  > > > unwind stack type:0 next_sp:          (null) mask:2 graph_idx:0
>  > > > ffffc900003e7808: ffffffff811a02e5 (ring_buffer_lock_reserve+0x1d5/0x580)
>  > > > ffffc900003e7810: ffffffff8119adc3 (rb_commit+0x93/0x350)
>  > > > ffffc900003e7818: ffffffff811b31d4 (function_trace_call+0x104/0x1f0)
>  > > > ffffc900003e7820: ffff8804f10ec000 (0xffff8804f10ec000)
>  > > > ffffc900003e7828: 0000000000000000 ...
>  > > > ffffc900003e7830: ffffffff8119b3ae (ring_buffer_unlock_commit+0x8e/0x120)
>  > > > ffffc900003e7838: 0000000000000001 (0x1)
>  > > > ffffc900003e7840: ffffea0002854e00 (0xffffea0002854e00)
>  > > > ffffc900003e7848: 000000000000000a (0xa)
>  > > > ffffc900003e7850: ffffea0002854ec0 (0xffffea0002854ec0)
>  > > > ffffc900003e7858: ffffea000287c480 (0xffffea000287c480)
>  > > 
>  > > The value reported by the warning contradicts the value reported by the
>  > > dump.  So this seems to have been caused by dumping the stack of a task
>  > > which is running on another CPU.  There are still some places in the
>  > > code where that's possible.  So I'm going to need to remove these
>  > > unwinder warnings for now.
>  > 
>  > I'll be submitting the following patch soon, which I think should
>  > silence the warning.  If the warning is recreatable, would you mind
>  > testing it?
> 
> I didn't wait long enough..
> 
> WARNING: kernel stack frame pointer at ffffc90001443f30 in kworker/u8:8:30468 has bad value           (null)
> unwind stack type:0 next_sp:          (null) mask:6 graph_idx:0

This is actually a separate issue.  The below patch should fix it.

Would you mind running it for a bit along with the other patch?


diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 5b21970..044d18e 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -36,6 +36,7 @@
 #include <asm/smap.h>
 #include <asm/pgtable_types.h>
 #include <asm/export.h>
+#include <asm/frame.h>
 #include <linux/err.h>
 
 .code64
@@ -408,17 +409,19 @@ END(__switch_to_asm)
  * r12: kernel thread arg
  */
 ENTRY(ret_from_fork)
+	FRAME_BEGIN			/* help unwinder find end of stack */
 	movq	%rax, %rdi
-	call	schedule_tail			/* rdi: 'prev' task parameter */
+	call	schedule_tail		/* rdi: 'prev' task parameter */
 
-	testq	%rbx, %rbx			/* from kernel_thread? */
-	jnz	1f				/* kernel threads are uncommon */
+	testq	%rbx, %rbx		/* from kernel_thread? */
+	jnz	1f			/* kernel threads are uncommon */
 
 2:
-	movq	%rsp, %rdi
+	leaq	FRAME_OFFSET(%rsp),%rdi	/* pt_regs pointer */
 	call	syscall_return_slowpath	/* returns with IRQs disabled */
 	TRACE_IRQS_ON			/* user mode is traced as IRQS on */
 	SWAPGS
+	FRAME_END
 	jmp	restore_regs_and_iret
 
 1:

Re: stack unwinder warning.

[Dave Jones]

On Fri, Jan 06, 2017 at 10:50:32AM -0600, Josh Poimboeuf wrote:

 > > WARNING: kernel stack frame pointer at ffffc90001443f30 in kworker/u8:8:30468 has bad value           (null)
 > > unwind stack type:0 next_sp:          (null) mask:6 graph_idx:0
 > 
 > This is actually a separate issue.  The below patch should fix it.
 > 
 > Would you mind running it for a bit along with the other patch?

haven't seen any reoccurance of that trace in the last day.

	Dave