Greetings, 0day kernel testing robot got the below dmesg and the first bad commit is https://git.kernel.org/pub/scm/linux/kernel/git/mel/linux.git mm-vmscan-node-lru-follow-up-v2r1 commit d5d54a2c5517f0818ad75a2f5b1d26a0dacae46a Author: Mel Gorman AuthorDate: Wed Jul 13 09:30:01 2016 +0100 Commit: Mel Gorman CommitDate: Wed Jul 13 09:30:01 2016 +0100 mm, memcg: move memcg limit enforcement from zones to nodes Memcg needs adjustment after moving LRUs to the node. Limits are tracked per memcg but the soft-limit excess is tracked per zone. As global page reclaim is based on the node, it is easy to imagine a situation where a zone soft limit is exceeded even though the memcg limit is fine. This patch moves the soft limit tree the node. Technically, all the variable names should also change but people are already familiar by the meaning of "mz" even if "mn" would be a more appropriate name now. Link: http://lkml.kernel.org/r/1467970510-21195-15-git-send-email-mgorman@techsingularity.net Signed-off-by: Mel Gorman Acked-by: Michal Hocko Cc: Hillf Danton Acked-by: Johannes Weiner Cc: Joonsoo Kim Cc: Minchan Kim Cc: Rik van Riel Cc: Vlastimil Babka Signed-off-by: Andrew Morton +------------------------------------------------------------------+------------+------------+------------+ | | 3eee0b402e | d5d54a2c55 | a3db8049ae | +------------------------------------------------------------------+------------+------------+------------+ | boot_successes | 1160 | 287 | 8 | | boot_failures | 14 | 27 | 20 | | invoked_oom-killer:gfp_mask=0x | 11 | 0 | 4 | | Mem-Info | 11 | 0 | 4 | | Out_of_memory:Kill_process | 9 | 0 | 12 | | backtrace:SyS_clone+0x | 8 | 10 | 8 | | BUG:kernel_test_crashed | 3 | 2 | | | Kernel_panic-not_syncing:Out_of_memory_and_no_killable_processes | 2 | 0 | 4 | | backtrace:do_mlock+0x | 1 | | | | backtrace:SyS_mlock+0x | 1 | | | | backtrace:_do_fork+0x | 5 | | | | backtrace:SyS_newlstat+0x | 1 | | | | backtrace:lock_torture_stats+0x | 1 | | | | backtrace:getname_flags+0x | 1 | 0 | 1 | | backtrace:SyS_symlink+0x | 1 | 0 | 1 | | warn_alloc_failed+0x | 1 | | | | backtrace:__mm_populate+0x | 1 | 2 | | | backtrace:SyS_mlockall+0x | 1 | 2 | | | WARNING:at_mm/memcontrol.c:#mem_cgroup_update_lru_size | 0 | 25 | 3 | | kernel_BUG_at_mm/memcontrol.c | 0 | 25 | 8 | | invalid_opcode:#[##]PREEMPT_SMP | 0 | 25 | 8 | | RIP:mem_cgroup_update_lru_size | 0 | 25 | | | Kernel_panic-not_syncing:Fatal_exception | 0 | 25 | 8 | | backtrace:SyS_newfstatat+0x | 0 | 6 | 1 | | backtrace:sock_map_fd+0x | 0 | 1 | | | backtrace:SyS_socket+0x | 0 | 1 | | | backtrace:do_sys_open+0x | 0 | 1 | | | backtrace:SyS_openat+0x | 0 | 1 | | | RIP:#:[<#>]RIP:mem_cgroup_update_lru_size | 0 | 0 | 8 | | backtrace:SyS_setsockopt+0x | 0 | 0 | 4 | | backtrace:sock_setsockopt+0x | 0 | 0 | 2 | | backtrace:common_nsleep+0x | 0 | 0 | 1 | | backtrace:SyS_clock_nanosleep+0x | 0 | 0 | 1 | | backtrace:_do_fork | 0 | 0 | 1 | | backtrace:SyS_clone | 0 | 0 | 1 | | backtrace:sock_common_setsockopt | 0 | 0 | 1 | | backtrace:SyS_setsockopt | 0 | 0 | 1 | +------------------------------------------------------------------+------------+------------+------------+ [ 25.517925] ODEBUG: Out of memory. ODEBUG disabled [ 25.665103] ------------[ cut here ]------------ [ 25.665103] ------------[ cut here ]------------ [ 25.666092] WARNING: CPU: 0 PID: 451 at mm/memcontrol.c:998 mem_cgroup_update_lru_size+0x111/0x120 [ 25.666092] WARNING: CPU: 0 PID: 451 at mm/memcontrol.c:998 mem_cgroup_update_lru_size+0x111/0x120 [ 25.668268] mem_cgroup_update_lru_size(ffff880010100008, 3, -4): lru_size 23 but empty [ 25.668268] mem_cgroup_update_lru_size(ffff880010100008, 3, -4): lru_size 23 but empty [ 25.669844] CPU: 0 PID: 451 Comm: trinity-main Not tainted 4.7.0-rc7-mm1-00218-gd5d54a2 #1 [ 25.669844] CPU: 0 PID: 451 Comm: trinity-main Not tainted 4.7.0-rc7-mm1-00218-gd5d54a2 #1 [ 25.670012] 0000000000000000 [ 25.670012] 0000000000000000 ffff880008677408 ffff880008677408 ffffffff8136ea65 ffffffff8136ea65 ffff880008677458 ffff880008677458 [ 25.670012] 0000000000000000 [ 25.670012] 0000000000000000 ffff880008677448 ffff880008677448 ffffffff810889fc ffffffff810889fc 000003e68100e2d9 000003e68100e2d9 [ 25.670012] 0000000000000003 [ 25.670012] 0000000000000003 fffffffffffffffc fffffffffffffffc ffffffff82b32cc0 ffffffff82b32cc0 ffff880010100008 ffff880010100008 [ 25.670012] Call Trace: [ 25.670012] Call Trace: [ 25.670012] [] dump_stack+0x86/0xc1 [ 25.670012] [] dump_stack+0x86/0xc1 [ 25.670012] [] __warn+0xbc/0xe0 [ 25.670012] [] __warn+0xbc/0xe0 [ 25.670012] [] warn_slowpath_fmt+0x4a/0x50 [ 25.670012] [] warn_slowpath_fmt+0x4a/0x50 [ 25.670012] [] mem_cgroup_update_lru_size+0x111/0x120 [ 25.670012] [] mem_cgroup_update_lru_size+0x111/0x120 [ 25.670012] [] isolate_lru_pages+0x15b/0x1f0 [ 25.670012] [] isolate_lru_pages+0x15b/0x1f0 [ 25.670012] [] shrink_active_list+0xbf/0x2b0 [ 25.670012] [] shrink_active_list+0xbf/0x2b0 [ 25.670012] [] shrink_node_memcg+0x301/0x490 [ 25.670012] [] shrink_node_memcg+0x301/0x490 [ 25.670012] [] shrink_node+0xbf/0x1d0 [ 25.670012] [] shrink_node+0xbf/0x1d0 [ 25.670012] [] do_try_to_free_pages+0xc4/0x270 [ 25.670012] [] do_try_to_free_pages+0xc4/0x270 [ 25.670012] [] try_to_free_pages+0xae/0xc0 [ 25.670012] [] try_to_free_pages+0xae/0xc0 [ 25.670012] [] __alloc_pages_slowpath+0x277/0xb90 [ 25.670012] [] __alloc_pages_slowpath+0x277/0xb90 [ 25.670012] [] __alloc_pages_nodemask+0x1d5/0x1f0 [ 25.670012] [] __alloc_pages_nodemask+0x1d5/0x1f0 [ 25.670012] [] new_slab+0x2c0/0x610 [ 25.670012] [] new_slab+0x2c0/0x610 [ 25.670012] [] ? proc_alloc_inode+0x18/0xb0 [ 25.670012] [] ? proc_alloc_inode+0x18/0xb0 [ 25.670012] [] ___slab_alloc+0x1a6/0x490 [ 25.670012] [] ___slab_alloc+0x1a6/0x490 [ 25.670012] [] ? proc_alloc_inode+0x18/0xb0 [ 25.670012] [] ? proc_alloc_inode+0x18/0xb0 [ 25.670012] [] ? proc_alloc_inode+0x18/0xb0 [ 25.670012] [] ? proc_alloc_inode+0x18/0xb0 [ 25.670012] [] __slab_alloc+0x50/0xa0 [ 25.670012] [] __slab_alloc+0x50/0xa0 [ 25.670012] [] ? proc_alloc_inode+0x18/0xb0 [ 25.670012] [] ? proc_alloc_inode+0x18/0xb0 [ 25.670012] [] kmem_cache_alloc+0x115/0x140 [ 25.670012] [] kmem_cache_alloc+0x115/0x140 [ 25.670012] [] proc_alloc_inode+0x18/0xb0 [ 25.670012] [] proc_alloc_inode+0x18/0xb0 [ 25.670012] [] alloc_inode+0x18/0x80 [ 25.670012] [] alloc_inode+0x18/0x80 [ 25.670012] [] new_inode_pseudo+0xc/0x60 [ 25.670012] [] new_inode_pseudo+0xc/0x60 [ 25.670012] [] proc_get_inode+0xf/0x130 [ 25.670012] [] proc_get_inode+0xf/0x130 [ 25.670012] [] proc_lookup_de+0x54/0xb0 [ 25.670012] [] proc_lookup_de+0x54/0xb0 [ 25.670012] [] proc_tgid_net_lookup+0x33/0x40 [ 25.670012] [] proc_tgid_net_lookup+0x33/0x40 [ 25.670012] [] lookup_slow+0x11b/0x1f0 [ 25.670012] [] lookup_slow+0x11b/0x1f0 [ 25.670012] [] walk_component+0x1ce/0x520 [ 25.670012] [] walk_component+0x1ce/0x520 [ 25.670012] [] ? link_path_walk+0x71/0x5d0 [ 25.670012] [] ? link_path_walk+0x71/0x5d0 [ 25.670012] [] ? trace_hardirqs_on+0xd/0x10 [ 25.670012] [] ? trace_hardirqs_on+0xd/0x10 [ 25.670012] [] path_lookupat+0x62/0x120 [ 25.670012] [] path_lookupat+0x62/0x120 [ 25.670012] [] filename_lookup+0x99/0x150 [ 25.670012] [] filename_lookup+0x99/0x150 [ 25.670012] [] ? trace_hardirqs_on+0xd/0x10 [ 25.670012] [] ? trace_hardirqs_on+0xd/0x10 [ 25.670012] [] ? getname_flags+0x4a/0x1e0 [ 25.670012] [] ? getname_flags+0x4a/0x1e0 [ 25.670012] [] ? kmem_cache_alloc+0x115/0x140 [ 25.670012] [] ? kmem_cache_alloc+0x115/0x140 [ 25.670012] [] user_path_at_empty+0x31/0x40 [ 25.670012] [] user_path_at_empty+0x31/0x40 [ 25.670012] [] vfs_fstatat+0x4e/0xa0 [ 25.670012] [] vfs_fstatat+0x4e/0xa0 [ 25.670012] [] SYSC_newfstatat+0x15/0x30 [ 25.670012] [] SYSC_newfstatat+0x15/0x30 [ 25.670012] [] ? trace_hardirqs_on_caller+0xf5/0x1b0 git bisect start a3db8049aec5ca8c90d86d1f09a63ce5ceeb66be 92d21ac74a9e3c09b0b01c764e530657e4c85c49 -- git bisect bad d52ed4edc9167152aa2976e04926c041f4eff9b8 # 15:02 0- 11 Merge 'yexl/git-format-patch-verify-on-lv-zheng-ACPI' into devel-spot-201607151924 git bisect bad a0f2d9d9e90d0ff3d6d9fdd92dec785d748122c2 # 15:02 0- 107 Merge 'linux-review/Javier-Martinez-Canillas/vb2-include-length-in-dmabuf-qbuf-debug-message/20160715-152805' into devel-spot-201607151924 git bisect bad f45065106a42183395249f4a36c87d80c83cb175 # 15:02 0- 176 Merge 'linux-review/Philippe-Reynes/net-ethernet-smsc9420-use-phydev-from-struct-net_device/20160715-172417' into devel-spot-201607151924 git bisect bad 519b4ca9bc0e8dbc1835988e4339eecda289f579 # 15:03 0- 180 Merge 'jpirko-mlxsw/combined_queue' into devel-spot-201607151924 git bisect bad 151f330b09503ff463f19f0116c93276fb7d9c1c # 15:03 0- 161 Merge 'arm-tegra/for-next' into devel-spot-201607151924 git bisect good 2b75d858d5e5005649a6878c95e673f75b54c85f # 15:06 140+ 104 0day base guard for 'devel-spot-201607151924' git bisect bad 51ce11d42369204dd8c0fb620cce050b367c6daa # 15:08 8- 8 Merge 'mel/mm-vmscan-node-lru-follow-up-v2r1' into devel-spot-201607151924 git bisect good 8b6f2fa3d440dfb5e9b246184e8af5a0c5aafdcb # 15:30 136+ 2 thp, mlock: do not mlock PTE-mapped file huge pages git bisect bad 2c80750fac5a84b15106733b96702a1819980038 # 16:05 3- 1 mm: vmstat: replace __count_zone_vm_events with a zone id equivalent git bisect good 67f413fd3aa33595d03a90e448e56f06366a34cf # 16:21 140+ 2 mm: update the comment in __isolate_free_page git bisect good 3313b5ca3cd67cf1751d70c6d9453072f4f3998a # 16:37 138+ 0 mm, vmscan: remove duplicate logic clearing node congestion and dirty state git bisect bad 68b0562046a73eb40b5b17866854a49b7512fc74 # 16:40 17- 1 mm, vmscan: only wakeup kswapd once per node for the requested classzone git bisect bad 80022ba40a24cfe6b0aadf720a0c112833477423 # 16:48 67- 3 mm, page_alloc: consider dirtyable memory in terms of nodes git bisect good 3eee0b402edcc91203b5486d0d269e8473add914 # 17:03 250+ 2 mm, vmscan: make shrink_node decisions more node-centric git bisect bad b69ad1f6f47bd028fb0573e2c94795ab77363e70 # 17:12 55- 1 mm, workingset: make working set detection node-aware git bisect bad d5d54a2c5517f0818ad75a2f5b1d26a0dacae46a # 17:17 1- 1 mm, memcg: move memcg limit enforcement from zones to nodes # first bad commit: [d5d54a2c5517f0818ad75a2f5b1d26a0dacae46a] mm, memcg: move memcg limit enforcement from zones to nodes git bisect good 3eee0b402edcc91203b5486d0d269e8473add914 # 17:30 906+ 14 mm, vmscan: make shrink_node decisions more node-centric # extra tests with CONFIG_DEBUG_INFO_REDUCED git bisect bad d5d54a2c5517f0818ad75a2f5b1d26a0dacae46a # 17:39 7- 6 mm, memcg: move memcg limit enforcement from zones to nodes # extra tests on HEAD of linux-devel/devel-spot-201607151924 git bisect bad a3db8049aec5ca8c90d86d1f09a63ce5ceeb66be # 17:39 0- 20 0day head guard for 'devel-spot-201607151924' # extra tests on tree/branch mel/mm-vmscan-node-lru-follow-up-v2r1 git bisect good d5502746ab169a810a4adc0ec760117790e477df # 18:01 906+ 112 mm, vmscan: Update all zone LRU sizes before updating memcg # extra tests on tree/branch linus/master git bisect good 194dc870a5890e855ecffb30f3b80ba7c88f96d6 # 18:49 910+ 9 Add braces to avoid "ambiguous a??elsea??" compiler warnings # extra tests on tree/branch linux-next/master git bisect good d4e661a48572b90ab727149e2f2ec087380a573b # 19:17 910+ 8 Add linux-next specific files for 20160728 This script may reproduce the error. ---------------------------------------------------------------------------- #!/bin/bash kernel=$1 initrd=quantal-core-x86_64.cgz wget --no-clobber https://github.com/fengguang/reproduce-kernel-bug/raw/master/initrd/$initrd kvm=( qemu-system-x86_64 -enable-kvm -cpu kvm64 -kernel $kernel -initrd $initrd -m 300 -smp 2 -device e1000,netdev=net0 -netdev user,id=net0 -boot order=nc -no-reboot -watchdog i6300esb -watchdog-action debug -rtc base=localtime -serial stdio -display none -monitor null ) append=( root=/dev/ram0 hung_task_panic=1 debug apic=debug sysrq_always_enabled rcupdate.rcu_cpu_stall_timeout=100 panic=-1 softlockup_panic=1 nmi_watchdog=panic oops=panic load_ramdisk=2 prompt_ramdisk=0 systemd.log_level=err ignore_loglevel earlyprintk=ttyS0,115200 console=ttyS0,115200 console=tty0 vga=normal rw drbd.minor_count=8 ) "${kvm[@]}" --append "${append[*]}" ---------------------------------------------------------------------------- --- 0-DAY kernel test infrastructure Open Source Technology Center https://lists.01.org/pipermail/lkp Intel Corporation