From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S932779AbcGOLnA (ORCPT ); Fri, 15 Jul 2016 07:43:00 -0400 Received: from www262.sakura.ne.jp ([202.181.97.72]:37875 "EHLO www262.sakura.ne.jp" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S932590AbcGOLmy (ORCPT ); Fri, 15 Jul 2016 07:42:54 -0400 Subject: Re: System freezes after OOM To: Ondrej Kozina , Michal Hocko References: <57837CEE.1010609@redhat.com> <9be09452-de7f-d8be-fd5d-4a80d1cd1ba3@redhat.com> <20160712064905.GA14586@dhcp22.suse.cz> <20160713111006.GF28723@dhcp22.suse.cz> <20160714125129.GA12289@dhcp22.suse.cz> <740b17f0-e1bb-b021-e9e1-ad6dcf5f033a@redhat.com> <20160714153120.GD12289@dhcp22.suse.cz> <9ca3459a-8226-b870-163e-58e2bb10df74@redhat.com> Cc: Mikulas Patocka , Jerome Marchand , Stanislav Kozina , linux-mm@kvack.org, linux-kernel@vger.kernel.org, dm-devel@redhat.com From: Tetsuo Handa Message-ID: <747214d9-be8c-2bbb-9b19-147541b3d439@I-love.SAKURA.ne.jp> Date: Fri, 15 Jul 2016 20:42:40 +0900 User-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64; rv:45.0) Gecko/20100101 Thunderbird/45.2.0 MIME-Version: 1.0 In-Reply-To: <9ca3459a-8226-b870-163e-58e2bb10df74@redhat.com> Content-Type: text/plain; charset=windows-1252 Content-Transfer-Encoding: 7bit Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org On 2016/07/15 2:07, Ondrej Kozina wrote: > On 07/14/2016 05:31 PM, Michal Hocko wrote: >> On Thu 14-07-16 16:08:28, Ondrej Kozina wrote: >> [...] >>> As Mikulas pointed out, this doesn't work. The system froze as well with the >>> patch above. Will try to tweak the patch with Mikulas's suggestion... >> >> Thank you for testing! Do you happen to have traces of the frozen >> processes? Does the flusher still gets throttled because the bias it >> gets is not sufficient. Or does it get throttled at a different place? >> > > Sure. Here it is (including sysrq+t and sysrq+w output): https://okozina.fedorapeople.org/bugs/swap_on_dmcrypt/4.7.0-rc7+/1/4.7.0-rc7+.log > Oh, this resembles another dm-crypt lockup problem reported last month. ( http://lkml.kernel.org/r/20160616212641.GA3308@sig21.net ) In Johannes's case, there are so many pending kcryptd_crypt work requests and mempool_alloc() is waiting at throttle_vm_writeout() or shrink_inactive_list(). [ 2378.279029] kswapd0 D ffff88003744f538 0 766 2 0x00000000 [ 2378.286167] ffff88003744f538 00ff88011b5ccd80 ffff88011b5d62d8 ffff88011ae58000 [ 2378.293628] ffff880037450000 ffff880037450000 00000001000984f2 ffff88003744f570 [ 2378.301168] ffff88011b5ccd80 ffff880037450000 ffff88003744f550 ffffffff81845cec [ 2378.308674] Call Trace: [ 2378.311154] [] schedule+0x8b/0xa3 [ 2378.316153] [] schedule_timeout+0x20b/0x285 [ 2378.322028] [] ? init_timer_key+0x112/0x112 [ 2378.327931] [] io_schedule_timeout+0xa0/0x102 [ 2378.333960] [] ? io_schedule_timeout+0xa0/0x102 [ 2378.340166] [] mempool_alloc+0x123/0x154 [ 2378.345781] [] ? wait_woken+0x72/0x72 [ 2378.351148] [] bio_alloc_bioset+0xe8/0x1d7 [ 2378.356910] [] alloc_tio+0x2d/0x47 [ 2378.361996] [] __split_and_process_bio+0x310/0x3a3 [ 2378.368470] [] dm_make_request+0xb5/0xe2 [ 2378.374078] [] generic_make_request+0xcc/0x180 [ 2378.380206] [] submit_bio+0xfd/0x145 [ 2378.385482] [] __swap_writepage+0x202/0x225 [ 2378.391349] [] ? preempt_count_sub+0xf0/0x100 [ 2378.397398] [] ? _raw_spin_unlock+0x31/0x44 [ 2378.403273] [] ? page_swapcount+0x45/0x4c [ 2378.408984] [] swap_writepage+0x3a/0x3e [ 2378.414530] [] pageout.isra.16+0x160/0x2a7 [ 2378.420320] [] shrink_page_list+0x5a0/0x8c4 [ 2378.426197] [] shrink_inactive_list+0x29e/0x4a1 [ 2378.432434] [] shrink_zone_memcg+0x4c1/0x661 [ 2378.438406] [] shrink_zone+0xdc/0x1e5 [ 2378.443742] [] ? shrink_zone+0xdc/0x1e5 [ 2378.449238] [] kswapd+0x6df/0x814 [ 2378.454222] [] ? mem_cgroup_shrink_node_zone+0x209/0x209 [ 2378.461196] [] kthread+0xff/0x107 [ 2378.466182] [] ret_from_fork+0x22/0x50 [ 2378.471631] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2378.769494] kworker/u8:4 D ffff8800c5dc3508 0 1592 2 0x00000000 [ 2378.776582] Workqueue: kcryptd kcryptd_crypt [ 2378.780887] ffff8800c5dc3508 00ff88011b7ccd80 ffff88011b7d62d8 ffff88011ae5a900 [ 2378.788399] ffff88011a605200 ffff8800c5dc4000 00000001000983f7 ffff8800c5dc3540 [ 2378.795930] ffff88011b7ccd80 0000000000000000 ffff8800c5dc3520 ffffffff81845cec [ 2378.803408] Call Trace: [ 2378.805879] [] schedule+0x8b/0xa3 [ 2378.810908] [] schedule_timeout+0x20b/0x285 [ 2378.816783] [] ? init_timer_key+0x112/0x112 [ 2378.822677] [] io_schedule_timeout+0xa0/0x102 [ 2378.828716] [] ? io_schedule_timeout+0xa0/0x102 [ 2378.834956] [] congestion_wait+0x84/0x160 [ 2378.840658] [] ? wait_woken+0x72/0x72 [ 2378.845997] [] throttle_vm_writeout+0x88/0xab [ 2378.852036] [] shrink_zone_memcg+0x635/0x661 [ 2378.857982] [] shrink_zone+0xdc/0x1e5 [ 2378.863309] [] ? shrink_zone+0xdc/0x1e5 [ 2378.868832] [] do_try_to_free_pages+0x1a5/0x2c3 [ 2378.875028] [] try_to_free_pages+0x123/0x21f [ 2378.880972] [] __alloc_pages_nodemask+0x4c9/0x978 [ 2378.887385] [] ? debug_smp_processor_id+0x17/0x19 [ 2378.893782] [] new_slab+0xbc/0x3bb [ 2378.898868] [] ___slab_alloc.constprop.22+0x2fb/0x37b [ 2378.905634] [] ? mempool_alloc_slab+0x15/0x17 [ 2378.911659] [] ? sched_clock+0x9/0xd [ 2378.916909] [] ? local_clock+0x20/0x22 [ 2378.922325] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2378.928877] [] ? sched_clock+0x9/0xd [ 2378.934138] [] ? local_clock+0x20/0x22 [ 2378.939555] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2378.946125] [] __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2378.953289] [] ? __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2378.960630] [] ? mempool_alloc_slab+0x15/0x17 [ 2378.966706] [] kmem_cache_alloc+0xa0/0x1d6 [ 2378.972503] [] ? mempool_alloc_slab+0x15/0x17 [ 2378.978567] [] mempool_alloc_slab+0x15/0x17 [ 2378.984426] [] mempool_alloc+0x72/0x154 [ 2378.989930] [] ? lockdep_init_map+0xc9/0x5a3 [ 2378.995866] [] ? local_clock+0x20/0x22 [ 2379.001300] [] bio_alloc_bioset+0xe8/0x1d7 [ 2379.007107] [] kcryptd_crypt+0x1ab/0x325 [ 2379.012704] [] ? process_one_work+0x1ad/0x4e2 [ 2379.018753] [] process_one_work+0x283/0x4e2 [ 2379.024629] [] ? put_lock_stats.isra.9+0xe/0x20 [ 2379.030851] [] worker_thread+0x285/0x370 [ 2379.036423] [] ? rescuer_thread+0x2d1/0x2d1 [ 2379.042309] [] kthread+0xff/0x107 [ 2379.047310] [] ret_from_fork+0x22/0x50 [ 2379.052726] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2379.059328] kworker/u8:6 D ffff8800c5ec3508 0 1594 2 0x00000000 [ 2379.066468] Workqueue: kcryptd kcryptd_crypt [ 2379.070808] ffff8800c5ec3508 00ff88011b7ccd80 ffff88011b7d62d8 ffff88011ae5a900 [ 2379.078296] ffff88003749a900 ffff8800c5ec4000 0000000100098467 ffff8800c5ec3540 [ 2379.085836] ffff88011b7ccd80 0000000000000000 ffff8800c5ec3520 ffffffff81845cec [ 2379.093315] Call Trace: [ 2379.095776] [] schedule+0x8b/0xa3 [ 2379.100785] [] schedule_timeout+0x20b/0x285 [ 2379.106627] [] ? init_timer_key+0x112/0x112 [ 2379.112494] [] io_schedule_timeout+0xa0/0x102 [ 2379.118524] [] ? io_schedule_timeout+0xa0/0x102 [ 2379.124740] [] congestion_wait+0x84/0x160 [ 2379.130432] [] ? wait_woken+0x72/0x72 [ 2379.135771] [] throttle_vm_writeout+0x88/0xab [ 2379.141839] [] shrink_zone_memcg+0x635/0x661 [ 2379.147810] [] shrink_zone+0xdc/0x1e5 [ 2379.153155] [] ? shrink_zone+0xdc/0x1e5 [ 2379.158651] [] do_try_to_free_pages+0x1a5/0x2c3 [ 2379.164881] [] try_to_free_pages+0x123/0x21f [ 2379.170861] [] __alloc_pages_nodemask+0x4c9/0x978 [ 2379.177292] [] ? get_partial_node.isra.19+0x353/0x3af [ 2379.184026] [] new_slab+0xbc/0x3bb [ 2379.189103] [] ___slab_alloc.constprop.22+0x2fb/0x37b [ 2379.195843] [] ? mempool_alloc_slab+0x15/0x17 [ 2379.201895] [] ? glue_xts_crypt_128bit+0x1a6/0x1d8 [ 2379.208357] [] ? sched_clock+0x9/0xd [ 2379.213610] [] ? local_clock+0x20/0x22 [ 2379.219050] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2379.225596] [] __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2379.232769] [] ? __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2379.240143] [] ? mempool_alloc_slab+0x15/0x17 [ 2379.246177] [] kmem_cache_alloc+0xa0/0x1d6 [ 2379.251957] [] ? mempool_alloc_slab+0x15/0x17 [ 2379.258024] [] mempool_alloc_slab+0x15/0x17 [ 2379.263907] [] mempool_alloc+0x72/0x154 [ 2379.269403] [] ? lockdep_init_map+0xc9/0x5a3 [ 2379.275354] [] ? local_clock+0x20/0x22 [ 2379.280754] [] bio_alloc_bioset+0xe8/0x1d7 [ 2379.286535] [] kcryptd_crypt+0x1ab/0x325 [ 2379.292143] [] ? process_one_work+0x1ad/0x4e2 [ 2379.298208] [] process_one_work+0x283/0x4e2 [ 2379.304117] [] ? put_lock_stats.isra.9+0xe/0x20 [ 2379.310341] [] worker_thread+0x285/0x370 [ 2379.315946] [] ? rescuer_thread+0x2d1/0x2d1 [ 2379.321840] [] kthread+0xff/0x107 [ 2379.326825] [] ret_from_fork+0x22/0x50 [ 2379.332299] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2385.193584] kworker/u8:1 D ffff880022e634b8 0 2342 2 0x00000000 [ 2385.200692] Workqueue: kcryptd kcryptd_crypt [ 2385.205023] ffff880022e634b8 00ff88011b3ccd80 ffff88011b3d62d8 ffff88011ae45200 [ 2385.212554] ffff88011a472900 ffff880022e64000 0000000100098b0a ffff880022e634f0 [ 2385.220052] ffff88011b3ccd80 ffff8800c5b19350 ffff880022e634d0 ffffffff81845cec [ 2385.227547] Call Trace: [ 2385.230002] [] schedule+0x8b/0xa3 [ 2385.235001] [] schedule_timeout+0x20b/0x285 [ 2385.240893] [] ? init_timer_key+0x112/0x112 [ 2385.246787] [] schedule_timeout_uninterruptible+0x1e/0x20 [ 2385.253858] [] ? schedule_timeout_uninterruptible+0x1e/0x20 [ 2385.261140] [] wait_iff_congested+0x92/0x1b4 [ 2385.267083] [] ? wait_woken+0x72/0x72 [ 2385.272448] [] shrink_inactive_list+0x3dc/0x4a1 [ 2385.278662] [] shrink_zone_memcg+0x4c1/0x661 [ 2385.284643] [] shrink_zone+0xdc/0x1e5 [ 2385.290006] [] ? shrink_zone+0xdc/0x1e5 [ 2385.295518] [] do_try_to_free_pages+0x1a5/0x2c3 [ 2385.301739] [] try_to_free_pages+0x123/0x21f [ 2385.307710] [] __alloc_pages_nodemask+0x4c9/0x978 [ 2385.314097] [] ? debug_smp_processor_id+0x17/0x19 [ 2385.320509] [] new_slab+0xbc/0x3bb [ 2385.325598] [] ___slab_alloc.constprop.22+0x2fb/0x37b [ 2385.332320] [] ? debug_smp_processor_id+0x17/0x19 [ 2385.338726] [] ? mempool_alloc_slab+0x15/0x17 [ 2385.344784] [] ? sched_clock+0x9/0xd [ 2385.350052] [] ? local_clock+0x20/0x22 [ 2385.355494] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2385.362063] [] __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2385.369247] [] ? __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2385.376630] [] ? mempool_alloc_slab+0x15/0x17 [ 2385.382689] [] kmem_cache_alloc+0xa0/0x1d6 [ 2385.388493] [] ? mempool_alloc_slab+0x15/0x17 [ 2385.394544] [] mempool_alloc_slab+0x15/0x17 [ 2385.400410] [] mempool_alloc+0x72/0x154 [ 2385.405915] [] ? lockdep_init_map+0xc9/0x5a3 [ 2385.411875] [] ? local_clock+0x20/0x22 [ 2385.417311] [] bio_alloc_bioset+0xe8/0x1d7 [ 2385.423082] [] kcryptd_crypt+0x1ab/0x325 [ 2385.428704] [] ? process_one_work+0x1ad/0x4e2 [ 2385.434771] [] process_one_work+0x283/0x4e2 [ 2385.440664] [] ? put_lock_stats.isra.9+0xe/0x20 [ 2385.446904] [] worker_thread+0x285/0x370 [ 2385.452510] [] ? rescuer_thread+0x2d1/0x2d1 [ 2385.458385] [] kthread+0xff/0x107 [ 2385.463379] [] ret_from_fork+0x22/0x50 [ 2385.468776] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2386.089621] kworker/u8:0 D ffff88010cd434b8 0 15543 2 0x00000000 [ 2386.096770] Workqueue: kcryptd kcryptd_crypt [ 2386.101060] ffff88010cd434b8 00ff88011b1ccd80 ffff88011b1d62d8 ffffffff81e1d540 [ 2386.108598] ffff8800c00ca900 ffff88010cd44000 00000001000982fe ffff88010cd434f0 [ 2386.116102] ffff88011b1ccd80 ffff8800c5b19350 ffff88010cd434d0 ffffffff81845cec [ 2386.123651] Call Trace: [ 2386.126132] [] schedule+0x8b/0xa3 [ 2386.131167] [] schedule_timeout+0x20b/0x285 [ 2386.137017] [] ? init_timer_key+0x112/0x112 [ 2386.142902] [] schedule_timeout_uninterruptible+0x1e/0x20 [ 2386.149999] [] ? schedule_timeout_uninterruptible+0x1e/0x20 [ 2386.157271] [] wait_iff_congested+0x92/0x1b4 [ 2386.163258] [] ? wait_woken+0x72/0x72 [ 2386.168622] [] shrink_inactive_list+0x3dc/0x4a1 [ 2386.174862] [] shrink_zone_memcg+0x4c1/0x661 [ 2386.180834] [] shrink_zone+0xdc/0x1e5 [ 2386.186154] [] ? shrink_zone+0xdc/0x1e5 [ 2386.191691] [] do_try_to_free_pages+0x1a5/0x2c3 [ 2386.197931] [] try_to_free_pages+0x123/0x21f [ 2386.203893] [] __alloc_pages_nodemask+0x4c9/0x978 [ 2386.210314] [] ? get_partial_node.isra.19+0x353/0x3af [ 2386.217057] [] new_slab+0x129/0x3bb [ 2386.222246] [] ___slab_alloc.constprop.22+0x2fb/0x37b [ 2386.228979] [] ? mempool_alloc_slab+0x15/0x17 [ 2386.235039] [] ? glue_xts_crypt_128bit+0x1a6/0x1d8 [ 2386.241529] [] ? sched_clock+0x9/0xd [ 2386.246790] [] ? local_clock+0x20/0x22 [ 2386.252248] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2386.258863] [] __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2386.266027] [] ? __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2386.273358] [] ? mempool_alloc_slab+0x15/0x17 [ 2386.279383] [] kmem_cache_alloc+0xa0/0x1d6 [ 2386.285196] [] ? mempool_alloc_slab+0x15/0x17 [ 2386.291246] [] mempool_alloc_slab+0x15/0x17 [ 2386.297146] [] mempool_alloc+0x72/0x154 [ 2386.302669] [] ? lockdep_init_map+0xc9/0x5a3 [ 2386.308622] [] ? local_clock+0x20/0x22 [ 2386.314073] [] bio_alloc_bioset+0xe8/0x1d7 [ 2386.319843] [] kcryptd_crypt+0x1ab/0x325 [ 2386.325443] [] ? process_one_work+0x1ad/0x4e2 [ 2386.331482] [] process_one_work+0x283/0x4e2 [ 2386.337331] [] ? put_lock_stats.isra.9+0xe/0x20 [ 2386.343528] [] worker_thread+0x285/0x370 [ 2386.349143] [] ? rescuer_thread+0x2d1/0x2d1 [ 2386.355055] [] kthread+0xff/0x107 [ 2386.360048] [] ret_from_fork+0x22/0x50 [ 2386.365471] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2419.047134] workqueue kcryptd: flags=0x2a [ 2419.051178] pwq 8: cpus=0-3 flags=0x4 nice=0 active=4/4 [ 2419.056687] in-flight: 1592:kcryptd_crypt, 1594:kcryptd_crypt, 2342:kcryptd_crypt, 15543:kcryptd_crypt [ 2419.066479] delayed: kcryptd_crypt, kcryptd_crypt, (...snipped...) kcryptd_crypt, (...too long to finish...) Why can't we stop queuing so many kcryptd_crypt work requests? From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-io0-f199.google.com (mail-io0-f199.google.com [209.85.223.199]) by kanga.kvack.org (Postfix) with ESMTP id 0E3F16B0005 for ; Fri, 15 Jul 2016 07:42:59 -0400 (EDT) Received: by mail-io0-f199.google.com with SMTP id r71so207598487ioi.3 for ; Fri, 15 Jul 2016 04:42:59 -0700 (PDT) Received: from www262.sakura.ne.jp (www262.sakura.ne.jp. [2001:e42:101:1:202:181:97:72]) by mx.google.com with ESMTPS id a28si6802640ote.165.2016.07.15.04.42.57 for (version=TLS1 cipher=AES128-SHA bits=128/128); Fri, 15 Jul 2016 04:42:57 -0700 (PDT) Subject: Re: System freezes after OOM References: <57837CEE.1010609@redhat.com> <9be09452-de7f-d8be-fd5d-4a80d1cd1ba3@redhat.com> <20160712064905.GA14586@dhcp22.suse.cz> <20160713111006.GF28723@dhcp22.suse.cz> <20160714125129.GA12289@dhcp22.suse.cz> <740b17f0-e1bb-b021-e9e1-ad6dcf5f033a@redhat.com> <20160714153120.GD12289@dhcp22.suse.cz> <9ca3459a-8226-b870-163e-58e2bb10df74@redhat.com> From: Tetsuo Handa Message-ID: <747214d9-be8c-2bbb-9b19-147541b3d439@I-love.SAKURA.ne.jp> Date: Fri, 15 Jul 2016 20:42:40 +0900 MIME-Version: 1.0 In-Reply-To: <9ca3459a-8226-b870-163e-58e2bb10df74@redhat.com> Content-Type: text/plain; charset=windows-1252 Content-Transfer-Encoding: 7bit Sender: owner-linux-mm@kvack.org List-ID: To: Ondrej Kozina , Michal Hocko Cc: Mikulas Patocka , Jerome Marchand , Stanislav Kozina , linux-mm@kvack.org, linux-kernel@vger.kernel.org, dm-devel@redhat.com On 2016/07/15 2:07, Ondrej Kozina wrote: > On 07/14/2016 05:31 PM, Michal Hocko wrote: >> On Thu 14-07-16 16:08:28, Ondrej Kozina wrote: >> [...] >>> As Mikulas pointed out, this doesn't work. The system froze as well with the >>> patch above. Will try to tweak the patch with Mikulas's suggestion... >> >> Thank you for testing! Do you happen to have traces of the frozen >> processes? Does the flusher still gets throttled because the bias it >> gets is not sufficient. Or does it get throttled at a different place? >> > > Sure. Here it is (including sysrq+t and sysrq+w output): https://okozina.fedorapeople.org/bugs/swap_on_dmcrypt/4.7.0-rc7+/1/4.7.0-rc7+.log > Oh, this resembles another dm-crypt lockup problem reported last month. ( http://lkml.kernel.org/r/20160616212641.GA3308@sig21.net ) In Johannes's case, there are so many pending kcryptd_crypt work requests and mempool_alloc() is waiting at throttle_vm_writeout() or shrink_inactive_list(). [ 2378.279029] kswapd0 D ffff88003744f538 0 766 2 0x00000000 [ 2378.286167] ffff88003744f538 00ff88011b5ccd80 ffff88011b5d62d8 ffff88011ae58000 [ 2378.293628] ffff880037450000 ffff880037450000 00000001000984f2 ffff88003744f570 [ 2378.301168] ffff88011b5ccd80 ffff880037450000 ffff88003744f550 ffffffff81845cec [ 2378.308674] Call Trace: [ 2378.311154] [] schedule+0x8b/0xa3 [ 2378.316153] [] schedule_timeout+0x20b/0x285 [ 2378.322028] [] ? init_timer_key+0x112/0x112 [ 2378.327931] [] io_schedule_timeout+0xa0/0x102 [ 2378.333960] [] ? io_schedule_timeout+0xa0/0x102 [ 2378.340166] [] mempool_alloc+0x123/0x154 [ 2378.345781] [] ? wait_woken+0x72/0x72 [ 2378.351148] [] bio_alloc_bioset+0xe8/0x1d7 [ 2378.356910] [] alloc_tio+0x2d/0x47 [ 2378.361996] [] __split_and_process_bio+0x310/0x3a3 [ 2378.368470] [] dm_make_request+0xb5/0xe2 [ 2378.374078] [] generic_make_request+0xcc/0x180 [ 2378.380206] [] submit_bio+0xfd/0x145 [ 2378.385482] [] __swap_writepage+0x202/0x225 [ 2378.391349] [] ? preempt_count_sub+0xf0/0x100 [ 2378.397398] [] ? _raw_spin_unlock+0x31/0x44 [ 2378.403273] [] ? page_swapcount+0x45/0x4c [ 2378.408984] [] swap_writepage+0x3a/0x3e [ 2378.414530] [] pageout.isra.16+0x160/0x2a7 [ 2378.420320] [] shrink_page_list+0x5a0/0x8c4 [ 2378.426197] [] shrink_inactive_list+0x29e/0x4a1 [ 2378.432434] [] shrink_zone_memcg+0x4c1/0x661 [ 2378.438406] [] shrink_zone+0xdc/0x1e5 [ 2378.443742] [] ? shrink_zone+0xdc/0x1e5 [ 2378.449238] [] kswapd+0x6df/0x814 [ 2378.454222] [] ? mem_cgroup_shrink_node_zone+0x209/0x209 [ 2378.461196] [] kthread+0xff/0x107 [ 2378.466182] [] ret_from_fork+0x22/0x50 [ 2378.471631] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2378.769494] kworker/u8:4 D ffff8800c5dc3508 0 1592 2 0x00000000 [ 2378.776582] Workqueue: kcryptd kcryptd_crypt [ 2378.780887] ffff8800c5dc3508 00ff88011b7ccd80 ffff88011b7d62d8 ffff88011ae5a900 [ 2378.788399] ffff88011a605200 ffff8800c5dc4000 00000001000983f7 ffff8800c5dc3540 [ 2378.795930] ffff88011b7ccd80 0000000000000000 ffff8800c5dc3520 ffffffff81845cec [ 2378.803408] Call Trace: [ 2378.805879] [] schedule+0x8b/0xa3 [ 2378.810908] [] schedule_timeout+0x20b/0x285 [ 2378.816783] [] ? init_timer_key+0x112/0x112 [ 2378.822677] [] io_schedule_timeout+0xa0/0x102 [ 2378.828716] [] ? io_schedule_timeout+0xa0/0x102 [ 2378.834956] [] congestion_wait+0x84/0x160 [ 2378.840658] [] ? wait_woken+0x72/0x72 [ 2378.845997] [] throttle_vm_writeout+0x88/0xab [ 2378.852036] [] shrink_zone_memcg+0x635/0x661 [ 2378.857982] [] shrink_zone+0xdc/0x1e5 [ 2378.863309] [] ? shrink_zone+0xdc/0x1e5 [ 2378.868832] [] do_try_to_free_pages+0x1a5/0x2c3 [ 2378.875028] [] try_to_free_pages+0x123/0x21f [ 2378.880972] [] __alloc_pages_nodemask+0x4c9/0x978 [ 2378.887385] [] ? debug_smp_processor_id+0x17/0x19 [ 2378.893782] [] new_slab+0xbc/0x3bb [ 2378.898868] [] ___slab_alloc.constprop.22+0x2fb/0x37b [ 2378.905634] [] ? mempool_alloc_slab+0x15/0x17 [ 2378.911659] [] ? sched_clock+0x9/0xd [ 2378.916909] [] ? local_clock+0x20/0x22 [ 2378.922325] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2378.928877] [] ? sched_clock+0x9/0xd [ 2378.934138] [] ? local_clock+0x20/0x22 [ 2378.939555] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2378.946125] [] __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2378.953289] [] ? __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2378.960630] [] ? mempool_alloc_slab+0x15/0x17 [ 2378.966706] [] kmem_cache_alloc+0xa0/0x1d6 [ 2378.972503] [] ? mempool_alloc_slab+0x15/0x17 [ 2378.978567] [] mempool_alloc_slab+0x15/0x17 [ 2378.984426] [] mempool_alloc+0x72/0x154 [ 2378.989930] [] ? lockdep_init_map+0xc9/0x5a3 [ 2378.995866] [] ? local_clock+0x20/0x22 [ 2379.001300] [] bio_alloc_bioset+0xe8/0x1d7 [ 2379.007107] [] kcryptd_crypt+0x1ab/0x325 [ 2379.012704] [] ? process_one_work+0x1ad/0x4e2 [ 2379.018753] [] process_one_work+0x283/0x4e2 [ 2379.024629] [] ? put_lock_stats.isra.9+0xe/0x20 [ 2379.030851] [] worker_thread+0x285/0x370 [ 2379.036423] [] ? rescuer_thread+0x2d1/0x2d1 [ 2379.042309] [] kthread+0xff/0x107 [ 2379.047310] [] ret_from_fork+0x22/0x50 [ 2379.052726] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2379.059328] kworker/u8:6 D ffff8800c5ec3508 0 1594 2 0x00000000 [ 2379.066468] Workqueue: kcryptd kcryptd_crypt [ 2379.070808] ffff8800c5ec3508 00ff88011b7ccd80 ffff88011b7d62d8 ffff88011ae5a900 [ 2379.078296] ffff88003749a900 ffff8800c5ec4000 0000000100098467 ffff8800c5ec3540 [ 2379.085836] ffff88011b7ccd80 0000000000000000 ffff8800c5ec3520 ffffffff81845cec [ 2379.093315] Call Trace: [ 2379.095776] [] schedule+0x8b/0xa3 [ 2379.100785] [] schedule_timeout+0x20b/0x285 [ 2379.106627] [] ? init_timer_key+0x112/0x112 [ 2379.112494] [] io_schedule_timeout+0xa0/0x102 [ 2379.118524] [] ? io_schedule_timeout+0xa0/0x102 [ 2379.124740] [] congestion_wait+0x84/0x160 [ 2379.130432] [] ? wait_woken+0x72/0x72 [ 2379.135771] [] throttle_vm_writeout+0x88/0xab [ 2379.141839] [] shrink_zone_memcg+0x635/0x661 [ 2379.147810] [] shrink_zone+0xdc/0x1e5 [ 2379.153155] [] ? shrink_zone+0xdc/0x1e5 [ 2379.158651] [] do_try_to_free_pages+0x1a5/0x2c3 [ 2379.164881] [] try_to_free_pages+0x123/0x21f [ 2379.170861] [] __alloc_pages_nodemask+0x4c9/0x978 [ 2379.177292] [] ? get_partial_node.isra.19+0x353/0x3af [ 2379.184026] [] new_slab+0xbc/0x3bb [ 2379.189103] [] ___slab_alloc.constprop.22+0x2fb/0x37b [ 2379.195843] [] ? mempool_alloc_slab+0x15/0x17 [ 2379.201895] [] ? glue_xts_crypt_128bit+0x1a6/0x1d8 [ 2379.208357] [] ? sched_clock+0x9/0xd [ 2379.213610] [] ? local_clock+0x20/0x22 [ 2379.219050] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2379.225596] [] __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2379.232769] [] ? __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2379.240143] [] ? mempool_alloc_slab+0x15/0x17 [ 2379.246177] [] kmem_cache_alloc+0xa0/0x1d6 [ 2379.251957] [] ? mempool_alloc_slab+0x15/0x17 [ 2379.258024] [] mempool_alloc_slab+0x15/0x17 [ 2379.263907] [] mempool_alloc+0x72/0x154 [ 2379.269403] [] ? lockdep_init_map+0xc9/0x5a3 [ 2379.275354] [] ? local_clock+0x20/0x22 [ 2379.280754] [] bio_alloc_bioset+0xe8/0x1d7 [ 2379.286535] [] kcryptd_crypt+0x1ab/0x325 [ 2379.292143] [] ? process_one_work+0x1ad/0x4e2 [ 2379.298208] [] process_one_work+0x283/0x4e2 [ 2379.304117] [] ? put_lock_stats.isra.9+0xe/0x20 [ 2379.310341] [] worker_thread+0x285/0x370 [ 2379.315946] [] ? rescuer_thread+0x2d1/0x2d1 [ 2379.321840] [] kthread+0xff/0x107 [ 2379.326825] [] ret_from_fork+0x22/0x50 [ 2379.332299] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2385.193584] kworker/u8:1 D ffff880022e634b8 0 2342 2 0x00000000 [ 2385.200692] Workqueue: kcryptd kcryptd_crypt [ 2385.205023] ffff880022e634b8 00ff88011b3ccd80 ffff88011b3d62d8 ffff88011ae45200 [ 2385.212554] ffff88011a472900 ffff880022e64000 0000000100098b0a ffff880022e634f0 [ 2385.220052] ffff88011b3ccd80 ffff8800c5b19350 ffff880022e634d0 ffffffff81845cec [ 2385.227547] Call Trace: [ 2385.230002] [] schedule+0x8b/0xa3 [ 2385.235001] [] schedule_timeout+0x20b/0x285 [ 2385.240893] [] ? init_timer_key+0x112/0x112 [ 2385.246787] [] schedule_timeout_uninterruptible+0x1e/0x20 [ 2385.253858] [] ? schedule_timeout_uninterruptible+0x1e/0x20 [ 2385.261140] [] wait_iff_congested+0x92/0x1b4 [ 2385.267083] [] ? wait_woken+0x72/0x72 [ 2385.272448] [] shrink_inactive_list+0x3dc/0x4a1 [ 2385.278662] [] shrink_zone_memcg+0x4c1/0x661 [ 2385.284643] [] shrink_zone+0xdc/0x1e5 [ 2385.290006] [] ? shrink_zone+0xdc/0x1e5 [ 2385.295518] [] do_try_to_free_pages+0x1a5/0x2c3 [ 2385.301739] [] try_to_free_pages+0x123/0x21f [ 2385.307710] [] __alloc_pages_nodemask+0x4c9/0x978 [ 2385.314097] [] ? debug_smp_processor_id+0x17/0x19 [ 2385.320509] [] new_slab+0xbc/0x3bb [ 2385.325598] [] ___slab_alloc.constprop.22+0x2fb/0x37b [ 2385.332320] [] ? debug_smp_processor_id+0x17/0x19 [ 2385.338726] [] ? mempool_alloc_slab+0x15/0x17 [ 2385.344784] [] ? sched_clock+0x9/0xd [ 2385.350052] [] ? local_clock+0x20/0x22 [ 2385.355494] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2385.362063] [] __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2385.369247] [] ? __slab_alloc.isra.17.constprop.21+0x57/0x8b [ 2385.376630] [] ? mempool_alloc_slab+0x15/0x17 [ 2385.382689] [] kmem_cache_alloc+0xa0/0x1d6 [ 2385.388493] [] ? mempool_alloc_slab+0x15/0x17 [ 2385.394544] [] mempool_alloc_slab+0x15/0x17 [ 2385.400410] [] mempool_alloc+0x72/0x154 [ 2385.405915] [] ? lockdep_init_map+0xc9/0x5a3 [ 2385.411875] [] ? local_clock+0x20/0x22 [ 2385.417311] [] bio_alloc_bioset+0xe8/0x1d7 [ 2385.423082] [] kcryptd_crypt+0x1ab/0x325 [ 2385.428704] [] ? process_one_work+0x1ad/0x4e2 [ 2385.434771] [] process_one_work+0x283/0x4e2 [ 2385.440664] [] ? put_lock_stats.isra.9+0xe/0x20 [ 2385.446904] [] worker_thread+0x285/0x370 [ 2385.452510] [] ? rescuer_thread+0x2d1/0x2d1 [ 2385.458385] [] kthread+0xff/0x107 [ 2385.463379] [] ret_from_fork+0x22/0x50 [ 2385.468776] [] ? kthread_create_on_node+0x1ea/0x1ea [ 2386.089621] kworker/u8:0 D ffff88010cd434b8 0 15543 2 0x00000000 [ 2386.096770] Workqueue: kcryptd kcryptd_crypt [ 2386.101060] ffff88010cd434b8 00ff88011b1ccd80 ffff88011b1d62d8 ffffffff81e1d540 [ 2386.108598] ffff8800c00ca900 ffff88010cd44000 00000001000982fe ffff88010cd434f0 [ 2386.116102] ffff88011b1ccd80 ffff8800c5b19350 ffff88010cd434d0 ffffffff81845cec [ 2386.123651] Call Trace: [ 2386.126132] [] schedule+0x8b/0xa3 [ 2386.131167] [] schedule_timeout+0x20b/0x285 [ 2386.137017] [] ? init_timer_key+0x112/0x112 [ 2386.142902] [] schedule_timeout_uninterruptible+0x1e/0x20 [ 2386.149999] [] ? schedule_timeout_uninterruptible+0x1e/0x20 [ 2386.157271] [] wait_iff_congested+0x92/0x1b4 [ 2386.163258] [] ? wait_woken+0x72/0x72 [ 2386.168622] [] shrink_inactive_list+0x3dc/0x4a1 [ 2386.174862] [] shrink_zone_memcg+0x4c1/0x661 [ 2386.180834] [] shrink_zone+0xdc/0x1e5 [ 2386.186154] [] ? shrink_zone+0xdc/0x1e5 [ 2386.191691] [] do_try_to_free_pages+0x1a5/0x2c3 [ 2386.197931] [] try_to_free_pages+0x123/0x21f [ 2386.203893] [] __alloc_pages_nodemask+0x4c9/0x978 [ 2386.210314] [] ? get_partial_node.isra.19+0x353/0x3af [ 2386.217057] [] new_slab+0x129/0x3bb [ 2386.222246] [] ___slab_alloc.constprop.22+0x2fb/0x37b [ 2386.228979] [] ? mempool_alloc_slab+0x15/0x17 [ 2386.235039] [] ? glue_xts_crypt_128bit+0x1a6/0x1d8 [ 2386.241529] [] ? sched_clock+0x9/0xd [ 2386.246790] [] ? local_clock+0x20/0x22 [ 2386.252248] [] ? __lock_acquire.isra.16+0x55e/0xb4c [ 2386.258863] [