Re: possible deadlock in shmem_fallocate (4)

[Hillf Danton <hdanton@sina.com>]

Thu, 26 Dec 2019 13:25:09 -0800
> syzbot found the following crash on:
> 
> HEAD commit:    46cf053e Linux 5.5-rc3
> git tree:       upstream
> console output: https://syzkaller.appspot.com/x/log.txt?x=162124aee00000
> kernel config:  https://syzkaller.appspot.com/x/.config?x=ed9d672709340e35
> dashboard link: https://syzkaller.appspot.com/bug?extid=7a0d9d0b26efefe61780
> compiler:       gcc (GCC) 9.0.0 20181231 (experimental)
> 
> Unfortunately, I don't have any reproducer for this crash yet.
> 
> IMPORTANT: if you fix the bug, please add the following tag to the commit:
> Reported-by: syzbot+7a0d9d0b26efefe61780@syzkaller.appspotmail.com
> 
> ======================================================
> WARNING: possible circular locking dependency detected
> 5.5.0-rc3-syzkaller #0 Not tainted
> ------------------------------------------------------
> kswapd0/1852 is trying to acquire lock:
> ffff888098919cd8 (&sb->s_type->i_mutex_key#13){+.+.}, at: inode_lock  
> include/linux/fs.h:791 [inline]
> ffff888098919cd8 (&sb->s_type->i_mutex_key#13){+.+.}, at:  
> shmem_fallocate+0x15a/0xd40 mm/shmem.c:2735
> 
> but task is already holding lock:
> ffffffff89a41e00 (fs_reclaim){+.+.}, at: __fs_reclaim_acquire+0x0/0x30  
> mm/page_alloc.c:4922
> 
> which lock already depends on the new lock.
> 
> 
> the existing dependency chain (in reverse order) is:
> 
> -> #1 (fs_reclaim){+.+.}:
>         __fs_reclaim_acquire mm/page_alloc.c:4084 [inline]
>         fs_reclaim_acquire.part.0+0x24/0x30 mm/page_alloc.c:4095
>         fs_reclaim_acquire mm/page_alloc.c:4695 [inline]
>         prepare_alloc_pages mm/page_alloc.c:4692 [inline]
>         __alloc_pages_nodemask+0x52d/0x910 mm/page_alloc.c:4744
>         alloc_pages_vma+0xdd/0x620 mm/mempolicy.c:2170
>         shmem_alloc_page+0xc0/0x180 mm/shmem.c:1499
>         shmem_alloc_and_acct_page+0x165/0x990 mm/shmem.c:1524
>         shmem_getpage_gfp+0x56d/0x29a0 mm/shmem.c:1838
>         shmem_getpage mm/shmem.c:154 [inline]
>         shmem_write_begin+0x105/0x1e0 mm/shmem.c:2487
>         generic_perform_write+0x23b/0x540 mm/filemap.c:3309
>         __generic_file_write_iter+0x25e/0x630 mm/filemap.c:3438
>         generic_file_write_iter+0x420/0x68e mm/filemap.c:3470
>         call_write_iter include/linux/fs.h:1902 [inline]
>         new_sync_write+0x4d3/0x770 fs/read_write.c:483
>         __vfs_write+0xe1/0x110 fs/read_write.c:496
>         vfs_write+0x268/0x5d0 fs/read_write.c:558
>         ksys_write+0x14f/0x290 fs/read_write.c:611
>         __do_sys_write fs/read_write.c:623 [inline]
>         __se_sys_write fs/read_write.c:620 [inline]
>         __x64_sys_write+0x73/0xb0 fs/read_write.c:620
>         do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294
>         entry_SYSCALL_64_after_hwframe+0x49/0xbe
> 
> -> #0 (&sb->s_type->i_mutex_key#13){+.+.}:
>         check_prev_add kernel/locking/lockdep.c:2476 [inline]
>         check_prevs_add kernel/locking/lockdep.c:2581 [inline]
>         validate_chain kernel/locking/lockdep.c:2971 [inline]
>         __lock_acquire+0x2596/0x4a00 kernel/locking/lockdep.c:3955
>         lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4485
>         down_write+0x93/0x150 kernel/locking/rwsem.c:1534
>         inode_lock include/linux/fs.h:791 [inline]
>         shmem_fallocate+0x15a/0xd40 mm/shmem.c:2735
>         ashmem_shrink_scan drivers/staging/android/ashmem.c:462 [inline]
>         ashmem_shrink_scan+0x370/0x510 drivers/staging/android/ashmem.c:437
>         do_shrink_slab+0x40f/0xad0 mm/vmscan.c:526
>         shrink_slab mm/vmscan.c:687 [inline]
>         shrink_slab+0x19a/0x680 mm/vmscan.c:660
>         shrink_node_memcgs mm/vmscan.c:2687 [inline]
>         shrink_node+0x46a/0x1ad0 mm/vmscan.c:2791
>         kswapd_shrink_node mm/vmscan.c:3539 [inline]
>         balance_pgdat+0x7c8/0x11f0 mm/vmscan.c:3697
>         kswapd+0x5c3/0xf30 mm/vmscan.c:3948
>         kthread+0x361/0x430 kernel/kthread.c:255
>         ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352
> 
> other info that might help us debug this:
> 
>   Possible unsafe locking scenario:
> 
>         CPU0                    CPU1
>         ----                    ----
>    lock(fs_reclaim);
>                                 lock(&sb->s_type->i_mutex_key#13);
>                                 lock(fs_reclaim);
>    lock(&sb->s_type->i_mutex_key#13);
> 
>   *** DEADLOCK ***
> 
> 2 locks held by kswapd0/1852:
>   #0: ffffffff89a41e00 (fs_reclaim){+.+.}, at: __fs_reclaim_acquire+0x0/0x30  mm/page_alloc.c:4922
>   #1: ffffffff89a1f948 (shrinker_rwsem){++++}, at: shrink_slab   mm/vmscan.c:677 [inline]
>   #1: ffffffff89a1f948 (shrinker_rwsem){++++}, at: shrink_slab+0xe6/0x680   mm/vmscan.c:660
> 
> stack backtrace:
> CPU: 0 PID: 1852 Comm: kswapd0 Not tainted 5.5.0-rc3-syzkaller #0
> Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS  
> Google 01/01/2011
> Call Trace:
>   __dump_stack lib/dump_stack.c:77 [inline]
>   dump_stack+0x197/0x210 lib/dump_stack.c:118
>   print_circular_bug.isra.0.cold+0x163/0x172 kernel/locking/lockdep.c:1685
>   check_noncircular+0x32e/0x3e0 kernel/locking/lockdep.c:1809
>   check_prev_add kernel/locking/lockdep.c:2476 [inline]
>   check_prevs_add kernel/locking/lockdep.c:2581 [inline]
>   validate_chain kernel/locking/lockdep.c:2971 [inline]
>   __lock_acquire+0x2596/0x4a00 kernel/locking/lockdep.c:3955
>   lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4485
>   down_write+0x93/0x150 kernel/locking/rwsem.c:1534
>   inode_lock include/linux/fs.h:791 [inline]
>   shmem_fallocate+0x15a/0xd40 mm/shmem.c:2735
>   ashmem_shrink_scan drivers/staging/android/ashmem.c:462 [inline]
>   ashmem_shrink_scan+0x370/0x510 drivers/staging/android/ashmem.c:437
>   do_shrink_slab+0x40f/0xad0 mm/vmscan.c:526
>   shrink_slab mm/vmscan.c:687 [inline]
>   shrink_slab+0x19a/0x680 mm/vmscan.c:660
>   shrink_node_memcgs mm/vmscan.c:2687 [inline]
>   shrink_node+0x46a/0x1ad0 mm/vmscan.c:2791
>   kswapd_shrink_node mm/vmscan.c:3539 [inline]
>   balance_pgdat+0x7c8/0x11f0 mm/vmscan.c:3697
>   kswapd+0x5c3/0xf30 mm/vmscan.c:3948
>   kthread+0x361/0x430 kernel/kthread.c:255
>   ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352

Simply move punch-hole out of page reclaiming context to avoid deadlock.

--- a/drivers/staging/android/ashmem.c
+++ b/drivers/staging/android/ashmem.c
@@ -24,6 +24,7 @@
 #include <linux/bitops.h>
 #include <linux/mutex.h>
 #include <linux/shmem_fs.h>
+#include <linux/workqueue.h>
 #include "ashmem.h"
 
 #define ASHMEM_NAME_PREFIX "dev/ashmem/"
@@ -70,6 +71,7 @@ struct ashmem_range {
 	size_t pgstart;
 	size_t pgend;
 	unsigned int purged;
+	struct work_struct work;
 };
 
 /* LRU list of unpinned pages, protected by ashmem_mutex */
@@ -201,6 +203,7 @@ static void range_del(struct ashmem_rang
 	list_del(&range->unpinned);
 	if (range_on_lru(range))
 		lru_del(range);
+	flush_work(&range->work);
 	kmem_cache_free(ashmem_range_cachep, range);
 }
 
@@ -419,6 +422,21 @@ out:
 	return ret;
 }
 
+static void ashmem_shrink_workfn(struct work_struct *__work)
+{
+	struct ashmem_range *range = container_of(__work, 
+					struct ashmem_range, work);
+	loff_t start = range->pgstart * PAGE_SIZE;
+	loff_t end = (range->pgend + 1) * PAGE_SIZE;
+	struct file *f = range->asma->file;
+
+	f->f_op->fallocate(f, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+				start, end - start);
+	fput(f);
+	if (atomic_dec_and_test(&ashmem_shrink_inflight))
+		wake_up_all(&ashmem_shrink_wait);
+}
+
 /*
  * ashmem_shrink - our cache shrinker, called from mm/vmscan.c
  *
@@ -448,8 +466,6 @@ ashmem_shrink_scan(struct shrinker *shri
 	while (!list_empty(&ashmem_lru_list)) {
 		struct ashmem_range *range =
 			list_first_entry(&ashmem_lru_list, typeof(*range), lru);
-		loff_t start = range->pgstart * PAGE_SIZE;
-		loff_t end = (range->pgend + 1) * PAGE_SIZE;
 		struct file *f = range->asma->file;
 
 		get_file(f);
@@ -459,12 +475,9 @@ ashmem_shrink_scan(struct shrinker *shri
 
 		freed += range_size(range);
 		mutex_unlock(&ashmem_mutex);
-		f->f_op->fallocate(f,
-				   FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
-				   start, end - start);
-		fput(f);
-		if (atomic_dec_and_test(&ashmem_shrink_inflight))
-			wake_up_all(&ashmem_shrink_wait);
+
+		queue_work(system_unbound_wq, &range->work);
+
 		if (!mutex_trylock(&ashmem_mutex))
 			goto out;
 		if (--sc->nr_to_scan <= 0)
@@ -729,6 +742,7 @@ static int ashmem_pin_unpin(struct ashme
 		range = kmem_cache_zalloc(ashmem_range_cachep, GFP_KERNEL);
 		if (!range)
 			return -ENOMEM;
+		INIT_WORK(&range->work, ashmem_shrink_workfn);
 	}
 
 	mutex_lock(&ashmem_mutex);