Re: possible deadlock in shmem_fallocate (2)

[Dmitry Vyukov <dvyukov@google.com>]

[-- Attachment #1: Type: text/plain, Size: 8009 bytes --]

On Fri, Aug 10, 2018 at 6:18 PM Matthew Wilcox <willy@infradead.org> wrote:
>
>
> This is another ashmem lockdep splat.  Forwarding to the appropriate ashmem
> people.


Let's test Tetsuo's patch

#syz test: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
master



> On Fri, Aug 10, 2018 at 04:59:02AM -0700, syzbot wrote:
> > Hello,
> >
> > syzbot found the following crash on:
> >
> > HEAD commit:    4110b42356f3 Add linux-next specific files for 20180810
> > git tree:       linux-next
> > console output: https://syzkaller.appspot.com/x/log.txt?x=1411d6e2400000
> > kernel config:  https://syzkaller.appspot.com/x/.config?x=1d80606e3795a4f5
> > dashboard link: https://syzkaller.appspot.com/bug?extid=4b8b031b89e6b96c4b2e
> > compiler:       gcc (GCC) 8.0.1 20180413 (experimental)
> > syzkaller repro:https://syzkaller.appspot.com/x/repro.syz?x=175052f8400000
> > C reproducer:   https://syzkaller.appspot.com/x/repro.c?x=11873622400000
> >
> > IMPORTANT: if you fix the bug, please add the following tag to the commit:
> > Reported-by: syzbot+4b8b031b89e6b96c4b2e@syzkaller.appspotmail.com
> >
> > random: sshd: uninitialized urandom read (32 bytes read)
> > random: sshd: uninitialized urandom read (32 bytes read)
> > random: sshd: uninitialized urandom read (32 bytes read)
> >
> > ======================================================
> > WARNING: possible circular locking dependency detected
> > 4.18.0-rc8-next-20180810+ #36 Not tainted
> > ------------------------------------------------------
> > syz-executor900/4483 is trying to acquire lock:
> > 00000000d2bfc8fe (&sb->s_type->i_mutex_key#9){++++}, at: inode_lock
> > include/linux/fs.h:765 [inline]
> > 00000000d2bfc8fe (&sb->s_type->i_mutex_key#9){++++}, at:
> > shmem_fallocate+0x18b/0x12e0 mm/shmem.c:2602
> >
> > but task is already holding lock:
> > 0000000025208078 (ashmem_mutex){+.+.}, at: ashmem_shrink_scan+0xb4/0x630
> > drivers/staging/android/ashmem.c:448
> >
> > which lock already depends on the new lock.
> >
> >
> > the existing dependency chain (in reverse order) is:
> >
> > -> #2 (ashmem_mutex){+.+.}:
> >        __mutex_lock_common kernel/locking/mutex.c:925 [inline]
> >        __mutex_lock+0x171/0x1700 kernel/locking/mutex.c:1073
> >        mutex_lock_nested+0x16/0x20 kernel/locking/mutex.c:1088
> >        ashmem_mmap+0x55/0x520 drivers/staging/android/ashmem.c:361
> >        call_mmap include/linux/fs.h:1844 [inline]
> >        mmap_region+0xf27/0x1c50 mm/mmap.c:1762
> >        do_mmap+0xa10/0x1220 mm/mmap.c:1535
> >        do_mmap_pgoff include/linux/mm.h:2298 [inline]
> >        vm_mmap_pgoff+0x213/0x2c0 mm/util.c:357
> >        ksys_mmap_pgoff+0x4da/0x660 mm/mmap.c:1585
> >        __do_sys_mmap arch/x86/kernel/sys_x86_64.c:100 [inline]
> >        __se_sys_mmap arch/x86/kernel/sys_x86_64.c:91 [inline]
> >        __x64_sys_mmap+0xe9/0x1b0 arch/x86/kernel/sys_x86_64.c:91
> >        do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290
> >        entry_SYSCALL_64_after_hwframe+0x49/0xbe
> >
> > -> #1 (&mm->mmap_sem){++++}:
> >        __might_fault+0x155/0x1e0 mm/memory.c:4568
> >        _copy_to_user+0x30/0x110 lib/usercopy.c:25
> >        copy_to_user include/linux/uaccess.h:155 [inline]
> >        filldir+0x1ea/0x3a0 fs/readdir.c:196
> >        dir_emit_dot include/linux/fs.h:3464 [inline]
> >        dir_emit_dots include/linux/fs.h:3475 [inline]
> >        dcache_readdir+0x13a/0x620 fs/libfs.c:193
> >        iterate_dir+0x48b/0x5d0 fs/readdir.c:51
> >        __do_sys_getdents fs/readdir.c:231 [inline]
> >        __se_sys_getdents fs/readdir.c:212 [inline]
> >        __x64_sys_getdents+0x29f/0x510 fs/readdir.c:212
> >        do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290
> >        entry_SYSCALL_64_after_hwframe+0x49/0xbe
> >
> > -> #0 (&sb->s_type->i_mutex_key#9){++++}:
> >        lock_acquire+0x1e4/0x540 kernel/locking/lockdep.c:3924
> >        down_write+0x8f/0x130 kernel/locking/rwsem.c:70
> >        inode_lock include/linux/fs.h:765 [inline]
> >        shmem_fallocate+0x18b/0x12e0 mm/shmem.c:2602
> >        ashmem_shrink_scan+0x236/0x630 drivers/staging/android/ashmem.c:455
> >        ashmem_ioctl+0x3ae/0x13a0 drivers/staging/android/ashmem.c:797
> >        vfs_ioctl fs/ioctl.c:46 [inline]
> >        file_ioctl fs/ioctl.c:501 [inline]
> >        do_vfs_ioctl+0x1de/0x1720 fs/ioctl.c:685
> >        ksys_ioctl+0xa9/0xd0 fs/ioctl.c:702
> >        __do_sys_ioctl fs/ioctl.c:709 [inline]
> >        __se_sys_ioctl fs/ioctl.c:707 [inline]
> >        __x64_sys_ioctl+0x73/0xb0 fs/ioctl.c:707
> >        do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290
> >        entry_SYSCALL_64_after_hwframe+0x49/0xbe
> >
> > other info that might help us debug this:
> >
> > Chain exists of:
> >   &sb->s_type->i_mutex_key#9 --> &mm->mmap_sem --> ashmem_mutex
> >
> >  Possible unsafe locking scenario:
> >
> >        CPU0                    CPU1
> >        ----                    ----
> >   lock(ashmem_mutex);
> >                                lock(&mm->mmap_sem);
> >                                lock(ashmem_mutex);
> >   lock(&sb->s_type->i_mutex_key#9);
> >
> >  *** DEADLOCK ***
> >
> > 1 lock held by syz-executor900/4483:
> >  #0: 0000000025208078 (ashmem_mutex){+.+.}, at:
> > ashmem_shrink_scan+0xb4/0x630 drivers/staging/android/ashmem.c:448
> >
> > stack backtrace:
> > CPU: 1 PID: 4483 Comm: syz-executor900 Not tainted 4.18.0-rc8-next-20180810+
> > #36
> > Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS
> > Google 01/01/2011
> > Call Trace:
> >  __dump_stack lib/dump_stack.c:77 [inline]
> >  dump_stack+0x1c9/0x2b4 lib/dump_stack.c:113
> >  print_circular_bug.isra.37.cold.58+0x1bd/0x27d
> > kernel/locking/lockdep.c:1227
> >  check_prev_add kernel/locking/lockdep.c:1867 [inline]
> >  check_prevs_add kernel/locking/lockdep.c:1980 [inline]
> >  validate_chain kernel/locking/lockdep.c:2421 [inline]
> >  __lock_acquire+0x3449/0x5020 kernel/locking/lockdep.c:3435
> >  lock_acquire+0x1e4/0x540 kernel/locking/lockdep.c:3924
> >  down_write+0x8f/0x130 kernel/locking/rwsem.c:70
> >  inode_lock include/linux/fs.h:765 [inline]
> >  shmem_fallocate+0x18b/0x12e0 mm/shmem.c:2602
> >  ashmem_shrink_scan+0x236/0x630 drivers/staging/android/ashmem.c:455
> >  ashmem_ioctl+0x3ae/0x13a0 drivers/staging/android/ashmem.c:797
> >  vfs_ioctl fs/ioctl.c:46 [inline]
> >  file_ioctl fs/ioctl.c:501 [inline]
> >  do_vfs_ioctl+0x1de/0x1720 fs/ioctl.c:685
> >  ksys_ioctl+0xa9/0xd0 fs/ioctl.c:702
> >  __do_sys_ioctl fs/ioctl.c:709 [inline]
> >  __se_sys_ioctl fs/ioctl.c:707 [inline]
> >  __x64_sys_ioctl+0x73/0xb0 fs/ioctl.c:707
> >  do_syscall_64+0x1b9/0x820 arch/x86/entry/common.c:290
> >  entry_SYSCALL_64_after_hwframe+0x49/0xbe
> > RIP: 0033:0x440099
> > Code: 18 89 d0 c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 00 48 89 f8 48 89 f7
> > 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff
> > 0f 83 fb 13 fc ff c3 66 2e 0f 1f 84 00 00 00 00
> > RSP: 002b:00007fff3613dbf8 EFLAGS: 00000217 ORIG_RAX: 0000000000000010
> > RAX: ffffffffffffffda RBX: 00000000004002c8 RCX: 0000000000440099
> > RDX: 00000
> >
> >
> > ---
> > This bug is generated by a bot. It may contain errors.
> > See https://goo.gl/tpsmEJ for more information about syzbot.
> > syzbot engineers can be reached at syzkaller@googlegroups.com.
> >
> > syzbot will keep track of this bug report. See:
> > https://goo.gl/tpsmEJ#bug-status-tracking for how to communicate with
> > syzbot.
> > syzbot can test patches for this bug, for details see:
> > https://goo.gl/tpsmEJ#testing-patches
> >
>
> --
> You received this message because you are subscribed to the Google Groups "syzkaller-bugs" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to syzkaller-bugs+unsubscribe@googlegroups.com.
> To view this discussion on the web visit https://groups.google.com/d/msgid/syzkaller-bugs/20180810161848.GB16533%40bombadil.infradead.org.
> For more options, visit https://groups.google.com/d/optout.

[-- Attachment #2: ashmem.patch --]
[-- Type: text/x-patch, Size: 3279 bytes --]

diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c
index 90a8a9f1ac7d..1a890c43a10a 100644
--- a/drivers/staging/android/ashmem.c
+++ b/drivers/staging/android/ashmem.c
@@ -75,6 +75,17 @@ struct ashmem_range {
 /* LRU list of unpinned pages, protected by ashmem_mutex */
 static LIST_HEAD(ashmem_lru_list);
 
+static struct workqueue_struct *ashmem_wq;
+static atomic_t ashmem_shrink_inflight = ATOMIC_INIT(0);
+static DECLARE_WAIT_QUEUE_HEAD(ashmem_shrink_wait);
+
+struct ashmem_shrink_work {
+	struct work_struct work;
+	struct file *file;
+	loff_t start;
+	loff_t end;
+};
+
 /*
  * long lru_count - The count of pages on our LRU list.
  *
@@ -292,6 +303,7 @@ static ssize_t ashmem_read_iter(struct kiocb *iocb, struct iov_iter *iter)
 	int ret = 0;
 
 	mutex_lock(&ashmem_mutex);
+	wait_event(ashmem_shrink_wait, !atomic_read(&ashmem_shrink_inflight));
 
 	/* If size is not set, or set to 0, always return EOF. */
 	if (asma->size == 0)
@@ -359,6 +371,7 @@ static int ashmem_mmap(struct file *file, struct vm_area_struct *vma)
 	int ret = 0;
 
 	mutex_lock(&ashmem_mutex);
+	wait_event(ashmem_shrink_wait, !atomic_read(&ashmem_shrink_inflight));
 
 	/* user needs to SET_SIZE before mapping */
 	if (!asma->size) {
@@ -421,6 +434,19 @@ static int ashmem_mmap(struct file *file, struct vm_area_struct *vma)
 	return ret;
 }
 
+static void ashmem_shrink_worker(struct work_struct *work)
+{
+	struct ashmem_shrink_work *w = container_of(work, typeof(*w), work);
+
+	w->file->f_op->fallocate(w->file,
+				 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+				 w->start, w->end - w->start);
+	fput(w->file);
+	kfree(w);
+	if (atomic_dec_and_test(&ashmem_shrink_inflight))
+		wake_up_all(&ashmem_shrink_wait);
+}
+
 /*
  * ashmem_shrink - our cache shrinker, called from mm/vmscan.c
  *
@@ -449,12 +475,18 @@ ashmem_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
 		return -1;
 
 	list_for_each_entry_safe(range, next, &ashmem_lru_list, lru) {
-		loff_t start = range->pgstart * PAGE_SIZE;
-		loff_t end = (range->pgend + 1) * PAGE_SIZE;
+		struct ashmem_shrink_work *w = kzalloc(sizeof(*w), GFP_ATOMIC);
+
+		if (!w)
+			break;
+		INIT_WORK(&w->work, ashmem_shrink_worker);
+		w->file = range->asma->file;
+		get_file(w->file);
+		w->start = range->pgstart * PAGE_SIZE;
+		w->end = (range->pgend + 1) * PAGE_SIZE;
+		atomic_inc(&ashmem_shrink_inflight);
+		queue_work(ashmem_wq, &w->work);
 
-		range->asma->file->f_op->fallocate(range->asma->file,
-				FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
-				start, end - start);
 		range->purged = ASHMEM_WAS_PURGED;
 		lru_del(range);
 
@@ -713,6 +745,7 @@ static int ashmem_pin_unpin(struct ashmem_area *asma, unsigned long cmd,
 		return -EFAULT;
 
 	mutex_lock(&ashmem_mutex);
+	wait_event(ashmem_shrink_wait, !atomic_read(&ashmem_shrink_inflight));
 
 	if (!asma->file)
 		goto out_unlock;
@@ -883,8 +916,15 @@ static int __init ashmem_init(void)
 		goto out_free2;
 	}
 
+	ashmem_wq = alloc_workqueue("ashmem_wq", WQ_MEM_RECLAIM, 0);
+	if (!ashmem_wq) {
+		pr_err("failed to create workqueue\n");
+		goto out_demisc;
+	}
+
 	ret = register_shrinker(&ashmem_shrinker);
 	if (ret) {
+		destroy_workqueue(ashmem_wq);
 		pr_err("failed to register shrinker!\n");
 		goto out_demisc;
 	}