[PATCH 1/6] f2fs: call f2fs_balance_fs outside of locked page

[PATCH 1/6] f2fs: call f2fs_balance_fs outside of locked page

[Jaegeuk Kim]

Otherwise, we can hit deadlock by waiting for the locked page in
move_data_block in GC.

 Thread A                     Thread B
 - do_page_mkwrite
  - f2fs_vm_page_mkwrite
   - lock_page
                              - f2fs_balance_fs
                                  - mutex_lock(gc_mutex)
                               - f2fs_gc
                                - do_garbage_collect
                                 - ra_data_block
                                  - grab_cache_page
   - f2fs_balance_fs
    - mutex_lock(gc_mutex)

Fixes: 39a8695824510 ("f2fs: refactor ->page_mkwrite() flow")
Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
---
 fs/f2fs/file.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index e7fcbd8c23f4..6cebc6681487 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -50,7 +50,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf)
 	struct page *page = vmf->page;
 	struct inode *inode = file_inode(vmf->vma->vm_file);
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-	struct dnode_of_data dn = { .node_changed = false };
+	struct dnode_of_data dn;
 	int err;
 
 	if (unlikely(f2fs_cp_error(sbi))) {
@@ -63,6 +63,9 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf)
 		goto err;
 	}
 
+	/* should do out of any locked page */
+	f2fs_balance_fs(sbi, true);
+
 	sb_start_pagefault(inode->i_sb);
 
 	f2fs_bug_on(sbi, f2fs_has_inline_data(inode));
@@ -120,8 +123,6 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf)
 out_sem:
 	up_read(&F2FS_I(inode)->i_mmap_sem);
 
-	f2fs_balance_fs(sbi, dn.node_changed);
-
 	sb_end_pagefault(inode->i_sb);
 err:
 	return block_page_mkwrite_return(err);
-- 
2.19.0.605.g01d371f741-goog

[PATCH 2/6] f2fs: declare nested quota_sem and remove unnecessary sems

[Jaegeuk Kim]

1.
f2fs_quota_sync
 -> down_read(&sbi->quota_sem)
 -> dquot_writeback_dquots
  -> f2fs_dquot_commit
   -> down_read(&sbi->quota_sem)

2.
f2fs_quota_sync
 -> down_read(&sbi->quota_sem)
  -> f2fs_write_data_pages
   -> f2fs_write_single_data_page
    -> down_write(&F2FS_I(inode)->i_sem)

f2fs_mkdir
 -> f2fs_do_add_link
   -> down_write(&F2FS_I(inode)->i_sem)
   -> f2fs_init_inode_metadata
    -> f2fs_new_node_page
     -> dquot_alloc_inode
      -> f2fs_dquot_mark_dquot_dirty
       -> down_read(&sbi->quota_sem)

Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
---
 fs/f2fs/super.c | 22 +++++-----------------
 1 file changed, 5 insertions(+), 17 deletions(-)

diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 5111e1ffe58a..15888ca02e7f 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -2156,39 +2156,30 @@ static void f2fs_truncate_quota_inode_pages(struct super_block *sb)
 static int f2fs_dquot_commit(struct dquot *dquot)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
-	int ret;
+	int ret = dquot_commit(dquot);
 
-	down_read(&sbi->quota_sem);
-	ret = dquot_commit(dquot);
 	if (ret < 0)
 		set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
-	up_read(&sbi->quota_sem);
 	return ret;
 }
 
 static int f2fs_dquot_acquire(struct dquot *dquot)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
-	int ret;
+	int ret = dquot_acquire(dquot);
 
-	down_read(&sbi->quota_sem);
-	ret = dquot_acquire(dquot);
 	if (ret < 0)
 		set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
-	up_read(&sbi->quota_sem);
 	return ret;
 }
 
 static int f2fs_dquot_release(struct dquot *dquot)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
-	int ret;
+	int ret = dquot_release(dquot);
 
-	down_read(&sbi->quota_sem);
-	ret = dquot_release(dquot);
 	if (ret < 0)
 		set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
-	up_read(&sbi->quota_sem);
 	return ret;
 }
 
@@ -2198,7 +2189,7 @@ static int f2fs_dquot_mark_dquot_dirty(struct dquot *dquot)
 	struct f2fs_sb_info *sbi = F2FS_SB(sb);
 	int ret;
 
-	down_read(&sbi->quota_sem);
+	down_read_nested(&sbi->quota_sem, SINGLE_DEPTH_NESTING);
 	ret = dquot_mark_dquot_dirty(dquot);
 
 	/* if we are using journalled quota */
@@ -2212,13 +2203,10 @@ static int f2fs_dquot_mark_dquot_dirty(struct dquot *dquot)
 static int f2fs_dquot_commit_info(struct super_block *sb, int type)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(sb);
-	int ret;
+	int ret = dquot_commit_info(sb, type);
 
-	down_read(&sbi->quota_sem);
-	ret = dquot_commit_info(sb, type);
 	if (ret < 0)
 		set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
-	up_read(&sbi->quota_sem);
 	return ret;
 }
 
-- 
2.19.0.605.g01d371f741-goog

Re: [f2fs-dev] [PATCH 2/6] f2fs: declare nested quota_sem and remove unnecessary sems

[Chao Yu]

On 2019/12/10 6:23, Jaegeuk Kim wrote:
> 1.
> f2fs_quota_sync
>  -> down_read(&sbi->quota_sem)
>  -> dquot_writeback_dquots
>   -> f2fs_dquot_commit
>    -> down_read(&sbi->quota_sem)
> 
> 2.
> f2fs_quota_sync
>  -> down_read(&sbi->quota_sem)
>   -> f2fs_write_data_pages
>    -> f2fs_write_single_data_page
>     -> down_write(&F2FS_I(inode)->i_sem)
> 
> f2fs_mkdir
>  -> f2fs_do_add_link
>    -> down_write(&F2FS_I(inode)->i_sem)
>    -> f2fs_init_inode_metadata
>     -> f2fs_new_node_page
>      -> dquot_alloc_inode
>       -> f2fs_dquot_mark_dquot_dirty
>        -> down_read(&sbi->quota_sem)
> 
> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>

Reviewed-by: Chao Yu <yuchao0@huawei.com>

Thanks,

[PATCH 3/6] f2fs: keep quota data on write_begin failure

[Jaegeuk Kim]

This patch avoids some unnecessary locks for quota files when write_begin
fails.

Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
---
 fs/f2fs/data.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index fc40a72f7827..3b2945121557 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -2600,14 +2600,16 @@ static void f2fs_write_failed(struct address_space *mapping, loff_t to)
 	struct inode *inode = mapping->host;
 	loff_t i_size = i_size_read(inode);
 
+	if (IS_NOQUOTA(inode))
+		return;
+
 	/* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
 	if (to > i_size && !f2fs_verity_in_progress(inode)) {
 		down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
 		down_write(&F2FS_I(inode)->i_mmap_sem);
 
 		truncate_pagecache(inode, i_size);
-		if (!IS_NOQUOTA(inode))
-			f2fs_truncate_blocks(inode, i_size, true);
+		f2fs_truncate_blocks(inode, i_size, true);
 
 		up_write(&F2FS_I(inode)->i_mmap_sem);
 		up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
-- 
2.19.0.605.g01d371f741-goog

Re: [f2fs-dev] [PATCH 3/6] f2fs: keep quota data on write_begin failure

[Chao Yu]

On 2019/12/10 6:23, Jaegeuk Kim wrote:
> This patch avoids some unnecessary locks for quota files when write_begin
> fails.
> 
> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>

Reviewed-by: Chao Yu <yuchao0@huawei.com>

Thanks,

[PATCH 4/6] f2fs: should avoid recursive filesystem ops

[Jaegeuk Kim]

We need to use GFP_NOFS, since we did f2fs_lock_op().

Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
---
 fs/f2fs/file.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index 6cebc6681487..eb653f700ade 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -1191,13 +1191,13 @@ static int __exchange_data_block(struct inode *src_inode,
 
 		src_blkaddr = f2fs_kvzalloc(F2FS_I_SB(src_inode),
 					array_size(olen, sizeof(block_t)),
-					GFP_KERNEL);
+					GFP_NOFS);
 		if (!src_blkaddr)
 			return -ENOMEM;
 
 		do_replace = f2fs_kvzalloc(F2FS_I_SB(src_inode),
 					array_size(olen, sizeof(int)),
-					GFP_KERNEL);
+					GFP_NOFS);
 		if (!do_replace) {
 			kvfree(src_blkaddr);
 			return -ENOMEM;
-- 
2.19.0.605.g01d371f741-goog

Re: [f2fs-dev] [PATCH 4/6] f2fs: should avoid recursive filesystem ops

[Chao Yu]

On 2019/12/10 6:23, Jaegeuk Kim wrote:
> We need to use GFP_NOFS, since we did f2fs_lock_op().
> 
> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>

Reviewed-by: Chao Yu <yuchao0@huawei.com>

Thanks,

[PATCH 5/6] f2fs: set GFP_NOFS when moving inline dentries

[Jaegeuk Kim]

Otherwise, it can cause circular locking dependency reported by mm.

Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
---
 fs/f2fs/inline.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
index 896db0416f0e..52f85ed07a15 100644
--- a/fs/f2fs/inline.c
+++ b/fs/f2fs/inline.c
@@ -368,7 +368,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct page *ipage,
 	struct f2fs_dentry_ptr src, dst;
 	int err;
 
-	page = f2fs_grab_cache_page(dir->i_mapping, 0, false);
+	page = f2fs_grab_cache_page(dir->i_mapping, 0, true);
 	if (!page) {
 		f2fs_put_page(ipage, 1);
 		return -ENOMEM;
-- 
2.19.0.605.g01d371f741-goog

Re: [f2fs-dev] [PATCH 5/6] f2fs: set GFP_NOFS when moving inline dentries

[Chao Yu]

On 2019/12/10 6:23, Jaegeuk Kim wrote:
> Otherwise, it can cause circular locking dependency reported by mm.
> 
> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>

Reviewed-by: Chao Yu <yuchao0@huawei.com>

Thanks,

[PATCH 6/6] f2fs: set I_LINKABLE early to avoid wrong access by vfs

[Jaegeuk Kim]

This patch moves setting I_LINKABLE early in rename2(whiteout) to avoid the
below warning.

[ 3189.163385] WARNING: CPU: 3 PID: 59523 at fs/inode.c:358 inc_nlink+0x32/0x40
[ 3189.246979] Call Trace:
[ 3189.248707]  f2fs_init_inode_metadata+0x2d6/0x440 [f2fs]
[ 3189.251399]  f2fs_add_inline_entry+0x162/0x8c0 [f2fs]
[ 3189.254010]  f2fs_add_dentry+0x69/0xe0 [f2fs]
[ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
[ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
[ 3189.261079]  vfs_rename+0x3f8/0xaa0
[ 3189.263056]  ? tomoyo_path_rename+0x44/0x60
[ 3189.265283]  ? do_renameat2+0x49b/0x550
[ 3189.267324]  do_renameat2+0x49b/0x550
[ 3189.269316]  __x64_sys_renameat2+0x20/0x30
[ 3189.271441]  do_syscall_64+0x5a/0x230
[ 3189.273410]  entry_SYSCALL_64_after_hwframe+0x49/0xbe
[ 3189.275848] RIP: 0033:0x7f270b4d9a49

Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
---
 fs/f2fs/namei.c | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index a1c507b0b4ac..5d9584281935 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -797,6 +797,7 @@ static int __f2fs_tmpfile(struct inode *dir, struct dentry *dentry,
 
 	if (whiteout) {
 		f2fs_i_links_write(inode, false);
+		inode->i_state |= I_LINKABLE;
 		*whiteout = inode;
 	} else {
 		d_tmpfile(dentry, inode);
@@ -867,6 +868,12 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
 			F2FS_I(old_dentry->d_inode)->i_projid)))
 		return -EXDEV;
 
+	if (flags & RENAME_WHITEOUT) {
+		err = f2fs_create_whiteout(old_dir, &whiteout);
+		if (err)
+			return err;
+	}
+
 	err = dquot_initialize(old_dir);
 	if (err)
 		goto out;
@@ -898,17 +905,11 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
 		}
 	}
 
-	if (flags & RENAME_WHITEOUT) {
-		err = f2fs_create_whiteout(old_dir, &whiteout);
-		if (err)
-			goto out_dir;
-	}
-
 	if (new_inode) {
 
 		err = -ENOTEMPTY;
 		if (old_dir_entry && !f2fs_empty_dir(new_inode))
-			goto out_whiteout;
+			goto out_dir;
 
 		err = -ENOENT;
 		new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name,
@@ -916,7 +917,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
 		if (!new_entry) {
 			if (IS_ERR(new_page))
 				err = PTR_ERR(new_page);
-			goto out_whiteout;
+			goto out_dir;
 		}
 
 		f2fs_balance_fs(sbi, true);
@@ -948,7 +949,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
 		err = f2fs_add_link(new_dentry, old_inode);
 		if (err) {
 			f2fs_unlock_op(sbi);
-			goto out_whiteout;
+			goto out_dir;
 		}
 
 		if (old_dir_entry)
@@ -972,7 +973,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
 				if (IS_ERR(old_page))
 					err = PTR_ERR(old_page);
 				f2fs_unlock_op(sbi);
-				goto out_whiteout;
+				goto out_dir;
 			}
 		}
 	}
@@ -991,7 +992,6 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
 	f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
 
 	if (whiteout) {
-		whiteout->i_state |= I_LINKABLE;
 		set_inode_flag(whiteout, FI_INC_LINK);
 		err = f2fs_add_link(old_dentry, whiteout);
 		if (err)
@@ -1027,15 +1027,14 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
 	f2fs_unlock_op(sbi);
 	if (new_page)
 		f2fs_put_page(new_page, 0);
-out_whiteout:
-	if (whiteout)
-		iput(whiteout);
 out_dir:
 	if (old_dir_entry)
 		f2fs_put_page(old_dir_page, 0);
 out_old:
 	f2fs_put_page(old_page, 0);
 out:
+	if (whiteout)
+		iput(whiteout);
 	return err;
 }
 
-- 
2.19.0.605.g01d371f741-goog

Re: [f2fs-dev] [PATCH 6/6] f2fs: set I_LINKABLE early to avoid wrong access by vfs

[Chao Yu]

On 2019/12/10 6:23, Jaegeuk Kim wrote:
> This patch moves setting I_LINKABLE early in rename2(whiteout) to avoid the
> below warning.
> 
> [ 3189.163385] WARNING: CPU: 3 PID: 59523 at fs/inode.c:358 inc_nlink+0x32/0x40
> [ 3189.246979] Call Trace:
> [ 3189.248707]  f2fs_init_inode_metadata+0x2d6/0x440 [f2fs]
> [ 3189.251399]  f2fs_add_inline_entry+0x162/0x8c0 [f2fs]
> [ 3189.254010]  f2fs_add_dentry+0x69/0xe0 [f2fs]
> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
> [ 3189.261079]  vfs_rename+0x3f8/0xaa0
> [ 3189.263056]  ? tomoyo_path_rename+0x44/0x60
> [ 3189.265283]  ? do_renameat2+0x49b/0x550
> [ 3189.267324]  do_renameat2+0x49b/0x550
> [ 3189.269316]  __x64_sys_renameat2+0x20/0x30
> [ 3189.271441]  do_syscall_64+0x5a/0x230
> [ 3189.273410]  entry_SYSCALL_64_after_hwframe+0x49/0xbe
> [ 3189.275848] RIP: 0033:0x7f270b4d9a49
> 
> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
> ---
>  fs/f2fs/namei.c | 27 +++++++++++++--------------
>  1 file changed, 13 insertions(+), 14 deletions(-)
> 
> diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
> index a1c507b0b4ac..5d9584281935 100644
> --- a/fs/f2fs/namei.c
> +++ b/fs/f2fs/namei.c
> @@ -797,6 +797,7 @@ static int __f2fs_tmpfile(struct inode *dir, struct dentry *dentry,
>  
>  	if (whiteout) {
>  		f2fs_i_links_write(inode, false);
> +		inode->i_state |= I_LINKABLE;
>  		*whiteout = inode;
>  	} else {
>  		d_tmpfile(dentry, inode);
> @@ -867,6 +868,12 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>  			F2FS_I(old_dentry->d_inode)->i_projid)))
>  		return -EXDEV;
>  
> +	if (flags & RENAME_WHITEOUT) {
> +		err = f2fs_create_whiteout(old_dir, &whiteout);
> +		if (err)
> +			return err;
> +	}

To record quota info correctly, we need to create whiteout inode after
dquot_initialize(old_dir)?

> +
>  	err = dquot_initialize(old_dir);
>  	if (err)
>  		goto out;
> @@ -898,17 +905,11 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>  		}
>  	}
>  
> -	if (flags & RENAME_WHITEOUT) {
> -		err = f2fs_create_whiteout(old_dir, &whiteout);
> -		if (err)
> -			goto out_dir;
> -	}
> -
>  	if (new_inode) {
>  
>  		err = -ENOTEMPTY;
>  		if (old_dir_entry && !f2fs_empty_dir(new_inode))
> -			goto out_whiteout;
> +			goto out_dir;
>  
>  		err = -ENOENT;
>  		new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name,
> @@ -916,7 +917,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>  		if (!new_entry) {
>  			if (IS_ERR(new_page))
>  				err = PTR_ERR(new_page);
> -			goto out_whiteout;
> +			goto out_dir;
>  		}
>  
>  		f2fs_balance_fs(sbi, true);
> @@ -948,7 +949,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>  		err = f2fs_add_link(new_dentry, old_inode);
>  		if (err) {
>  			f2fs_unlock_op(sbi);
> -			goto out_whiteout;
> +			goto out_dir;
>  		}
>  
>  		if (old_dir_entry)
> @@ -972,7 +973,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>  				if (IS_ERR(old_page))
>  					err = PTR_ERR(old_page);
>  				f2fs_unlock_op(sbi);
> -				goto out_whiteout;
> +				goto out_dir;
>  			}
>  		}
>  	}
> @@ -991,7 +992,6 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>  	f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
>  
>  	if (whiteout) {
> -		whiteout->i_state |= I_LINKABLE;
>  		set_inode_flag(whiteout, FI_INC_LINK);
>  		err = f2fs_add_link(old_dentry, whiteout);

[ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
[ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]

Does the call stack point here? if so, we have set I_LINKABLE before
f2fs_add_link(), why the warning still be triggered?

Thanks,

>  		if (err)
> @@ -1027,15 +1027,14 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>  	f2fs_unlock_op(sbi);
>  	if (new_page)
>  		f2fs_put_page(new_page, 0);
> -out_whiteout:
> -	if (whiteout)
> -		iput(whiteout);
>  out_dir:
>  	if (old_dir_entry)
>  		f2fs_put_page(old_dir_page, 0);
>  out_old:
>  	f2fs_put_page(old_page, 0);
>  out:
> +	if (whiteout)
> +		iput(whiteout);
>  	return err;
>  }
>  
> 

Re: [f2fs-dev] [PATCH 6/6] f2fs: set I_LINKABLE early to avoid wrong access by vfs

[Jaegeuk Kim]

On 12/10, Chao Yu wrote:
> On 2019/12/10 6:23, Jaegeuk Kim wrote:
> > This patch moves setting I_LINKABLE early in rename2(whiteout) to avoid the
> > below warning.
> > 
> > [ 3189.163385] WARNING: CPU: 3 PID: 59523 at fs/inode.c:358 inc_nlink+0x32/0x40
> > [ 3189.246979] Call Trace:
> > [ 3189.248707]  f2fs_init_inode_metadata+0x2d6/0x440 [f2fs]
> > [ 3189.251399]  f2fs_add_inline_entry+0x162/0x8c0 [f2fs]
> > [ 3189.254010]  f2fs_add_dentry+0x69/0xe0 [f2fs]
> > [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
> > [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
> > [ 3189.261079]  vfs_rename+0x3f8/0xaa0
> > [ 3189.263056]  ? tomoyo_path_rename+0x44/0x60
> > [ 3189.265283]  ? do_renameat2+0x49b/0x550
> > [ 3189.267324]  do_renameat2+0x49b/0x550
> > [ 3189.269316]  __x64_sys_renameat2+0x20/0x30
> > [ 3189.271441]  do_syscall_64+0x5a/0x230
> > [ 3189.273410]  entry_SYSCALL_64_after_hwframe+0x49/0xbe
> > [ 3189.275848] RIP: 0033:0x7f270b4d9a49
> > 
> > Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
> > ---
> >  fs/f2fs/namei.c | 27 +++++++++++++--------------
> >  1 file changed, 13 insertions(+), 14 deletions(-)
> > 
> > diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
> > index a1c507b0b4ac..5d9584281935 100644
> > --- a/fs/f2fs/namei.c
> > +++ b/fs/f2fs/namei.c
> > @@ -797,6 +797,7 @@ static int __f2fs_tmpfile(struct inode *dir, struct dentry *dentry,
> >  
> >  	if (whiteout) {
> >  		f2fs_i_links_write(inode, false);
> > +		inode->i_state |= I_LINKABLE;
> >  		*whiteout = inode;
> >  	} else {
> >  		d_tmpfile(dentry, inode);
> > @@ -867,6 +868,12 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >  			F2FS_I(old_dentry->d_inode)->i_projid)))
> >  		return -EXDEV;
> >  
> > +	if (flags & RENAME_WHITEOUT) {
> > +		err = f2fs_create_whiteout(old_dir, &whiteout);
> > +		if (err)
> > +			return err;
> > +	}
> 
> To record quota info correctly, we need to create whiteout inode after
> dquot_initialize(old_dir)?

__f2fs_tmpfile() will do it.

> 
> > +
> >  	err = dquot_initialize(old_dir);
> >  	if (err)
> >  		goto out;
> > @@ -898,17 +905,11 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >  		}
> >  	}
> >  
> > -	if (flags & RENAME_WHITEOUT) {
> > -		err = f2fs_create_whiteout(old_dir, &whiteout);
> > -		if (err)
> > -			goto out_dir;
> > -	}
> > -
> >  	if (new_inode) {
> >  
> >  		err = -ENOTEMPTY;
> >  		if (old_dir_entry && !f2fs_empty_dir(new_inode))
> > -			goto out_whiteout;
> > +			goto out_dir;
> >  
> >  		err = -ENOENT;
> >  		new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name,
> > @@ -916,7 +917,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >  		if (!new_entry) {
> >  			if (IS_ERR(new_page))
> >  				err = PTR_ERR(new_page);
> > -			goto out_whiteout;
> > +			goto out_dir;
> >  		}
> >  
> >  		f2fs_balance_fs(sbi, true);
> > @@ -948,7 +949,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >  		err = f2fs_add_link(new_dentry, old_inode);
> >  		if (err) {
> >  			f2fs_unlock_op(sbi);
> > -			goto out_whiteout;
> > +			goto out_dir;
> >  		}
> >  
> >  		if (old_dir_entry)
> > @@ -972,7 +973,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >  				if (IS_ERR(old_page))
> >  					err = PTR_ERR(old_page);
> >  				f2fs_unlock_op(sbi);
> > -				goto out_whiteout;
> > +				goto out_dir;
> >  			}
> >  		}
> >  	}
> > @@ -991,7 +992,6 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >  	f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
> >  
> >  	if (whiteout) {
> > -		whiteout->i_state |= I_LINKABLE;
> >  		set_inode_flag(whiteout, FI_INC_LINK);
> >  		err = f2fs_add_link(old_dentry, whiteout);
> 
> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
> 
> Does the call stack point here? if so, we have set I_LINKABLE before
> f2fs_add_link(), why the warning still be triggered?
> 
> Thanks,
> 
> >  		if (err)
> > @@ -1027,15 +1027,14 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >  	f2fs_unlock_op(sbi);
> >  	if (new_page)
> >  		f2fs_put_page(new_page, 0);
> > -out_whiteout:
> > -	if (whiteout)
> > -		iput(whiteout);
> >  out_dir:
> >  	if (old_dir_entry)
> >  		f2fs_put_page(old_dir_page, 0);
> >  out_old:
> >  	f2fs_put_page(old_page, 0);
> >  out:
> > +	if (whiteout)
> > +		iput(whiteout);
> >  	return err;
> >  }
> >  
> > 

Re: [f2fs-dev] [PATCH 6/6] f2fs: set I_LINKABLE early to avoid wrong access by vfs

[Chao Yu]

On 2019/12/11 9:21, Jaegeuk Kim wrote:
> On 12/10, Chao Yu wrote:
>> On 2019/12/10 6:23, Jaegeuk Kim wrote:
>>> This patch moves setting I_LINKABLE early in rename2(whiteout) to avoid the
>>> below warning.
>>>
>>> [ 3189.163385] WARNING: CPU: 3 PID: 59523 at fs/inode.c:358 inc_nlink+0x32/0x40
>>> [ 3189.246979] Call Trace:
>>> [ 3189.248707]  f2fs_init_inode_metadata+0x2d6/0x440 [f2fs]
>>> [ 3189.251399]  f2fs_add_inline_entry+0x162/0x8c0 [f2fs]
>>> [ 3189.254010]  f2fs_add_dentry+0x69/0xe0 [f2fs]
>>> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
>>> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
>>> [ 3189.261079]  vfs_rename+0x3f8/0xaa0
>>> [ 3189.263056]  ? tomoyo_path_rename+0x44/0x60
>>> [ 3189.265283]  ? do_renameat2+0x49b/0x550
>>> [ 3189.267324]  do_renameat2+0x49b/0x550
>>> [ 3189.269316]  __x64_sys_renameat2+0x20/0x30
>>> [ 3189.271441]  do_syscall_64+0x5a/0x230
>>> [ 3189.273410]  entry_SYSCALL_64_after_hwframe+0x49/0xbe
>>> [ 3189.275848] RIP: 0033:0x7f270b4d9a49
>>>
>>> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
>>> ---
>>>  fs/f2fs/namei.c | 27 +++++++++++++--------------
>>>  1 file changed, 13 insertions(+), 14 deletions(-)
>>>
>>> diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
>>> index a1c507b0b4ac..5d9584281935 100644
>>> --- a/fs/f2fs/namei.c
>>> +++ b/fs/f2fs/namei.c
>>> @@ -797,6 +797,7 @@ static int __f2fs_tmpfile(struct inode *dir, struct dentry *dentry,
>>>  
>>>  	if (whiteout) {
>>>  		f2fs_i_links_write(inode, false);
>>> +		inode->i_state |= I_LINKABLE;
>>>  		*whiteout = inode;
>>>  	} else {
>>>  		d_tmpfile(dentry, inode);
>>> @@ -867,6 +868,12 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>  			F2FS_I(old_dentry->d_inode)->i_projid)))
>>>  		return -EXDEV;
>>>  
>>> +	if (flags & RENAME_WHITEOUT) {
>>> +		err = f2fs_create_whiteout(old_dir, &whiteout);
>>> +		if (err)
>>> +			return err;
>>> +	}
>>
>> To record quota info correctly, we need to create whiteout inode after
>> dquot_initialize(old_dir)?
> 
> __f2fs_tmpfile() will do it.

Okay.

Any comments on below question?

> 
>>
>>> +
>>>  	err = dquot_initialize(old_dir);
>>>  	if (err)
>>>  		goto out;
>>> @@ -898,17 +905,11 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>  		}
>>>  	}
>>>  
>>> -	if (flags & RENAME_WHITEOUT) {
>>> -		err = f2fs_create_whiteout(old_dir, &whiteout);
>>> -		if (err)
>>> -			goto out_dir;
>>> -	}
>>> -
>>>  	if (new_inode) {
>>>  
>>>  		err = -ENOTEMPTY;
>>>  		if (old_dir_entry && !f2fs_empty_dir(new_inode))
>>> -			goto out_whiteout;
>>> +			goto out_dir;
>>>  
>>>  		err = -ENOENT;
>>>  		new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name,
>>> @@ -916,7 +917,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>  		if (!new_entry) {
>>>  			if (IS_ERR(new_page))
>>>  				err = PTR_ERR(new_page);
>>> -			goto out_whiteout;
>>> +			goto out_dir;
>>>  		}
>>>  
>>>  		f2fs_balance_fs(sbi, true);
>>> @@ -948,7 +949,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>  		err = f2fs_add_link(new_dentry, old_inode);
>>>  		if (err) {
>>>  			f2fs_unlock_op(sbi);
>>> -			goto out_whiteout;
>>> +			goto out_dir;
>>>  		}
>>>  
>>>  		if (old_dir_entry)
>>> @@ -972,7 +973,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>  				if (IS_ERR(old_page))
>>>  					err = PTR_ERR(old_page);
>>>  				f2fs_unlock_op(sbi);
>>> -				goto out_whiteout;
>>> +				goto out_dir;
>>>  			}
>>>  		}
>>>  	}
>>> @@ -991,7 +992,6 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>  	f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
>>>  
>>>  	if (whiteout) {
>>> -		whiteout->i_state |= I_LINKABLE;
>>>  		set_inode_flag(whiteout, FI_INC_LINK);
>>>  		err = f2fs_add_link(old_dentry, whiteout);
>>
>> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
>> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
>>
>> Does the call stack point here? if so, we have set I_LINKABLE before
>> f2fs_add_link(), why the warning still be triggered?

Am I missing something?

Thanks,

>>
>> Thanks,
>>
>>>  		if (err)
>>> @@ -1027,15 +1027,14 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>  	f2fs_unlock_op(sbi);
>>>  	if (new_page)
>>>  		f2fs_put_page(new_page, 0);
>>> -out_whiteout:
>>> -	if (whiteout)
>>> -		iput(whiteout);
>>>  out_dir:
>>>  	if (old_dir_entry)
>>>  		f2fs_put_page(old_dir_page, 0);
>>>  out_old:
>>>  	f2fs_put_page(old_page, 0);
>>>  out:
>>> +	if (whiteout)
>>> +		iput(whiteout);
>>>  	return err;
>>>  }
>>>  
>>>
> .
> 

Re: [f2fs-dev] [PATCH 6/6] f2fs: set I_LINKABLE early to avoid wrong access by vfs

[Jaegeuk Kim]

On 12/11, Chao Yu wrote:
> On 2019/12/11 9:21, Jaegeuk Kim wrote:
> > On 12/10, Chao Yu wrote:
> >> On 2019/12/10 6:23, Jaegeuk Kim wrote:
> >>> This patch moves setting I_LINKABLE early in rename2(whiteout) to avoid the
> >>> below warning.
> >>>
> >>> [ 3189.163385] WARNING: CPU: 3 PID: 59523 at fs/inode.c:358 inc_nlink+0x32/0x40
> >>> [ 3189.246979] Call Trace:
> >>> [ 3189.248707]  f2fs_init_inode_metadata+0x2d6/0x440 [f2fs]
> >>> [ 3189.251399]  f2fs_add_inline_entry+0x162/0x8c0 [f2fs]
> >>> [ 3189.254010]  f2fs_add_dentry+0x69/0xe0 [f2fs]
> >>> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
> >>> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
> >>> [ 3189.261079]  vfs_rename+0x3f8/0xaa0
> >>> [ 3189.263056]  ? tomoyo_path_rename+0x44/0x60
> >>> [ 3189.265283]  ? do_renameat2+0x49b/0x550
> >>> [ 3189.267324]  do_renameat2+0x49b/0x550
> >>> [ 3189.269316]  __x64_sys_renameat2+0x20/0x30
> >>> [ 3189.271441]  do_syscall_64+0x5a/0x230
> >>> [ 3189.273410]  entry_SYSCALL_64_after_hwframe+0x49/0xbe
> >>> [ 3189.275848] RIP: 0033:0x7f270b4d9a49
> >>>
> >>> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
> >>> ---
> >>>  fs/f2fs/namei.c | 27 +++++++++++++--------------
> >>>  1 file changed, 13 insertions(+), 14 deletions(-)
> >>>
> >>> diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
> >>> index a1c507b0b4ac..5d9584281935 100644
> >>> --- a/fs/f2fs/namei.c
> >>> +++ b/fs/f2fs/namei.c
> >>> @@ -797,6 +797,7 @@ static int __f2fs_tmpfile(struct inode *dir, struct dentry *dentry,
> >>>  
> >>>  	if (whiteout) {
> >>>  		f2fs_i_links_write(inode, false);
> >>> +		inode->i_state |= I_LINKABLE;
> >>>  		*whiteout = inode;
> >>>  	} else {
> >>>  		d_tmpfile(dentry, inode);
> >>> @@ -867,6 +868,12 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>  			F2FS_I(old_dentry->d_inode)->i_projid)))
> >>>  		return -EXDEV;
> >>>  
> >>> +	if (flags & RENAME_WHITEOUT) {
> >>> +		err = f2fs_create_whiteout(old_dir, &whiteout);
> >>> +		if (err)
> >>> +			return err;
> >>> +	}
> >>
> >> To record quota info correctly, we need to create whiteout inode after
> >> dquot_initialize(old_dir)?
> > 
> > __f2fs_tmpfile() will do it.
> 
> Okay.
> 
> Any comments on below question?
> 
> > 
> >>
> >>> +
> >>>  	err = dquot_initialize(old_dir);
> >>>  	if (err)
> >>>  		goto out;
> >>> @@ -898,17 +905,11 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>  		}
> >>>  	}
> >>>  
> >>> -	if (flags & RENAME_WHITEOUT) {
> >>> -		err = f2fs_create_whiteout(old_dir, &whiteout);
> >>> -		if (err)
> >>> -			goto out_dir;
> >>> -	}
> >>> -
> >>>  	if (new_inode) {
> >>>  
> >>>  		err = -ENOTEMPTY;
> >>>  		if (old_dir_entry && !f2fs_empty_dir(new_inode))
> >>> -			goto out_whiteout;
> >>> +			goto out_dir;
> >>>  
> >>>  		err = -ENOENT;
> >>>  		new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name,
> >>> @@ -916,7 +917,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>  		if (!new_entry) {
> >>>  			if (IS_ERR(new_page))
> >>>  				err = PTR_ERR(new_page);
> >>> -			goto out_whiteout;
> >>> +			goto out_dir;
> >>>  		}
> >>>  
> >>>  		f2fs_balance_fs(sbi, true);
> >>> @@ -948,7 +949,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>  		err = f2fs_add_link(new_dentry, old_inode);
> >>>  		if (err) {
> >>>  			f2fs_unlock_op(sbi);
> >>> -			goto out_whiteout;
> >>> +			goto out_dir;
> >>>  		}
> >>>  
> >>>  		if (old_dir_entry)
> >>> @@ -972,7 +973,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>  				if (IS_ERR(old_page))
> >>>  					err = PTR_ERR(old_page);
> >>>  				f2fs_unlock_op(sbi);
> >>> -				goto out_whiteout;
> >>> +				goto out_dir;
> >>>  			}
> >>>  		}
> >>>  	}
> >>> @@ -991,7 +992,6 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>  	f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
> >>>  
> >>>  	if (whiteout) {
> >>> -		whiteout->i_state |= I_LINKABLE;
> >>>  		set_inode_flag(whiteout, FI_INC_LINK);
> >>>  		err = f2fs_add_link(old_dentry, whiteout);
> >>
> >> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
> >> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
> >>
> >> Does the call stack point here? if so, we have set I_LINKABLE before
> >> f2fs_add_link(), why the warning still be triggered?
> 
> Am I missing something?

Not sure exactly tho, I suspect some races before/after unlock_new_inode().

> 
> Thanks,
> 
> >>
> >> Thanks,
> >>
> >>>  		if (err)
> >>> @@ -1027,15 +1027,14 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>  	f2fs_unlock_op(sbi);
> >>>  	if (new_page)
> >>>  		f2fs_put_page(new_page, 0);
> >>> -out_whiteout:
> >>> -	if (whiteout)
> >>> -		iput(whiteout);
> >>>  out_dir:
> >>>  	if (old_dir_entry)
> >>>  		f2fs_put_page(old_dir_page, 0);
> >>>  out_old:
> >>>  	f2fs_put_page(old_page, 0);
> >>>  out:
> >>> +	if (whiteout)
> >>> +		iput(whiteout);
> >>>  	return err;
> >>>  }
> >>>  
> >>>
> > .
> > 

Re: [f2fs-dev] [PATCH 6/6] f2fs: set I_LINKABLE early to avoid wrong access by vfs

[Chao Yu]

On 2019/12/11 9:31, Jaegeuk Kim wrote:
> On 12/11, Chao Yu wrote:
>> On 2019/12/11 9:21, Jaegeuk Kim wrote:
>>> On 12/10, Chao Yu wrote:
>>>> On 2019/12/10 6:23, Jaegeuk Kim wrote:
>>>>> This patch moves setting I_LINKABLE early in rename2(whiteout) to avoid the
>>>>> below warning.
>>>>>
>>>>> [ 3189.163385] WARNING: CPU: 3 PID: 59523 at fs/inode.c:358 inc_nlink+0x32/0x40
>>>>> [ 3189.246979] Call Trace:
>>>>> [ 3189.248707]  f2fs_init_inode_metadata+0x2d6/0x440 [f2fs]
>>>>> [ 3189.251399]  f2fs_add_inline_entry+0x162/0x8c0 [f2fs]
>>>>> [ 3189.254010]  f2fs_add_dentry+0x69/0xe0 [f2fs]
>>>>> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
>>>>> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
>>>>> [ 3189.261079]  vfs_rename+0x3f8/0xaa0
>>>>> [ 3189.263056]  ? tomoyo_path_rename+0x44/0x60
>>>>> [ 3189.265283]  ? do_renameat2+0x49b/0x550
>>>>> [ 3189.267324]  do_renameat2+0x49b/0x550
>>>>> [ 3189.269316]  __x64_sys_renameat2+0x20/0x30
>>>>> [ 3189.271441]  do_syscall_64+0x5a/0x230
>>>>> [ 3189.273410]  entry_SYSCALL_64_after_hwframe+0x49/0xbe
>>>>> [ 3189.275848] RIP: 0033:0x7f270b4d9a49
>>>>>
>>>>> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
>>>>> ---
>>>>>  fs/f2fs/namei.c | 27 +++++++++++++--------------
>>>>>  1 file changed, 13 insertions(+), 14 deletions(-)
>>>>>
>>>>> diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
>>>>> index a1c507b0b4ac..5d9584281935 100644
>>>>> --- a/fs/f2fs/namei.c
>>>>> +++ b/fs/f2fs/namei.c
>>>>> @@ -797,6 +797,7 @@ static int __f2fs_tmpfile(struct inode *dir, struct dentry *dentry,
>>>>>  
>>>>>  	if (whiteout) {
>>>>>  		f2fs_i_links_write(inode, false);
>>>>> +		inode->i_state |= I_LINKABLE;
>>>>>  		*whiteout = inode;
>>>>>  	} else {
>>>>>  		d_tmpfile(dentry, inode);
>>>>> @@ -867,6 +868,12 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>>>  			F2FS_I(old_dentry->d_inode)->i_projid)))
>>>>>  		return -EXDEV;
>>>>>  
>>>>> +	if (flags & RENAME_WHITEOUT) {
>>>>> +		err = f2fs_create_whiteout(old_dir, &whiteout);
>>>>> +		if (err)
>>>>> +			return err;
>>>>> +	}
>>>>
>>>> To record quota info correctly, we need to create whiteout inode after
>>>> dquot_initialize(old_dir)?
>>>
>>> __f2fs_tmpfile() will do it.
>>
>> Okay.
>>
>> Any comments on below question?
>>
>>>
>>>>
>>>>> +
>>>>>  	err = dquot_initialize(old_dir);
>>>>>  	if (err)
>>>>>  		goto out;
>>>>> @@ -898,17 +905,11 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>>>  		}
>>>>>  	}
>>>>>  
>>>>> -	if (flags & RENAME_WHITEOUT) {
>>>>> -		err = f2fs_create_whiteout(old_dir, &whiteout);
>>>>> -		if (err)
>>>>> -			goto out_dir;
>>>>> -	}
>>>>> -
>>>>>  	if (new_inode) {
>>>>>  
>>>>>  		err = -ENOTEMPTY;
>>>>>  		if (old_dir_entry && !f2fs_empty_dir(new_inode))
>>>>> -			goto out_whiteout;
>>>>> +			goto out_dir;
>>>>>  
>>>>>  		err = -ENOENT;
>>>>>  		new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name,
>>>>> @@ -916,7 +917,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>>>  		if (!new_entry) {
>>>>>  			if (IS_ERR(new_page))
>>>>>  				err = PTR_ERR(new_page);
>>>>> -			goto out_whiteout;
>>>>> +			goto out_dir;
>>>>>  		}
>>>>>  
>>>>>  		f2fs_balance_fs(sbi, true);
>>>>> @@ -948,7 +949,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>>>  		err = f2fs_add_link(new_dentry, old_inode);
>>>>>  		if (err) {
>>>>>  			f2fs_unlock_op(sbi);
>>>>> -			goto out_whiteout;
>>>>> +			goto out_dir;
>>>>>  		}
>>>>>  
>>>>>  		if (old_dir_entry)
>>>>> @@ -972,7 +973,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>>>  				if (IS_ERR(old_page))
>>>>>  					err = PTR_ERR(old_page);
>>>>>  				f2fs_unlock_op(sbi);
>>>>> -				goto out_whiteout;
>>>>> +				goto out_dir;
>>>>>  			}
>>>>>  		}
>>>>>  	}
>>>>> @@ -991,7 +992,6 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>>>  	f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
>>>>>  
>>>>>  	if (whiteout) {
>>>>> -		whiteout->i_state |= I_LINKABLE;
>>>>>  		set_inode_flag(whiteout, FI_INC_LINK);
>>>>>  		err = f2fs_add_link(old_dentry, whiteout);
>>>>
>>>> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
>>>> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
>>>>
>>>> Does the call stack point here? if so, we have set I_LINKABLE before
>>>> f2fs_add_link(), why the warning still be triggered?
>>
>> Am I missing something?
> 
> Not sure exactly tho, I suspect some races before/after unlock_new_inode().

Alright, I doubt some races on whiteout->i_state updating, as we set I_LINKABLE
w/o holding inode.i_lock.

Could you have a try with holding i_lock?

Thanks,

> 
>>
>> Thanks,
>>
>>>>
>>>> Thanks,
>>>>
>>>>>  		if (err)
>>>>> @@ -1027,15 +1027,14 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
>>>>>  	f2fs_unlock_op(sbi);
>>>>>  	if (new_page)
>>>>>  		f2fs_put_page(new_page, 0);
>>>>> -out_whiteout:
>>>>> -	if (whiteout)
>>>>> -		iput(whiteout);
>>>>>  out_dir:
>>>>>  	if (old_dir_entry)
>>>>>  		f2fs_put_page(old_dir_page, 0);
>>>>>  out_old:
>>>>>  	f2fs_put_page(old_page, 0);
>>>>>  out:
>>>>> +	if (whiteout)
>>>>> +		iput(whiteout);
>>>>>  	return err;
>>>>>  }
>>>>>  
>>>>>
>>> .
>>>
> .
> 

Re: [f2fs-dev] [PATCH 6/6] f2fs: set I_LINKABLE early to avoid wrong access by vfs

[Jaegeuk Kim]

On 12/11, Chao Yu wrote:
> On 2019/12/11 9:31, Jaegeuk Kim wrote:
> > On 12/11, Chao Yu wrote:
> >> On 2019/12/11 9:21, Jaegeuk Kim wrote:
> >>> On 12/10, Chao Yu wrote:
> >>>> On 2019/12/10 6:23, Jaegeuk Kim wrote:
> >>>>> This patch moves setting I_LINKABLE early in rename2(whiteout) to avoid the
> >>>>> below warning.
> >>>>>
> >>>>> [ 3189.163385] WARNING: CPU: 3 PID: 59523 at fs/inode.c:358 inc_nlink+0x32/0x40
> >>>>> [ 3189.246979] Call Trace:
> >>>>> [ 3189.248707]  f2fs_init_inode_metadata+0x2d6/0x440 [f2fs]
> >>>>> [ 3189.251399]  f2fs_add_inline_entry+0x162/0x8c0 [f2fs]
> >>>>> [ 3189.254010]  f2fs_add_dentry+0x69/0xe0 [f2fs]
> >>>>> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
> >>>>> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
> >>>>> [ 3189.261079]  vfs_rename+0x3f8/0xaa0
> >>>>> [ 3189.263056]  ? tomoyo_path_rename+0x44/0x60
> >>>>> [ 3189.265283]  ? do_renameat2+0x49b/0x550
> >>>>> [ 3189.267324]  do_renameat2+0x49b/0x550
> >>>>> [ 3189.269316]  __x64_sys_renameat2+0x20/0x30
> >>>>> [ 3189.271441]  do_syscall_64+0x5a/0x230
> >>>>> [ 3189.273410]  entry_SYSCALL_64_after_hwframe+0x49/0xbe
> >>>>> [ 3189.275848] RIP: 0033:0x7f270b4d9a49
> >>>>>
> >>>>> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
> >>>>> ---
> >>>>>  fs/f2fs/namei.c | 27 +++++++++++++--------------
> >>>>>  1 file changed, 13 insertions(+), 14 deletions(-)
> >>>>>
> >>>>> diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
> >>>>> index a1c507b0b4ac..5d9584281935 100644
> >>>>> --- a/fs/f2fs/namei.c
> >>>>> +++ b/fs/f2fs/namei.c
> >>>>> @@ -797,6 +797,7 @@ static int __f2fs_tmpfile(struct inode *dir, struct dentry *dentry,
> >>>>>  
> >>>>>  	if (whiteout) {
> >>>>>  		f2fs_i_links_write(inode, false);
> >>>>> +		inode->i_state |= I_LINKABLE;
> >>>>>  		*whiteout = inode;
> >>>>>  	} else {
> >>>>>  		d_tmpfile(dentry, inode);
> >>>>> @@ -867,6 +868,12 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>>>  			F2FS_I(old_dentry->d_inode)->i_projid)))
> >>>>>  		return -EXDEV;
> >>>>>  
> >>>>> +	if (flags & RENAME_WHITEOUT) {
> >>>>> +		err = f2fs_create_whiteout(old_dir, &whiteout);
> >>>>> +		if (err)
> >>>>> +			return err;
> >>>>> +	}
> >>>>
> >>>> To record quota info correctly, we need to create whiteout inode after
> >>>> dquot_initialize(old_dir)?
> >>>
> >>> __f2fs_tmpfile() will do it.
> >>
> >> Okay.
> >>
> >> Any comments on below question?
> >>
> >>>
> >>>>
> >>>>> +
> >>>>>  	err = dquot_initialize(old_dir);
> >>>>>  	if (err)
> >>>>>  		goto out;
> >>>>> @@ -898,17 +905,11 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>>>  		}
> >>>>>  	}
> >>>>>  
> >>>>> -	if (flags & RENAME_WHITEOUT) {
> >>>>> -		err = f2fs_create_whiteout(old_dir, &whiteout);
> >>>>> -		if (err)
> >>>>> -			goto out_dir;
> >>>>> -	}
> >>>>> -
> >>>>>  	if (new_inode) {
> >>>>>  
> >>>>>  		err = -ENOTEMPTY;
> >>>>>  		if (old_dir_entry && !f2fs_empty_dir(new_inode))
> >>>>> -			goto out_whiteout;
> >>>>> +			goto out_dir;
> >>>>>  
> >>>>>  		err = -ENOENT;
> >>>>>  		new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name,
> >>>>> @@ -916,7 +917,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>>>  		if (!new_entry) {
> >>>>>  			if (IS_ERR(new_page))
> >>>>>  				err = PTR_ERR(new_page);
> >>>>> -			goto out_whiteout;
> >>>>> +			goto out_dir;
> >>>>>  		}
> >>>>>  
> >>>>>  		f2fs_balance_fs(sbi, true);
> >>>>> @@ -948,7 +949,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>>>  		err = f2fs_add_link(new_dentry, old_inode);
> >>>>>  		if (err) {
> >>>>>  			f2fs_unlock_op(sbi);
> >>>>> -			goto out_whiteout;
> >>>>> +			goto out_dir;
> >>>>>  		}
> >>>>>  
> >>>>>  		if (old_dir_entry)
> >>>>> @@ -972,7 +973,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>>>  				if (IS_ERR(old_page))
> >>>>>  					err = PTR_ERR(old_page);
> >>>>>  				f2fs_unlock_op(sbi);
> >>>>> -				goto out_whiteout;
> >>>>> +				goto out_dir;
> >>>>>  			}
> >>>>>  		}
> >>>>>  	}
> >>>>> @@ -991,7 +992,6 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>>>  	f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
> >>>>>  
> >>>>>  	if (whiteout) {
> >>>>> -		whiteout->i_state |= I_LINKABLE;
> >>>>>  		set_inode_flag(whiteout, FI_INC_LINK);
> >>>>>  		err = f2fs_add_link(old_dentry, whiteout);
> >>>>
> >>>> [ 3189.256353]  f2fs_do_add_link+0xc5/0x100 [f2fs]
> >>>> [ 3189.258774]  f2fs_rename2+0xabf/0x1010 [f2fs]
> >>>>
> >>>> Does the call stack point here? if so, we have set I_LINKABLE before
> >>>> f2fs_add_link(), why the warning still be triggered?
> >>
> >> Am I missing something?
> > 
> > Not sure exactly tho, I suspect some races before/after unlock_new_inode().
> 
> Alright, I doubt some races on whiteout->i_state updating, as we set I_LINKABLE
> w/o holding inode.i_lock.
> 
> Could you have a try with holding i_lock?

I don't see the warning with this patch, and jumped to another issue. I'd like
to take a look at that later.

Thanks,

> 
> Thanks,
> 
> > 
> >>
> >> Thanks,
> >>
> >>>>
> >>>> Thanks,
> >>>>
> >>>>>  		if (err)
> >>>>> @@ -1027,15 +1027,14 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
> >>>>>  	f2fs_unlock_op(sbi);
> >>>>>  	if (new_page)
> >>>>>  		f2fs_put_page(new_page, 0);
> >>>>> -out_whiteout:
> >>>>> -	if (whiteout)
> >>>>> -		iput(whiteout);
> >>>>>  out_dir:
> >>>>>  	if (old_dir_entry)
> >>>>>  		f2fs_put_page(old_dir_page, 0);
> >>>>>  out_old:
> >>>>>  	f2fs_put_page(old_page, 0);
> >>>>>  out:
> >>>>> +	if (whiteout)
> >>>>> +		iput(whiteout);
> >>>>>  	return err;
> >>>>>  }
> >>>>>  
> >>>>>
> >>> .
> >>>
> > .
> > 

Re: [f2fs-dev] [PATCH 1/6] f2fs: call f2fs_balance_fs outside of locked page

[Chao Yu]

On 2019/12/10 6:23, Jaegeuk Kim wrote:
> Otherwise, we can hit deadlock by waiting for the locked page in
> move_data_block in GC.
> 
>  Thread A                     Thread B
>  - do_page_mkwrite
>   - f2fs_vm_page_mkwrite
>    - lock_page
>                               - f2fs_balance_fs
>                                   - mutex_lock(gc_mutex)
>                                - f2fs_gc
>                                 - do_garbage_collect
>                                  - ra_data_block
>                                   - grab_cache_page
>    - f2fs_balance_fs
>     - mutex_lock(gc_mutex)
> 
> Fixes: 39a8695824510 ("f2fs: refactor ->page_mkwrite() flow")
> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>

Reviewed-by: Chao Yu <yuchao0@huawei.com>

Thanks,

Re: [PATCH 1/6] f2fs: call f2fs_balance_fs outside of locked page

[Ondřej Jirman]

Hello,

On Mon, Dec 09, 2019 at 02:23:40PM -0800, Jaegeuk Kim wrote:
> Otherwise, we can hit deadlock by waiting for the locked page in
> move_data_block in GC.

I had the task hangs on 5.6 shortly after boot. (f2fs as rootfs).
See below for stacktrace.

So I went through the changelog in f2fs and noticed this patch as
a suspect, and after reverting it the hung task panics went away.

I reverted it manually, because the master changed too much for
a clean revert:

https://megous.com/git/linux/commit/?h=orange-pi-5.6&id=9983bdae4974edc2af6ff547a401ae397388b6b5

regards,
	o.

INFO: task kworker/u16:2:341 blocked for more than 122 seconds.
      Not tainted 5.6.0-rc2-00254-g9a029a493dc16 #4
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
kworker/u16:2   D    0   341      2 0x00000000
Workqueue: writeback wb_workfn (flush-179:0)
Backtrace:
[<c0912bd0>] (__schedule) from [<c0913274>] (schedule+0x78/0xf4)
 r10:ede1a000 r9:00000000 r8:ede1ba60 r7:ec417290 r6:00000002 r5:ede1a000
 r4:ee8e8000
[<c09131fc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
 r5:00000001 r4:ec417280
[<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f6c>] (down_write+0x6c/0x70)
 r10:ec417280 r9:ede1bd80 r8:ee128000 r7:00000001 r6:00000000 r5:eff0afc4
 r4:ec417280
[<c0915f00>] (down_write) from [<c0435b68>] (f2fs_write_single_data_page+0x608/0x7ac)
 r5:eff0afc4 r4:ec4170e0
[<c0435560>] (f2fs_write_single_data_page) from [<c0435fc0>] (f2fs_write_cache_pages+0x2b4/0x7c4)
 r10:ede1bc28 r9:ec4171e0 r8:ec4170e0 r7:00000001 r6:ede1bd80 r5:00000001
 r4:eff0afc4
[<c0435d0c>] (f2fs_write_cache_pages) from [<c0436814>] (f2fs_write_data_pages+0x344/0x35c)
 r10:0000012c r9:ee12802c r8:ee128000 r7:00000004 r6:ec4171e0 r5:ec4170e0
 r4:ede1bd80
[<c04364d0>] (f2fs_write_data_pages) from [<c0267fa0>] (do_writepages+0x3c/0xd4)
 r10:0000012c r9:c0e03d00 r8:00001400 r7:c0264e94 r6:ede1bd80 r5:ec4171e0
 r4:ec4170e0
[<c0267f64>] (do_writepages) from [<c0310d24>] (__writeback_single_inode+0x44/0x454)
 r7:ec4171e0 r6:ede1beac r5:ede1bd80 r4:ec4170e0
[<c0310ce0>] (__writeback_single_inode) from [<c0311338>] (writeback_sb_inodes+0x204/0x4b0)
 r10:0000012c r9:c0e03d00 r8:ec417148 r7:ec4170e0 r6:ede1beac r5:ec417188
 r4:eebed848
[<c0311134>] (writeback_sb_inodes) from [<c0311634>] (__writeback_inodes_wb+0x50/0xe4)
 r10:ee7128e8 r9:c0e03d00 r8:eebed85c r7:ede1beac r6:00000000 r5:eebed848
 r4:ee120000
[<c03115e4>] (__writeback_inodes_wb) from [<c031195c>] (wb_writeback+0x294/0x338)
 r10:00020800 r9:ede1a000 r8:c0e04e64 r7:eebed848 r6:000192d0 r5:ede1beac
 r4:eebed848
[<c03116c8>] (wb_writeback) from [<c0312e98>] (wb_workfn+0x3e0/0x54c)
 r10:ee894005 r9:eebed84c r8:eebed948 r7:eebed848 r6:00000000 r5:eebed954
 r4:00002b6e
[<c0312ab8>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
 r10:ee894005 r9:00000200 r8:00000000 r7:ee894000 r6:ef044400 r5:edb1c700
 r4:eebed954
[<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
 r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:edb1c714
 r4:edb1c700
[<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
 r10:ef125e90 r9:ec0f235c r8:edb1c700 r7:ede1a000 r6:00000000 r5:ec0f2300
 r4:ec0f2340
[<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
Exception stack(0xede1bfb0 to 0xede1bff8)
bfa0:                                     00000000 00000000 00000000 00000000
bfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
bfe0: 00000000 00000000 00000000 00000000 00000013 00000000
 r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
 r4:ec0f2300
NMI backtrace for cpu 2
CPU: 2 PID: 52 Comm: khungtaskd Not tainted 5.6.0-rc2-00254-g9a029a493dc16 #4
Hardware name: Allwinner A83t board
Backtrace:
[<c010db5c>] (dump_backtrace) from [<c010dee0>] (show_stack+0x20/0x24)
 r7:00000000 r6:60060013 r5:00000000 r4:c0e9ab10



>  Thread A                     Thread B
>  - do_page_mkwrite
>   - f2fs_vm_page_mkwrite
>    - lock_page
>                               - f2fs_balance_fs
>                                   - mutex_lock(gc_mutex)
>                                - f2fs_gc
>                                 - do_garbage_collect
>                                  - ra_data_block
>                                   - grab_cache_page
>    - f2fs_balance_fs
>     - mutex_lock(gc_mutex)
> 
> Fixes: 39a8695824510 ("f2fs: refactor ->page_mkwrite() flow")
> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
> ---
>  fs/f2fs/file.c | 7 ++++---
>  1 file changed, 4 insertions(+), 3 deletions(-)
> 
> diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
> index e7fcbd8c23f4..6cebc6681487 100644
> --- a/fs/f2fs/file.c
> +++ b/fs/f2fs/file.c
> @@ -50,7 +50,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf)
>  	struct page *page = vmf->page;
>  	struct inode *inode = file_inode(vmf->vma->vm_file);
>  	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
> -	struct dnode_of_data dn = { .node_changed = false };
> +	struct dnode_of_data dn;
>  	int err;
>  
>  	if (unlikely(f2fs_cp_error(sbi))) {
> @@ -63,6 +63,9 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf)
>  		goto err;
>  	}
>  
> +	/* should do out of any locked page */
> +	f2fs_balance_fs(sbi, true);
> +
>  	sb_start_pagefault(inode->i_sb);
>  
>  	f2fs_bug_on(sbi, f2fs_has_inline_data(inode));
> @@ -120,8 +123,6 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf)
>  out_sem:
>  	up_read(&F2FS_I(inode)->i_mmap_sem);
>  
> -	f2fs_balance_fs(sbi, dn.node_changed);
> -
>  	sb_end_pagefault(inode->i_sb);
>  err:
>  	return block_page_mkwrite_return(err);
> -- 
> 2.19.0.605.g01d371f741-goog
> 

Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

Hello,

I observe hung background f2fs task on 5.6-rc2+ shortly after boot, leading to
panics. I use f2fs as rootfs. See below for stack trace. The reads continue to
work (if I disable panic on hung task). But sync blocks, writes block, etc.

It does not happen on all my f2fs filesystems, so it may depend on workload
or my particular filesystem state. It happens on two separate devices though,
both 32-bit, and doesn't happen on a 64-bit device. (might be a false lead,
though)

I went through the f2fs-for-5.6 tag/branch and reverted each patch right
down to:

  4c8ff7095bef64fc47e996a938f7d57f9e077da3 f2fs: support data compression

I could still reproduce the issue.

After reverting the data compression too, I could no longer reproduce the
issue. Perhaps not very helpful, since that patch is huge.

I tried to collect some information. Some ftrace logs, etc. Not sure what would
help debug this. Let me know if I can help debug this more.

Symptoms look the same as this issue:

  https://www.mail-archive.com/linux-f2fs-devel@lists.sourceforge.net/msg15298.html

ftrace: https://megous.com/dl/tmp/f2fs-debug-info.txt

longer ftrace: https://megous.com/dl/tmp/f2fs-debug-info-full.txt

/dev/root / f2fs rw,lazytime,relatime,background_gc=on,discard,no_heap,user_xattr,inline_xattr,acl,inline_data,inline_dentry,flush_merge,fastboot,extent_cache,mode=adaptive,active_logs=6,alloc_mode=default,fsync_mode=posix 0 0

thank you and regards,
	Ondrej

f2fs/status:

=====[ partition info(mmcblk0p2). #0, RW, CP: Good]=====
[SB: 1] [CP: 2] [SIT: 2] [NAT: 34] [SSA: 15] [MAIN: 7414(OverProv:248 Resv:129)]

Utilization: 17% (634784 valid blocks, 3172662 discard blocks)
  - Node: 57661 (Inode: 57305, Other: 356)
  - Data: 577123
  - Inline_xattr Inode: 2586
  - Inline_data Inode: 922
  - Inline_dentry Inode: 166
  - Compressed Inode: 0, Blocks: 0
  - Orphan/Append/Update Inode: 0, 0, 0

Main area: 7414 segs, 7414 secs 7414 zones
  - COLD  data: 6041, 6041, 6041
  - WARM  data: 141, 141, 141
  - HOT   data: 137, 137, 137
  - Dir   dnode: 44, 44, 44
  - File   dnode: 24, 24, 24
  - Indir nodes: 2, 2, 2

  - Valid: 1097
  - Dirty: 136
  - Prefree: 0
  - Free: 6181 (6181)

CP calls: 18 (BG: 0)
  - cp blocks : 126
  - sit blocks : 21
  - nat blocks : 16
  - ssa blocks : 0
GC calls: 1 (BG: 1)
  - data segments : 1 (1)
  - node segments : 0 (0)
Try to move 265 blocks (BG: 265)
  - data blocks : 265 (265)
  - node blocks : 0 (0)
Skipped : atomic write 0 (0)
BG skip : IO: 13, Other: 0

Extent Cache:
  - Hit Count: L1-1:619 L1-2:0 L2:0
  - Hit Ratio: 42% (619 / 1463)
  - Inner Struct Count: tree: 2349(0), node: 1183

Balancing F2FS Async:
  - DIO (R:    0, W:    0)
  - IO_R (Data:    0, Node:    0, Meta:    0
  - IO_W (CP:    1, Data:    0, Flush: (   0    2    1), Discard: (   0    0)) cmd:   13 undiscard:  36
  - inmem:    0, atomic IO:    0 (Max.    0), volatile IO:    0 (Max.    0)
  - nodes:   18 in 4219
  - dents:    1 in dirs:   1 (   4)
  - datas:  267 in files:   0
  - quota datas:    0 in quota files:   0
  - meta:    0 in  301
  - imeta:    2
  - NATs:        43/     4294
  - SITs:         4/     7414
  - free_nids:      3217/  3902615
  - alloc_nids:         0

Distribution of User Blocks: [ valid | invalid | free ]
  [--------|-|-----------------------------------------]

IPU: 15 blocks
SSR: 0 blocks in 0 segments
LFS: 67 blocks in 0 segments

BDF: 99, avg. vblocks: 455

Memory: 20583 KB
  - static: 2194 KB
  - cached: 308 KB
  - paged : 18080 KB



dmesg:

[  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
[  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
[  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  246.758052] kworker/u16:1   D    0    58      2 0x00000000
[  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
[  246.758099] Backtrace:
[  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
[  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
[  246.758132]  r4:da4d3600
[  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
[  246.758152]  r5:00000001 r4:da283e00
[  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
[  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
[  246.758169]  r4:da283e00
[  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)
[  246.758190]  r5:eff213b0 r4:da283c60
[  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
[  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
[  246.758206]  r4:eff213b0
[  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
[  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
[  246.758223]  r4:da645d80
[  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
[  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
[  246.758246]  r4:da283c60
[  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
[  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
[  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
[  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
[  246.758274]  r4:d9dc9848
[  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
[  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
[  246.758289]  r4:da5a8800
[  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
[  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
[  246.758305]  r4:d9dc9848
[  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
[  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
[  246.758321]  r4:000031e6
[  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
[  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
[  246.758343]  r4:d9dc9954
[  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
[  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
[  246.758359]  r4:da5eb000
[  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
[  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
[  246.758377]  r4:dabf3240
[  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
[  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
[  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
[  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
[  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
[  246.758416]  r4:da5fe000

Re: Writes stoped working on f2fs after the compression support was added

[Chao Yu]

Hi,

Thanks for the report.

On 2020/2/23 2:17, Ondřej Jirman wrote:
> Hello,
> 
> I observe hung background f2fs task on 5.6-rc2+ shortly after boot, leading to
> panics. I use f2fs as rootfs. See below for stack trace. The reads continue to
> work (if I disable panic on hung task). But sync blocks, writes block, etc.
> 
> It does not happen on all my f2fs filesystems, so it may depend on workload
> or my particular filesystem state. It happens on two separate devices though,
> both 32-bit, and doesn't happen on a 64-bit device. (might be a false lead,
> though)
> 
> I went through the f2fs-for-5.6 tag/branch and reverted each patch right
> down to:
> 
>   4c8ff7095bef64fc47e996a938f7d57f9e077da3 f2fs: support data compression
> 
> I could still reproduce the issue.
> 
> After reverting the data compression too, I could no longer reproduce the
> issue. Perhaps not very helpful, since that patch is huge.
> 
> I tried to collect some information. Some ftrace logs, etc. Not sure what would
> help debug this. Let me know if I can help debug this more.
> 
> Symptoms look the same as this issue:
> 
>   https://www.mail-archive.com/linux-f2fs-devel@lists.sourceforge.net/msg15298.html
> 
> ftrace: https://megous.com/dl/tmp/f2fs-debug-info.txt
> 
> longer ftrace: https://megous.com/dl/tmp/f2fs-debug-info-full.txt

I checked the full trace log, however I didn't find any clue to troubleshot this issue.

[snip]

> dmesg:

Could you dump all other task stack info via "echo "t" > /proc/sysrq-trigger"?

> 
> [  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
> [  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
> [  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> [  246.758052] kworker/u16:1   D    0    58      2 0x00000000
> [  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
> [  246.758099] Backtrace:
> [  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
> [  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
> [  246.758132]  r4:da4d3600
> [  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
> [  246.758152]  r5:00000001 r4:da283e00
> [  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
> [  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
> [  246.758169]  r4:da283e00
> [  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)

I'm not sure what is this semaphore, I suspect this is F2FS_I(inode)->i_sem, in order to make
sure of this, can you help to add below function, and use them to replace
all {down,up}_{write,read}(&.i_sem) invoking? then reproduce this issue and catch the log.

Thanks,

> [  246.758190]  r5:eff213b0 r4:da283c60
> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> [  246.758206]  r4:eff213b0
> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> [  246.758223]  r4:da645d80
> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> [  246.758246]  r4:da283c60
> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> [  246.758274]  r4:d9dc9848
> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> [  246.758289]  r4:da5a8800
> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> [  246.758305]  r4:d9dc9848
> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> [  246.758321]  r4:000031e6
> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> [  246.758343]  r4:d9dc9954
> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> [  246.758359]  r4:da5eb000
> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> [  246.758377]  r4:dabf3240
> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> [  246.758416]  r4:da5fe000
> .
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Chao Yu]

On 2020/2/24 18:37, Chao Yu wrote:
> Hi,
> 
> Thanks for the report.
> 
> On 2020/2/23 2:17, Ondřej Jirman wrote:
>> Hello,
>>
>> I observe hung background f2fs task on 5.6-rc2+ shortly after boot, leading to
>> panics. I use f2fs as rootfs. See below for stack trace. The reads continue to
>> work (if I disable panic on hung task). But sync blocks, writes block, etc.
>>
>> It does not happen on all my f2fs filesystems, so it may depend on workload
>> or my particular filesystem state. It happens on two separate devices though,
>> both 32-bit, and doesn't happen on a 64-bit device. (might be a false lead,
>> though)
>>
>> I went through the f2fs-for-5.6 tag/branch and reverted each patch right
>> down to:
>>
>>   4c8ff7095bef64fc47e996a938f7d57f9e077da3 f2fs: support data compression
>>
>> I could still reproduce the issue.
>>
>> After reverting the data compression too, I could no longer reproduce the
>> issue. Perhaps not very helpful, since that patch is huge.
>>
>> I tried to collect some information. Some ftrace logs, etc. Not sure what would
>> help debug this. Let me know if I can help debug this more.
>>
>> Symptoms look the same as this issue:
>>
>>   https://www.mail-archive.com/linux-f2fs-devel@lists.sourceforge.net/msg15298.html
>>
>> ftrace: https://megous.com/dl/tmp/f2fs-debug-info.txt
>>
>> longer ftrace: https://megous.com/dl/tmp/f2fs-debug-info-full.txt
> 
> I checked the full trace log, however I didn't find any clue to troubleshot this issue.
> 
> [snip]
> 
>> dmesg:
> 
> Could you dump all other task stack info via "echo "t" > /proc/sysrq-trigger"?
> 
>>
>> [  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
>> [  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
>> [  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
>> [  246.758052] kworker/u16:1   D    0    58      2 0x00000000
>> [  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
>> [  246.758099] Backtrace:
>> [  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
>> [  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
>> [  246.758132]  r4:da4d3600
>> [  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
>> [  246.758152]  r5:00000001 r4:da283e00
>> [  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
>> [  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
>> [  246.758169]  r4:da283e00
>> [  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)
> 
> I'm not sure what is this semaphore, I suspect this is F2FS_I(inode)->i_sem, in order to make
> sure of this, can you help to add below function, and use them to replace
> all {down,up}_{write,read}(&.i_sem) invoking? then reproduce this issue and catch the log.

Sorry, just forgot attaching below function.

void inode_down_write(struct inode *inode)
{
	printk("%s from %pS\n", __func__, __builtin_return_address(0));
	down_write(&F2FS_I(inode)->i_sem);
}

void inode_up_write(struct inode *inode)
{
	up_write(&F2FS_I(inode)->i_sem);
	printk("%s from %pS\n", __func__, __builtin_return_address(0));
}

void inode_down_read(struct inode *inode)
{
	printk("%s from %pS\n", __func__, __builtin_return_address(0));
	down_read(&F2FS_I(inode)->i_sem);
}

void inode_up_read(struct inode *inode)
{
	up_read(&F2FS_I(inode)->i_sem);
	printk("%s from %pS\n", __func__, __builtin_return_address(0));
}

> 
> Thanks,
> 
>> [  246.758190]  r5:eff213b0 r4:da283c60
>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
>> [  246.758206]  r4:eff213b0
>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
>> [  246.758223]  r4:da645d80
>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
>> [  246.758246]  r4:da283c60
>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
>> [  246.758274]  r4:d9dc9848
>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
>> [  246.758289]  r4:da5a8800
>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
>> [  246.758305]  r4:d9dc9848
>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
>> [  246.758321]  r4:000031e6
>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
>> [  246.758343]  r4:d9dc9954
>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
>> [  246.758359]  r4:da5eb000
>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
>> [  246.758377]  r4:dabf3240
>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
>> [  246.758416]  r4:da5fe000
>> .
>>
> 
> 
> _______________________________________________
> Linux-f2fs-devel mailing list
> Linux-f2fs-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

Hello,

On Mon, Feb 24, 2020 at 06:41:03PM +0800, Chao Yu wrote:
> On 2020/2/24 18:37, Chao Yu wrote:
> > Hi,
> > 
> > Thanks for the report.
> > 
> > Could you dump all other task stack info via "echo "t" > /proc/sysrq-trigger"?
> > 
> >>
> >> [  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
> >> [  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
> >> [  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> >> [  246.758052] kworker/u16:1   D    0    58      2 0x00000000
> >> [  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
> >> [  246.758099] Backtrace:
> >> [  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
> >> [  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
> >> [  246.758132]  r4:da4d3600
> >> [  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
> >> [  246.758152]  r5:00000001 r4:da283e00
> >> [  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
> >> [  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
> >> [  246.758169]  r4:da283e00
> >> [  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)
> > 
> > I'm not sure what is this semaphore, I suspect this is F2FS_I(inode)->i_sem, in order to make
> > sure of this, can you help to add below function, and use them to replace
> > all {down,up}_{write,read}(&.i_sem) invoking? then reproduce this issue and catch the log.
> 
> Sorry, just forgot attaching below function.
> 
> void inode_down_write(struct inode *inode)
> {
> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> 	down_write(&F2FS_I(inode)->i_sem);
> }
> 
> void inode_up_write(struct inode *inode)
> {
> 	up_write(&F2FS_I(inode)->i_sem);
> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> }
> 
> void inode_down_read(struct inode *inode)
> {
> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> 	down_read(&F2FS_I(inode)->i_sem);
> }
> 
> void inode_up_read(struct inode *inode)
> {
> 	up_read(&F2FS_I(inode)->i_sem);
> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> }
> 

Here's the log and vmlinux file that may help mapping the code addresses back to
code, hope it helps:

https://megous.com/dl/tmp/f2fs-dmesg-log
https://megous.com/dl/tmp/f2fs-log-build-artifacts.tar.gz

thank you,
	o.

> > Thanks,
> > 
> >> [  246.758190]  r5:eff213b0 r4:da283c60
> >> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> >> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> >> [  246.758206]  r4:eff213b0
> >> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> >> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> >> [  246.758223]  r4:da645d80
> >> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> >> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> >> [  246.758246]  r4:da283c60
> >> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> >> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> >> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> >> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> >> [  246.758274]  r4:d9dc9848
> >> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> >> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> >> [  246.758289]  r4:da5a8800
> >> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> >> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> >> [  246.758305]  r4:d9dc9848
> >> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> >> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> >> [  246.758321]  r4:000031e6
> >> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> >> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> >> [  246.758343]  r4:d9dc9954
> >> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> >> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> >> [  246.758359]  r4:da5eb000
> >> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> >> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> >> [  246.758377]  r4:dabf3240
> >> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> >> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> >> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> >> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> >> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> >> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> >> [  246.758416]  r4:da5fe000
> >> .
> >>
> > 
> > 
> > _______________________________________________
> > Linux-f2fs-devel mailing list
> > Linux-f2fs-devel@lists.sourceforge.net
> > https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> > 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

On Mon, Feb 24, 2020 at 02:58:37PM +0100, megi xff wrote:
> Hello,
> 
> On Mon, Feb 24, 2020 at 06:41:03PM +0800, Chao Yu wrote:
> > On 2020/2/24 18:37, Chao Yu wrote:
> > > Hi,
> > > 
> > > Thanks for the report.
> > > 
> > > Could you dump all other task stack info via "echo "t" > /proc/sysrq-trigger"?
> > > 
> > >>
> > >> [  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
> > >> [  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
> > >> [  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> > >> [  246.758052] kworker/u16:1   D    0    58      2 0x00000000
> > >> [  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
> > >> [  246.758099] Backtrace:
> > >> [  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
> > >> [  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
> > >> [  246.758132]  r4:da4d3600
> > >> [  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
> > >> [  246.758152]  r5:00000001 r4:da283e00
> > >> [  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
> > >> [  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
> > >> [  246.758169]  r4:da283e00
> > >> [  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)
> > > 
> > > I'm not sure what is this semaphore, I suspect this is F2FS_I(inode)->i_sem, in order to make
> > > sure of this, can you help to add below function, and use them to replace
> > > all {down,up}_{write,read}(&.i_sem) invoking? then reproduce this issue and catch the log.
> > 
> > Sorry, just forgot attaching below function.
> > 
> > void inode_down_write(struct inode *inode)
> > {
> > 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> > 	down_write(&F2FS_I(inode)->i_sem);
> > }
> > 
> > void inode_up_write(struct inode *inode)
> > {
> > 	up_write(&F2FS_I(inode)->i_sem);
> > 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> > }
> > 
> > void inode_down_read(struct inode *inode)
> > {
> > 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> > 	down_read(&F2FS_I(inode)->i_sem);
> > }
> > 
> > void inode_up_read(struct inode *inode)
> > {
> > 	up_read(&F2FS_I(inode)->i_sem);
> > 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> > }
> > 
> 
> Here's the log and vmlinux file that may help mapping the code addresses back to
> code, hope it helps:
> 
> https://megous.com/dl/tmp/f2fs-dmesg-log
> https://megous.com/dl/tmp/f2fs-log-build-artifacts.tar.gz

Just by a looks of it:

root@tbs2[/proc/sys/kernel] # dmesg | grep up_write | wc -l
324
root@tbs2[/proc/sys/kernel] # dmesg | grep down_write | wc -l
347

there seems to be a mismatch of lock/unlock counts.

root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
16
root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
16

regards,
	o.

> thank you,
> 	o.
> 
> > > Thanks,
> > > 
> > >> [  246.758190]  r5:eff213b0 r4:da283c60
> > >> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> > >> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> > >> [  246.758206]  r4:eff213b0
> > >> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> > >> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> > >> [  246.758223]  r4:da645d80
> > >> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> > >> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> > >> [  246.758246]  r4:da283c60
> > >> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> > >> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> > >> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> > >> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> > >> [  246.758274]  r4:d9dc9848
> > >> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> > >> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> > >> [  246.758289]  r4:da5a8800
> > >> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> > >> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> > >> [  246.758305]  r4:d9dc9848
> > >> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> > >> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> > >> [  246.758321]  r4:000031e6
> > >> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> > >> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> > >> [  246.758343]  r4:d9dc9954
> > >> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> > >> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> > >> [  246.758359]  r4:da5eb000
> > >> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> > >> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> > >> [  246.758377]  r4:dabf3240
> > >> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> > >> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> > >> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> > >> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> > >> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> > >> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> > >> [  246.758416]  r4:da5fe000
> > >> .
> > >>
> > > 
> > > 
> > > _______________________________________________
> > > Linux-f2fs-devel mailing list
> > > Linux-f2fs-devel@lists.sourceforge.net
> > > https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> > > 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

On Mon, Feb 24, 2020 at 03:03:49PM +0100, megi xff wrote:
> On Mon, Feb 24, 2020 at 02:58:37PM +0100, megi xff wrote:
> > Hello,
> > 
> > On Mon, Feb 24, 2020 at 06:41:03PM +0800, Chao Yu wrote:
> > > On 2020/2/24 18:37, Chao Yu wrote:
> > > > Hi,
> > > > 
> > > > Thanks for the report.
> > > > 
> > > > Could you dump all other task stack info via "echo "t" > /proc/sysrq-trigger"?
> > > > 
> > > >>
> > > >> [  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
> > > >> [  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
> > > >> [  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> > > >> [  246.758052] kworker/u16:1   D    0    58      2 0x00000000
> > > >> [  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
> > > >> [  246.758099] Backtrace:
> > > >> [  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
> > > >> [  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
> > > >> [  246.758132]  r4:da4d3600
> > > >> [  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
> > > >> [  246.758152]  r5:00000001 r4:da283e00
> > > >> [  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
> > > >> [  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
> > > >> [  246.758169]  r4:da283e00
> > > >> [  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)
> > > > 
> > > > I'm not sure what is this semaphore, I suspect this is F2FS_I(inode)->i_sem, in order to make
> > > > sure of this, can you help to add below function, and use them to replace
> > > > all {down,up}_{write,read}(&.i_sem) invoking? then reproduce this issue and catch the log.
> > > 
> > > Sorry, just forgot attaching below function.
> > > 
> > > void inode_down_write(struct inode *inode)
> > > {
> > > 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> > > 	down_write(&F2FS_I(inode)->i_sem);
> > > }
> > > 
> > > void inode_up_write(struct inode *inode)
> > > {
> > > 	up_write(&F2FS_I(inode)->i_sem);
> > > 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> > > }
> > > 
> > > void inode_down_read(struct inode *inode)
> > > {
> > > 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> > > 	down_read(&F2FS_I(inode)->i_sem);
> > > }
> > > 
> > > void inode_up_read(struct inode *inode)
> > > {
> > > 	up_read(&F2FS_I(inode)->i_sem);
> > > 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> > > }
> > > 
> > 
> > Here's the log and vmlinux file that may help mapping the code addresses back to
> > code, hope it helps:
> > 
> > https://megous.com/dl/tmp/f2fs-dmesg-log
> > https://megous.com/dl/tmp/f2fs-log-build-artifacts.tar.gz
> 
> Just by a looks of it:
> 
> root@tbs2[/proc/sys/kernel] # dmesg | grep up_write | wc -l
> 324
> root@tbs2[/proc/sys/kernel] # dmesg | grep down_write | wc -l
> 347
> 
> there seems to be a mismatch of lock/unlock counts.
 
Sorry, a wrong grep expression.

root@tbs2[~] # dmesg | grep inode_down_write | wc -l
357
root@tbs2[~] # dmesg | grep inode_up_write | wc -l
357
root@tbs2[~] # dmesg | grep inode_up_read | wc -l
16
root@tbs2[~] # dmesg | grep inode_down_read | wc -l
16

So it's probably not inode locking.

> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
> 16
> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
> 16
> 
> regards,
> 	o.
> 
> > thank you,
> > 	o.
> > 
> > > > Thanks,
> > > > 
> > > >> [  246.758190]  r5:eff213b0 r4:da283c60
> > > >> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> > > >> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> > > >> [  246.758206]  r4:eff213b0
> > > >> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> > > >> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> > > >> [  246.758223]  r4:da645d80
> > > >> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> > > >> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> > > >> [  246.758246]  r4:da283c60
> > > >> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> > > >> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> > > >> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> > > >> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> > > >> [  246.758274]  r4:d9dc9848
> > > >> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> > > >> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> > > >> [  246.758289]  r4:da5a8800
> > > >> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> > > >> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> > > >> [  246.758305]  r4:d9dc9848
> > > >> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> > > >> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> > > >> [  246.758321]  r4:000031e6
> > > >> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> > > >> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> > > >> [  246.758343]  r4:d9dc9954
> > > >> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> > > >> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> > > >> [  246.758359]  r4:da5eb000
> > > >> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> > > >> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> > > >> [  246.758377]  r4:dabf3240
> > > >> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> > > >> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> > > >> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> > > >> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> > > >> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> > > >> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> > > >> [  246.758416]  r4:da5fe000
> > > >> .
> > > >>
> > > > 
> > > > 
> > > > _______________________________________________
> > > > Linux-f2fs-devel mailing list
> > > > Linux-f2fs-devel@lists.sourceforge.net
> > > > https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> > > > 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Chao Yu]

On 2020/2/24 22:31, Ondřej Jirman wrote:
> On Mon, Feb 24, 2020 at 03:03:49PM +0100, megi xff wrote:
>> On Mon, Feb 24, 2020 at 02:58:37PM +0100, megi xff wrote:
>>> Hello,
>>>
>>> On Mon, Feb 24, 2020 at 06:41:03PM +0800, Chao Yu wrote:
>>>> On 2020/2/24 18:37, Chao Yu wrote:
>>>>> Hi,
>>>>>
>>>>> Thanks for the report.
>>>>>
>>>>> Could you dump all other task stack info via "echo "t" > /proc/sysrq-trigger"?
>>>>>
>>>>>>
>>>>>> [  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
>>>>>> [  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
>>>>>> [  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
>>>>>> [  246.758052] kworker/u16:1   D    0    58      2 0x00000000
>>>>>> [  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
>>>>>> [  246.758099] Backtrace:
>>>>>> [  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
>>>>>> [  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
>>>>>> [  246.758132]  r4:da4d3600
>>>>>> [  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
>>>>>> [  246.758152]  r5:00000001 r4:da283e00
>>>>>> [  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
>>>>>> [  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
>>>>>> [  246.758169]  r4:da283e00
>>>>>> [  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)
>>>>>
>>>>> I'm not sure what is this semaphore, I suspect this is F2FS_I(inode)->i_sem, in order to make
>>>>> sure of this, can you help to add below function, and use them to replace
>>>>> all {down,up}_{write,read}(&.i_sem) invoking? then reproduce this issue and catch the log.
>>>>
>>>> Sorry, just forgot attaching below function.
>>>>
>>>> void inode_down_write(struct inode *inode)
>>>> {
>>>> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
>>>> 	down_write(&F2FS_I(inode)->i_sem);
>>>> }
>>>>
>>>> void inode_up_write(struct inode *inode)
>>>> {
>>>> 	up_write(&F2FS_I(inode)->i_sem);
>>>> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
>>>> }
>>>>
>>>> void inode_down_read(struct inode *inode)
>>>> {
>>>> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
>>>> 	down_read(&F2FS_I(inode)->i_sem);
>>>> }
>>>>
>>>> void inode_up_read(struct inode *inode)
>>>> {
>>>> 	up_read(&F2FS_I(inode)->i_sem);
>>>> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
>>>> }
>>>>
>>>
>>> Here's the log and vmlinux file that may help mapping the code addresses back to
>>> code, hope it helps:
>>>
>>> https://megous.com/dl/tmp/f2fs-dmesg-log
>>> https://megous.com/dl/tmp/f2fs-log-build-artifacts.tar.gz
>>
>> Just by a looks of it:
>>
>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_write | wc -l
>> 324
>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_write | wc -l
>> 347
>>
>> there seems to be a mismatch of lock/unlock counts.
>  
> Sorry, a wrong grep expression.
> 
> root@tbs2[~] # dmesg | grep inode_down_write | wc -l
> 357
> root@tbs2[~] # dmesg | grep inode_up_write | wc -l
> 357
> root@tbs2[~] # dmesg | grep inode_up_read | wc -l
> 16
> root@tbs2[~] # dmesg | grep inode_down_read | wc -l
> 16

I don't know why we have consistent down/up pair, but through disassembled
code, I doubt it's the f2fs_inode->i_sem.

c0435d7c:       ebf54af8        bl      c0188964 <printk>
c0435d80:       e1a00006        mov     r0, r6
c0435d84:       eb138135        bl      c0916260 <down_write>

inode_down_write()

c0435d88:       e284ce1d        add     ip, r4, #464    ; 0x1d0

We are stuck here.

[  430.675754] [<c0916260>] (down_write) from [<c0435d88>] (f2fs_write_single_data_page+0x600/0x7d8)
                                                ^^^^^^^^^
[  430.675764] [<c0435788>] (f2fs_write_single_data_page) from [<c0436214>] (f2fs_write_cache_pages+0x2b4/0x7c4)


c0435d8c:       e14b0ad4        ldrd    r0, [fp, #-164] ; 0xffffff5c
c0435d90:       e1cc20d0        ldrd    r2, [ip]
c0435d94:       e1520000        cmp     r2, r0
c0435d98:       e0d33001        sbcs    r3, r3, r1
c0435d9c:       b1cc00f0        strdlt  r0, [ip]
c0435da0:       e1a00006        mov     r0, r6
c0435da4:       ebf52227        bl      c017e648 <up_write>
c0435da8:       e51b2098        ldr     r2, [fp, #-152] ; 0xffffff68
c0435dac:       e30c0730        movw    r0, #50992      ; 0xc730
c0435db0:       e59f11a4        ldr     r1, [pc, #420]  ; c0435f5c <f2fs_write_single_data_page+0x7d4>
c0435db4:       e34c00b6        movt    r0, #49334      ; 0xc0b6
c0435db8:       ebf54ae9        bl      c0188964 <printk>

inode_up_write()

Thanks,

> 
> So it's probably not inode locking.
> 
>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
>> 16
>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
>> 16
>>
>> regards,
>> 	o.
>>
>>> thank you,
>>> 	o.
>>>
>>>>> Thanks,
>>>>>
>>>>>> [  246.758190]  r5:eff213b0 r4:da283c60
>>>>>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
>>>>>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
>>>>>> [  246.758206]  r4:eff213b0
>>>>>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
>>>>>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
>>>>>> [  246.758223]  r4:da645d80
>>>>>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
>>>>>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
>>>>>> [  246.758246]  r4:da283c60
>>>>>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
>>>>>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
>>>>>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
>>>>>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
>>>>>> [  246.758274]  r4:d9dc9848
>>>>>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
>>>>>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
>>>>>> [  246.758289]  r4:da5a8800
>>>>>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
>>>>>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
>>>>>> [  246.758305]  r4:d9dc9848
>>>>>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
>>>>>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
>>>>>> [  246.758321]  r4:000031e6
>>>>>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
>>>>>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
>>>>>> [  246.758343]  r4:d9dc9954
>>>>>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
>>>>>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
>>>>>> [  246.758359]  r4:da5eb000
>>>>>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
>>>>>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
>>>>>> [  246.758377]  r4:dabf3240
>>>>>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
>>>>>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
>>>>>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
>>>>>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
>>>>>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
>>>>>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
>>>>>> [  246.758416]  r4:da5fe000
>>>>>> .
>>>>>>
>>>>>
>>>>>
>>>>> _______________________________________________
>>>>> Linux-f2fs-devel mailing list
>>>>> Linux-f2fs-devel@lists.sourceforge.net
>>>>> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
>>>>>
> .
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Chao Yu]

On 2020/2/25 19:24, Chao Yu wrote:
> On 2020/2/24 22:31, Ondřej Jirman wrote:
>> On Mon, Feb 24, 2020 at 03:03:49PM +0100, megi xff wrote:
>>> On Mon, Feb 24, 2020 at 02:58:37PM +0100, megi xff wrote:
>>>> Hello,
>>>>
>>>> On Mon, Feb 24, 2020 at 06:41:03PM +0800, Chao Yu wrote:
>>>>> On 2020/2/24 18:37, Chao Yu wrote:
>>>>>> Hi,
>>>>>>
>>>>>> Thanks for the report.
>>>>>>
>>>>>> Could you dump all other task stack info via "echo "t" > /proc/sysrq-trigger"?
>>>>>>
>>>>>>>
>>>>>>> [  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
>>>>>>> [  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
>>>>>>> [  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
>>>>>>> [  246.758052] kworker/u16:1   D    0    58      2 0x00000000
>>>>>>> [  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
>>>>>>> [  246.758099] Backtrace:
>>>>>>> [  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
>>>>>>> [  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
>>>>>>> [  246.758132]  r4:da4d3600
>>>>>>> [  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
>>>>>>> [  246.758152]  r5:00000001 r4:da283e00
>>>>>>> [  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
>>>>>>> [  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
>>>>>>> [  246.758169]  r4:da283e00
>>>>>>> [  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)
>>>>>>
>>>>>> I'm not sure what is this semaphore, I suspect this is F2FS_I(inode)->i_sem, in order to make
>>>>>> sure of this, can you help to add below function, and use them to replace
>>>>>> all {down,up}_{write,read}(&.i_sem) invoking? then reproduce this issue and catch the log.
>>>>>
>>>>> Sorry, just forgot attaching below function.
>>>>>
>>>>> void inode_down_write(struct inode *inode)
>>>>> {
>>>>> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
>>>>> 	down_write(&F2FS_I(inode)->i_sem);
>>>>> }
>>>>>
>>>>> void inode_up_write(struct inode *inode)
>>>>> {
>>>>> 	up_write(&F2FS_I(inode)->i_sem);
>>>>> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
>>>>> }
>>>>>
>>>>> void inode_down_read(struct inode *inode)
>>>>> {
>>>>> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
>>>>> 	down_read(&F2FS_I(inode)->i_sem);
>>>>> }
>>>>>
>>>>> void inode_up_read(struct inode *inode)
>>>>> {
>>>>> 	up_read(&F2FS_I(inode)->i_sem);
>>>>> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
>>>>> }
>>>>>
>>>>
>>>> Here's the log and vmlinux file that may help mapping the code addresses back to
>>>> code, hope it helps:
>>>>
>>>> https://megous.com/dl/tmp/f2fs-dmesg-log
>>>> https://megous.com/dl/tmp/f2fs-log-build-artifacts.tar.gz
>>>
>>> Just by a looks of it:
>>>
>>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_write | wc -l
>>> 324
>>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_write | wc -l
>>> 347
>>>
>>> there seems to be a mismatch of lock/unlock counts.
>>  
>> Sorry, a wrong grep expression.
>>
>> root@tbs2[~] # dmesg | grep inode_down_write | wc -l
>> 357
>> root@tbs2[~] # dmesg | grep inode_up_write | wc -l
>> 357
>> root@tbs2[~] # dmesg | grep inode_up_read | wc -l
>> 16
>> root@tbs2[~] # dmesg | grep inode_down_read | wc -l
>> 16
> 
> I don't know why we have consistent down/up pair, but through disassembled
> code, I doubt it's the f2fs_inode->i_sem.
> 
> c0435d7c:       ebf54af8        bl      c0188964 <printk>
> c0435d80:       e1a00006        mov     r0, r6
> c0435d84:       eb138135        bl      c0916260 <down_write>
> 
> inode_down_write()
> 
> c0435d88:       e284ce1d        add     ip, r4, #464    ; 0x1d0
> 
> We are stuck here.
> 
> [  430.675754] [<c0916260>] (down_write) from [<c0435d88>] (f2fs_write_single_data_page+0x600/0x7d8)
>                                                 ^^^^^^^^^
> [  430.675764] [<c0435788>] (f2fs_write_single_data_page) from [<c0436214>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> 
> 
> c0435d8c:       e14b0ad4        ldrd    r0, [fp, #-164] ; 0xffffff5c
> c0435d90:       e1cc20d0        ldrd    r2, [ip]
> c0435d94:       e1520000        cmp     r2, r0
> c0435d98:       e0d33001        sbcs    r3, r3, r1
> c0435d9c:       b1cc00f0        strdlt  r0, [ip]
> c0435da0:       e1a00006        mov     r0, r6
> c0435da4:       ebf52227        bl      c017e648 <up_write>
> c0435da8:       e51b2098        ldr     r2, [fp, #-152] ; 0xffffff68
> c0435dac:       e30c0730        movw    r0, #50992      ; 0xc730
> c0435db0:       e59f11a4        ldr     r1, [pc, #420]  ; c0435f5c <f2fs_write_single_data_page+0x7d4>
> c0435db4:       e34c00b6        movt    r0, #49334      ; 0xc0b6
> c0435db8:       ebf54ae9        bl      c0188964 <printk>
> 
> inode_up_write()

Can we have a try with below diff to avoid call down_write under page lock?

Not sure it can solve this issue.

diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index cb41260ca941..f145c1ea977d 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -2650,11 +2650,6 @@ int f2fs_write_single_data_page(struct page *page, int *submitted,

 	if (err) {
 		file_set_keep_isize(inode);
-	} else {
-		down_write(&F2FS_I(inode)->i_sem);
-		if (F2FS_I(inode)->last_disk_size < psize)
-			F2FS_I(inode)->last_disk_size = psize;
-		up_write(&F2FS_I(inode)->i_sem);
 	}

 done:
@@ -2675,6 +2670,14 @@ int f2fs_write_single_data_page(struct page *page, int *submitted,
 		submitted = NULL;
 	}
 	unlock_page(page);
+
+	if (!err) {
+		down_write(&F2FS_I(inode)->i_sem);
+		if (F2FS_I(inode)->last_disk_size < psize)
+			F2FS_I(inode)->last_disk_size = psize;
+		up_write(&F2FS_I(inode)->i_sem);
+	}
+
 	if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
 					!F2FS_I(inode)->cp_task)
 		f2fs_balance_fs(sbi, need_balance_fs);

Thanks,

> 
> Thanks,
> 
>>
>> So it's probably not inode locking.
>>
>>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
>>> 16
>>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
>>> 16
>>>
>>> regards,
>>> 	o.
>>>
>>>> thank you,
>>>> 	o.
>>>>
>>>>>> Thanks,
>>>>>>
>>>>>>> [  246.758190]  r5:eff213b0 r4:da283c60
>>>>>>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
>>>>>>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
>>>>>>> [  246.758206]  r4:eff213b0
>>>>>>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
>>>>>>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
>>>>>>> [  246.758223]  r4:da645d80
>>>>>>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
>>>>>>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
>>>>>>> [  246.758246]  r4:da283c60
>>>>>>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
>>>>>>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
>>>>>>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
>>>>>>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
>>>>>>> [  246.758274]  r4:d9dc9848
>>>>>>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
>>>>>>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
>>>>>>> [  246.758289]  r4:da5a8800
>>>>>>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
>>>>>>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
>>>>>>> [  246.758305]  r4:d9dc9848
>>>>>>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
>>>>>>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
>>>>>>> [  246.758321]  r4:000031e6
>>>>>>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
>>>>>>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
>>>>>>> [  246.758343]  r4:d9dc9954
>>>>>>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
>>>>>>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
>>>>>>> [  246.758359]  r4:da5eb000
>>>>>>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
>>>>>>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
>>>>>>> [  246.758377]  r4:dabf3240
>>>>>>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
>>>>>>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
>>>>>>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
>>>>>>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
>>>>>>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
>>>>>>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
>>>>>>> [  246.758416]  r4:da5fe000
>>>>>>> .
>>>>>>>
>>>>>>
>>>>>>
>>>>>> _______________________________________________
>>>>>> Linux-f2fs-devel mailing list
>>>>>> Linux-f2fs-devel@lists.sourceforge.net
>>>>>> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
>>>>>>
>> .
>>
> 
> 
> _______________________________________________
> Linux-f2fs-devel mailing list
> Linux-f2fs-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

Hello,

On Tue, Feb 25, 2020 at 07:24:12PM +0800, Chao Yu wrote:
> On 2020/2/24 22:31, Ondřej Jirman wrote:
> >> Just by a looks of it:
> >>
> >> root@tbs2[/proc/sys/kernel] # dmesg | grep up_write | wc -l
> >> 324
> >> root@tbs2[/proc/sys/kernel] # dmesg | grep down_write | wc -l
> >> 347
> >>
> >> there seems to be a mismatch of lock/unlock counts.
> >  
> > Sorry, a wrong grep expression.
> > 
> > root@tbs2[~] # dmesg | grep inode_down_write | wc -l
> > 357
> > root@tbs2[~] # dmesg | grep inode_up_write | wc -l
> > 357
> > root@tbs2[~] # dmesg | grep inode_up_read | wc -l
> > 16
> > root@tbs2[~] # dmesg | grep inode_down_read | wc -l
> > 16
> 
> I don't know why we have consistent down/up pair, but through disassembled
> code, I doubt it's the f2fs_inode->i_sem.

Because we were counting attempts, and not a successful lock. ;)

> c0435d7c:       ebf54af8        bl      c0188964 <printk>
> c0435d80:       e1a00006        mov     r0, r6
> c0435d84:       eb138135        bl      c0916260 <down_write>
> 
> inode_down_write()
> 
> c0435d88:       e284ce1d        add     ip, r4, #464    ; 0x1d0
> 
> We are stuck here.
> 
> [  430.675754] [<c0916260>] (down_write) from [<c0435d88>] (f2fs_write_single_data_page+0x600/0x7d8)
>                                                 ^^^^^^^^^
> [  430.675764] [<c0435788>] (f2fs_write_single_data_page) from [<c0436214>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> 
> 
> c0435d8c:       e14b0ad4        ldrd    r0, [fp, #-164] ; 0xffffff5c
> c0435d90:       e1cc20d0        ldrd    r2, [ip]
> c0435d94:       e1520000        cmp     r2, r0
> c0435d98:       e0d33001        sbcs    r3, r3, r1
> c0435d9c:       b1cc00f0        strdlt  r0, [ip]
> c0435da0:       e1a00006        mov     r0, r6
> c0435da4:       ebf52227        bl      c017e648 <up_write>
> c0435da8:       e51b2098        ldr     r2, [fp, #-152] ; 0xffffff68
> c0435dac:       e30c0730        movw    r0, #50992      ; 0xc730
> c0435db0:       e59f11a4        ldr     r1, [pc, #420]  ; c0435f5c <f2fs_write_single_data_page+0x7d4>
> c0435db4:       e34c00b6        movt    r0, #49334      ; 0xc0b6
> c0435db8:       ebf54ae9        bl      c0188964 <printk>
> 
> inode_up_write()

The patch you sent helped so far. I'll keep the tablet running for a while,
but so far the issue did not reappear within a few minutes after boot as
usual.

thank you and regards,
	o.

> Thanks,
> 
> > 
> > So it's probably not inode locking.
> > 
> >> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
> >> 16
> >> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
> >> 16
> >>
> >> regards,
> >> 	o.
> >>
> >>> thank you,
> >>> 	o.
> >>>
> >>>>> Thanks,
> >>>>>
> >>>>>> [  246.758190]  r5:eff213b0 r4:da283c60
> >>>>>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> >>>>>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> >>>>>> [  246.758206]  r4:eff213b0
> >>>>>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> >>>>>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> >>>>>> [  246.758223]  r4:da645d80
> >>>>>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> >>>>>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> >>>>>> [  246.758246]  r4:da283c60
> >>>>>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> >>>>>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> >>>>>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> >>>>>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> >>>>>> [  246.758274]  r4:d9dc9848
> >>>>>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> >>>>>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> >>>>>> [  246.758289]  r4:da5a8800
> >>>>>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> >>>>>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> >>>>>> [  246.758305]  r4:d9dc9848
> >>>>>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> >>>>>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> >>>>>> [  246.758321]  r4:000031e6
> >>>>>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> >>>>>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> >>>>>> [  246.758343]  r4:d9dc9954
> >>>>>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> >>>>>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> >>>>>> [  246.758359]  r4:da5eb000
> >>>>>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> >>>>>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> >>>>>> [  246.758377]  r4:dabf3240
> >>>>>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> >>>>>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> >>>>>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> >>>>>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> >>>>>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> >>>>>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> >>>>>> [  246.758416]  r4:da5fe000
> >>>>>> .
> >>>>>>
> >>>>>
> >>>>>
> >>>>> _______________________________________________
> >>>>> Linux-f2fs-devel mailing list
> >>>>> Linux-f2fs-devel@lists.sourceforge.net
> >>>>> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> >>>>>
> > .
> > 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

On Tue, Feb 25, 2020 at 01:08:14PM +0100, megi xff wrote:
> Hello,
> 
> On Tue, Feb 25, 2020 at 07:24:12PM +0800, Chao Yu wrote:
> > On 2020/2/24 22:31, Ondřej Jirman wrote:
> > >> Just by a looks of it:
> > >>
> > >> root@tbs2[/proc/sys/kernel] # dmesg | grep up_write | wc -l
> > >> 324
> > >> root@tbs2[/proc/sys/kernel] # dmesg | grep down_write | wc -l
> > >> 347
> > >>
> > >> there seems to be a mismatch of lock/unlock counts.
> > >  
> > > Sorry, a wrong grep expression.
> > > 
> > > root@tbs2[~] # dmesg | grep inode_down_write | wc -l
> > > 357
> > > root@tbs2[~] # dmesg | grep inode_up_write | wc -l
> > > 357
> > > root@tbs2[~] # dmesg | grep inode_up_read | wc -l
> > > 16
> > > root@tbs2[~] # dmesg | grep inode_down_read | wc -l
> > > 16
> > 
> > I don't know why we have consistent down/up pair, but through disassembled
> > code, I doubt it's the f2fs_inode->i_sem.
> 
> Because we were counting attempts, and not a successful lock. ;)
> 
> > c0435d7c:       ebf54af8        bl      c0188964 <printk>
> > c0435d80:       e1a00006        mov     r0, r6
> > c0435d84:       eb138135        bl      c0916260 <down_write>
> > 
> > inode_down_write()
> > 
> > c0435d88:       e284ce1d        add     ip, r4, #464    ; 0x1d0
> > 
> > We are stuck here.
> > 
> > [  430.675754] [<c0916260>] (down_write) from [<c0435d88>] (f2fs_write_single_data_page+0x600/0x7d8)
> >                                                 ^^^^^^^^^
> > [  430.675764] [<c0435788>] (f2fs_write_single_data_page) from [<c0436214>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> > 
> > 
> > c0435d8c:       e14b0ad4        ldrd    r0, [fp, #-164] ; 0xffffff5c
> > c0435d90:       e1cc20d0        ldrd    r2, [ip]
> > c0435d94:       e1520000        cmp     r2, r0
> > c0435d98:       e0d33001        sbcs    r3, r3, r1
> > c0435d9c:       b1cc00f0        strdlt  r0, [ip]
> > c0435da0:       e1a00006        mov     r0, r6
> > c0435da4:       ebf52227        bl      c017e648 <up_write>
> > c0435da8:       e51b2098        ldr     r2, [fp, #-152] ; 0xffffff68
> > c0435dac:       e30c0730        movw    r0, #50992      ; 0xc730
> > c0435db0:       e59f11a4        ldr     r1, [pc, #420]  ; c0435f5c <f2fs_write_single_data_page+0x7d4>
> > c0435db4:       e34c00b6        movt    r0, #49334      ; 0xc0b6
> > c0435db8:       ebf54ae9        bl      c0188964 <printk>
> > 
> > inode_up_write()
> 
> The patch you sent helped so far. I'll keep the tablet running for a while,
> but so far the issue did not reappear within a few minutes after boot as
> usual.

So this time it just took several times longer to appear (8-20mins to the hang):

https://megous.com/dl/tmp/dmesg1
https://megous.com/dl/tmp/dmesg2

thank you and regards,
	o.

> thank you and regards,
> 	o.
> 
> > Thanks,
> > 
> > > 
> > > So it's probably not inode locking.
> > > 
> > >> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
> > >> 16
> > >> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
> > >> 16
> > >>
> > >> regards,
> > >> 	o.
> > >>
> > >>> thank you,
> > >>> 	o.
> > >>>
> > >>>>> Thanks,
> > >>>>>
> > >>>>>> [  246.758190]  r5:eff213b0 r4:da283c60
> > >>>>>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> > >>>>>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> > >>>>>> [  246.758206]  r4:eff213b0
> > >>>>>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> > >>>>>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> > >>>>>> [  246.758223]  r4:da645d80
> > >>>>>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> > >>>>>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> > >>>>>> [  246.758246]  r4:da283c60
> > >>>>>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> > >>>>>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> > >>>>>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> > >>>>>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> > >>>>>> [  246.758274]  r4:d9dc9848
> > >>>>>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> > >>>>>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> > >>>>>> [  246.758289]  r4:da5a8800
> > >>>>>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> > >>>>>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> > >>>>>> [  246.758305]  r4:d9dc9848
> > >>>>>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> > >>>>>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> > >>>>>> [  246.758321]  r4:000031e6
> > >>>>>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> > >>>>>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> > >>>>>> [  246.758343]  r4:d9dc9954
> > >>>>>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> > >>>>>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> > >>>>>> [  246.758359]  r4:da5eb000
> > >>>>>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> > >>>>>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> > >>>>>> [  246.758377]  r4:dabf3240
> > >>>>>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> > >>>>>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> > >>>>>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> > >>>>>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> > >>>>>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> > >>>>>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> > >>>>>> [  246.758416]  r4:da5fe000
> > >>>>>> .
> > >>>>>>
> > >>>>>
> > >>>>>
> > >>>>> _______________________________________________
> > >>>>> Linux-f2fs-devel mailing list
> > >>>>> Linux-f2fs-devel@lists.sourceforge.net
> > >>>>> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> > >>>>>
> > > .
> > > 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Chao Yu]

On 2020/2/25 20:27, Ondřej Jirman wrote:
> On Tue, Feb 25, 2020 at 01:08:14PM +0100, megi xff wrote:
>> Hello,
>>
>> On Tue, Feb 25, 2020 at 07:24:12PM +0800, Chao Yu wrote:
>>> On 2020/2/24 22:31, Ondřej Jirman wrote:
>>>>> Just by a looks of it:
>>>>>
>>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_write | wc -l
>>>>> 324
>>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_write | wc -l
>>>>> 347
>>>>>
>>>>> there seems to be a mismatch of lock/unlock counts.
>>>>  
>>>> Sorry, a wrong grep expression.
>>>>
>>>> root@tbs2[~] # dmesg | grep inode_down_write | wc -l
>>>> 357
>>>> root@tbs2[~] # dmesg | grep inode_up_write | wc -l
>>>> 357
>>>> root@tbs2[~] # dmesg | grep inode_up_read | wc -l
>>>> 16
>>>> root@tbs2[~] # dmesg | grep inode_down_read | wc -l
>>>> 16
>>>
>>> I don't know why we have consistent down/up pair, but through disassembled
>>> code, I doubt it's the f2fs_inode->i_sem.
>>
>> Because we were counting attempts, and not a successful lock. ;)

[  214.508943] inode_down_write from f2fs_write_cache_pages+0x2b4/0x7c4
[  306.213325] inode_down_write from f2fs_write_cache_pages+0x2b4/0x7c4

Actually, down_write count is 320, up_write count is 319, kworker didn't call
up_write at 214.508943.

>>
>>> c0435d7c:       ebf54af8        bl      c0188964 <printk>
>>> c0435d80:       e1a00006        mov     r0, r6
>>> c0435d84:       eb138135        bl      c0916260 <down_write>
>>>
>>> inode_down_write()
>>>
>>> c0435d88:       e284ce1d        add     ip, r4, #464    ; 0x1d0
>>>
>>> We are stuck here.
>>>
>>> [  430.675754] [<c0916260>] (down_write) from [<c0435d88>] (f2fs_write_single_data_page+0x600/0x7d8)
>>>                                                 ^^^^^^^^^
>>> [  430.675764] [<c0435788>] (f2fs_write_single_data_page) from [<c0436214>] (f2fs_write_cache_pages+0x2b4/0x7c4)
>>>
>>>
>>> c0435d8c:       e14b0ad4        ldrd    r0, [fp, #-164] ; 0xffffff5c
>>> c0435d90:       e1cc20d0        ldrd    r2, [ip]
>>> c0435d94:       e1520000        cmp     r2, r0
>>> c0435d98:       e0d33001        sbcs    r3, r3, r1
>>> c0435d9c:       b1cc00f0        strdlt  r0, [ip]
>>> c0435da0:       e1a00006        mov     r0, r6
>>> c0435da4:       ebf52227        bl      c017e648 <up_write>
>>> c0435da8:       e51b2098        ldr     r2, [fp, #-152] ; 0xffffff68
>>> c0435dac:       e30c0730        movw    r0, #50992      ; 0xc730
>>> c0435db0:       e59f11a4        ldr     r1, [pc, #420]  ; c0435f5c <f2fs_write_single_data_page+0x7d4>
>>> c0435db4:       e34c00b6        movt    r0, #49334      ; 0xc0b6
>>> c0435db8:       ebf54ae9        bl      c0188964 <printk>
>>>
>>> inode_up_write()
>>
>> The patch you sent helped so far. I'll keep the tablet running for a while,
>> but so far the issue did not reappear within a few minutes after boot as
>> usual.
> 
> So this time it just took several times longer to appear (8-20mins to the hang):
> 
> https://megous.com/dl/tmp/dmesg1
> https://megous.com/dl/tmp/dmesg2

Alright, I still didn't see any possible deadlock in f2fs.

Can you try below patch? I'd like to see whether spinlock can cause the same issue.

From 3e9e8daf922eaa2c5db195ce278e89e10191c516 Mon Sep 17 00:00:00 2001
From: Chao Yu <yuchao0@huawei.com>
Date: Wed, 26 Feb 2020 09:53:03 +0800
Subject: [PATCH] fix

Signed-off-by: Chao Yu <yuchao0@huawei.com>
---
 fs/f2fs/compress.c | 4 ++--
 fs/f2fs/data.c     | 4 ++--
 fs/f2fs/f2fs.h     | 5 +++--
 fs/f2fs/file.c     | 4 ++--
 fs/f2fs/super.c    | 1 +
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
index b4ff25dc55a9..6de0872ad881 100644
--- a/fs/f2fs/compress.c
+++ b/fs/f2fs/compress.c
@@ -906,10 +906,10 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
 	f2fs_put_dnode(&dn);
 	f2fs_unlock_op(sbi);

-	down_write(&fi->i_sem);
+	spin_lock(&fi->i_size_lock);
 	if (fi->last_disk_size < psize)
 		fi->last_disk_size = psize;
-	up_write(&fi->i_sem);
+	spin_unlock(&fi->i_size_lock);

 	f2fs_put_rpages(cc);
 	f2fs_destroy_compress_ctx(cc);
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index cb41260ca941..5c9b072cf0de 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -2651,10 +2651,10 @@ int f2fs_write_single_data_page(struct page *page, int *submitted,
 	if (err) {
 		file_set_keep_isize(inode);
 	} else {
-		down_write(&F2FS_I(inode)->i_sem);
+		spin_lock(&F2FS_I(inode)->i_size_lock);
 		if (F2FS_I(inode)->last_disk_size < psize)
 			F2FS_I(inode)->last_disk_size = psize;
-		up_write(&F2FS_I(inode)->i_sem);
+		spin_unlock(&F2FS_I(inode)->i_size_lock);
 	}

 done:
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 4a02edc2454b..1a8af2020e72 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -701,6 +701,7 @@ struct f2fs_inode_info {
 	struct task_struct *cp_task;	/* separate cp/wb IO stats*/
 	nid_t i_xattr_nid;		/* node id that contains xattrs */
 	loff_t	last_disk_size;		/* lastly written file size */
+	spinlock_t i_size_lock;		/* protect last_disk_size */

 #ifdef CONFIG_QUOTA
 	struct dquot *i_dquot[MAXQUOTAS];
@@ -2882,9 +2883,9 @@ static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
 	if (!f2fs_is_time_consistent(inode))
 		return false;

-	down_read(&F2FS_I(inode)->i_sem);
+	spin_lock(&F2FS_I(inode)->i_size_lock);
 	ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
-	up_read(&F2FS_I(inode)->i_sem);
+	spin_unlock(&F2FS_I(inode)->i_size_lock);

 	return ret;
 }
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index fdb492c2f248..56fe18fbb2ef 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -938,10 +938,10 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
 		if (err)
 			return err;

-		down_write(&F2FS_I(inode)->i_sem);
+		spin_lock(&F2FS_I(inode)->i_size_lock);
 		inode->i_mtime = inode->i_ctime = current_time(inode);
 		F2FS_I(inode)->last_disk_size = i_size_read(inode);
-		up_write(&F2FS_I(inode)->i_sem);
+		spin_unlock(&F2FS_I(inode)->i_size_lock);
 	}

 	__setattr_copy(inode, attr);
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 0b16204d3b7d..2d0e5d1269f5 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -957,6 +957,7 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb)
 	/* Initialize f2fs-specific inode info */
 	atomic_set(&fi->dirty_pages, 0);
 	init_rwsem(&fi->i_sem);
+	spin_lock_init(&fi->i_size_lock);
 	INIT_LIST_HEAD(&fi->dirty_list);
 	INIT_LIST_HEAD(&fi->gdirty_list);
 	INIT_LIST_HEAD(&fi->inmem_ilist);
-- 
2.18.0.rc1




> 
> thank you and regards,
> 	o.
> 
>> thank you and regards,
>> 	o.
>>
>>> Thanks,
>>>
>>>>
>>>> So it's probably not inode locking.
>>>>
>>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
>>>>> 16
>>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
>>>>> 16
>>>>>
>>>>> regards,
>>>>> 	o.
>>>>>
>>>>>> thank you,
>>>>>> 	o.
>>>>>>
>>>>>>>> Thanks,
>>>>>>>>
>>>>>>>>> [  246.758190]  r5:eff213b0 r4:da283c60
>>>>>>>>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
>>>>>>>>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
>>>>>>>>> [  246.758206]  r4:eff213b0
>>>>>>>>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
>>>>>>>>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
>>>>>>>>> [  246.758223]  r4:da645d80
>>>>>>>>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
>>>>>>>>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
>>>>>>>>> [  246.758246]  r4:da283c60
>>>>>>>>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
>>>>>>>>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
>>>>>>>>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
>>>>>>>>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
>>>>>>>>> [  246.758274]  r4:d9dc9848
>>>>>>>>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
>>>>>>>>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
>>>>>>>>> [  246.758289]  r4:da5a8800
>>>>>>>>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
>>>>>>>>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
>>>>>>>>> [  246.758305]  r4:d9dc9848
>>>>>>>>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
>>>>>>>>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
>>>>>>>>> [  246.758321]  r4:000031e6
>>>>>>>>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
>>>>>>>>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
>>>>>>>>> [  246.758343]  r4:d9dc9954
>>>>>>>>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
>>>>>>>>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
>>>>>>>>> [  246.758359]  r4:da5eb000
>>>>>>>>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
>>>>>>>>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
>>>>>>>>> [  246.758377]  r4:dabf3240
>>>>>>>>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
>>>>>>>>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
>>>>>>>>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
>>>>>>>>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
>>>>>>>>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
>>>>>>>>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
>>>>>>>>> [  246.758416]  r4:da5fe000
>>>>>>>>> .
>>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> _______________________________________________
>>>>>>>> Linux-f2fs-devel mailing list
>>>>>>>> Linux-f2fs-devel@lists.sourceforge.net
>>>>>>>> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
>>>>>>>>
>>>> .
>>>>
> .
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

On Wed, Feb 26, 2020 at 09:58:03AM +0800, Chao Yu wrote:
> On 2020/2/25 20:27, Ondřej Jirman wrote:
> > So this time it just took several times longer to appear (8-20mins to the hang):
> > 
> > https://megous.com/dl/tmp/dmesg1
> > https://megous.com/dl/tmp/dmesg2
> 
> Alright, I still didn't see any possible deadlock in f2fs.
> 
> Can you try below patch? I'd like to see whether spinlock can cause the same issue.

Uptime 60 minutes and it didn't hang so far. I applied it on top of the previous
patch:
  
  https://megous.com/git/linux/log/?h=f2fs-debug-5.6

regards,
	o.

> From 3e9e8daf922eaa2c5db195ce278e89e10191c516 Mon Sep 17 00:00:00 2001
> From: Chao Yu <yuchao0@huawei.com>
> Date: Wed, 26 Feb 2020 09:53:03 +0800
> Subject: [PATCH] fix
> 
> Signed-off-by: Chao Yu <yuchao0@huawei.com>
> ---
>  fs/f2fs/compress.c | 4 ++--
>  fs/f2fs/data.c     | 4 ++--
>  fs/f2fs/f2fs.h     | 5 +++--
>  fs/f2fs/file.c     | 4 ++--
>  fs/f2fs/super.c    | 1 +
>  5 files changed, 10 insertions(+), 8 deletions(-)
> 
> diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
> index b4ff25dc55a9..6de0872ad881 100644
> --- a/fs/f2fs/compress.c
> +++ b/fs/f2fs/compress.c
> @@ -906,10 +906,10 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
>  	f2fs_put_dnode(&dn);
>  	f2fs_unlock_op(sbi);
> 
> -	down_write(&fi->i_sem);
> +	spin_lock(&fi->i_size_lock);
>  	if (fi->last_disk_size < psize)
>  		fi->last_disk_size = psize;
> -	up_write(&fi->i_sem);
> +	spin_unlock(&fi->i_size_lock);
> 
>  	f2fs_put_rpages(cc);
>  	f2fs_destroy_compress_ctx(cc);
> diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
> index cb41260ca941..5c9b072cf0de 100644
> --- a/fs/f2fs/data.c
> +++ b/fs/f2fs/data.c
> @@ -2651,10 +2651,10 @@ int f2fs_write_single_data_page(struct page *page, int *submitted,
>  	if (err) {
>  		file_set_keep_isize(inode);
>  	} else {
> -		down_write(&F2FS_I(inode)->i_sem);
> +		spin_lock(&F2FS_I(inode)->i_size_lock);
>  		if (F2FS_I(inode)->last_disk_size < psize)
>  			F2FS_I(inode)->last_disk_size = psize;
> -		up_write(&F2FS_I(inode)->i_sem);
> +		spin_unlock(&F2FS_I(inode)->i_size_lock);
>  	}
> 
>  done:
> diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
> index 4a02edc2454b..1a8af2020e72 100644
> --- a/fs/f2fs/f2fs.h
> +++ b/fs/f2fs/f2fs.h
> @@ -701,6 +701,7 @@ struct f2fs_inode_info {
>  	struct task_struct *cp_task;	/* separate cp/wb IO stats*/
>  	nid_t i_xattr_nid;		/* node id that contains xattrs */
>  	loff_t	last_disk_size;		/* lastly written file size */
> +	spinlock_t i_size_lock;		/* protect last_disk_size */
> 
>  #ifdef CONFIG_QUOTA
>  	struct dquot *i_dquot[MAXQUOTAS];
> @@ -2882,9 +2883,9 @@ static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
>  	if (!f2fs_is_time_consistent(inode))
>  		return false;
> 
> -	down_read(&F2FS_I(inode)->i_sem);
> +	spin_lock(&F2FS_I(inode)->i_size_lock);
>  	ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
> -	up_read(&F2FS_I(inode)->i_sem);
> +	spin_unlock(&F2FS_I(inode)->i_size_lock);
> 
>  	return ret;
>  }
> diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
> index fdb492c2f248..56fe18fbb2ef 100644
> --- a/fs/f2fs/file.c
> +++ b/fs/f2fs/file.c
> @@ -938,10 +938,10 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
>  		if (err)
>  			return err;
> 
> -		down_write(&F2FS_I(inode)->i_sem);
> +		spin_lock(&F2FS_I(inode)->i_size_lock);
>  		inode->i_mtime = inode->i_ctime = current_time(inode);
>  		F2FS_I(inode)->last_disk_size = i_size_read(inode);
> -		up_write(&F2FS_I(inode)->i_sem);
> +		spin_unlock(&F2FS_I(inode)->i_size_lock);
>  	}
> 
>  	__setattr_copy(inode, attr);
> diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
> index 0b16204d3b7d..2d0e5d1269f5 100644
> --- a/fs/f2fs/super.c
> +++ b/fs/f2fs/super.c
> @@ -957,6 +957,7 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb)
>  	/* Initialize f2fs-specific inode info */
>  	atomic_set(&fi->dirty_pages, 0);
>  	init_rwsem(&fi->i_sem);
> +	spin_lock_init(&fi->i_size_lock);
>  	INIT_LIST_HEAD(&fi->dirty_list);
>  	INIT_LIST_HEAD(&fi->gdirty_list);
>  	INIT_LIST_HEAD(&fi->inmem_ilist);
> -- 
> 2.18.0.rc1
> 
> 
> 
> 
> > 
> > thank you and regards,
> > 	o.
> > 
> >> thank you and regards,
> >> 	o.
> >>
> >>> Thanks,
> >>>
> >>>>
> >>>> So it's probably not inode locking.
> >>>>
> >>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
> >>>>> 16
> >>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
> >>>>> 16
> >>>>>
> >>>>> regards,
> >>>>> 	o.
> >>>>>
> >>>>>> thank you,
> >>>>>> 	o.
> >>>>>>
> >>>>>>>> Thanks,
> >>>>>>>>
> >>>>>>>>> [  246.758190]  r5:eff213b0 r4:da283c60
> >>>>>>>>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> >>>>>>>>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> >>>>>>>>> [  246.758206]  r4:eff213b0
> >>>>>>>>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> >>>>>>>>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> >>>>>>>>> [  246.758223]  r4:da645d80
> >>>>>>>>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> >>>>>>>>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> >>>>>>>>> [  246.758246]  r4:da283c60
> >>>>>>>>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> >>>>>>>>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> >>>>>>>>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> >>>>>>>>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> >>>>>>>>> [  246.758274]  r4:d9dc9848
> >>>>>>>>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> >>>>>>>>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> >>>>>>>>> [  246.758289]  r4:da5a8800
> >>>>>>>>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> >>>>>>>>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> >>>>>>>>> [  246.758305]  r4:d9dc9848
> >>>>>>>>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> >>>>>>>>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> >>>>>>>>> [  246.758321]  r4:000031e6
> >>>>>>>>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> >>>>>>>>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> >>>>>>>>> [  246.758343]  r4:d9dc9954
> >>>>>>>>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> >>>>>>>>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> >>>>>>>>> [  246.758359]  r4:da5eb000
> >>>>>>>>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> >>>>>>>>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> >>>>>>>>> [  246.758377]  r4:dabf3240
> >>>>>>>>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> >>>>>>>>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> >>>>>>>>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> >>>>>>>>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> >>>>>>>>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> >>>>>>>>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> >>>>>>>>> [  246.758416]  r4:da5fe000
> >>>>>>>>> .
> >>>>>>>>>
> >>>>>>>>
> >>>>>>>>
> >>>>>>>> _______________________________________________
> >>>>>>>> Linux-f2fs-devel mailing list
> >>>>>>>> Linux-f2fs-devel@lists.sourceforge.net
> >>>>>>>> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> >>>>>>>>
> >>>> .
> >>>>
> > .
> > 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

On Wed, Feb 26, 2020 at 01:11:43PM +0100, megi xff wrote:
> On Wed, Feb 26, 2020 at 09:58:03AM +0800, Chao Yu wrote:
> > On 2020/2/25 20:27, Ondřej Jirman wrote:
> > > So this time it just took several times longer to appear (8-20mins to the hang):
> > > 
> > > https://megous.com/dl/tmp/dmesg1
> > > https://megous.com/dl/tmp/dmesg2
> > 
> > Alright, I still didn't see any possible deadlock in f2fs.
> > 
> > Can you try below patch? I'd like to see whether spinlock can cause the same issue.
> 
> Uptime 60 minutes and it didn't hang so far. I applied it on top of the previous
> patch:
>   
>   https://megous.com/git/linux/log/?h=f2fs-debug-5.6

No issue after 7h uptime either. So I guess this patch solved it for some
reason.

regards,
	o.

> regards,
> 	o.
> 
> > From 3e9e8daf922eaa2c5db195ce278e89e10191c516 Mon Sep 17 00:00:00 2001
> > From: Chao Yu <yuchao0@huawei.com>
> > Date: Wed, 26 Feb 2020 09:53:03 +0800
> > Subject: [PATCH] fix
> > 
> > Signed-off-by: Chao Yu <yuchao0@huawei.com>
> > ---
> >  fs/f2fs/compress.c | 4 ++--
> >  fs/f2fs/data.c     | 4 ++--
> >  fs/f2fs/f2fs.h     | 5 +++--
> >  fs/f2fs/file.c     | 4 ++--
> >  fs/f2fs/super.c    | 1 +
> >  5 files changed, 10 insertions(+), 8 deletions(-)
> > 
> > diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
> > index b4ff25dc55a9..6de0872ad881 100644
> > --- a/fs/f2fs/compress.c
> > +++ b/fs/f2fs/compress.c
> > @@ -906,10 +906,10 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
> >  	f2fs_put_dnode(&dn);
> >  	f2fs_unlock_op(sbi);
> > 
> > -	down_write(&fi->i_sem);
> > +	spin_lock(&fi->i_size_lock);
> >  	if (fi->last_disk_size < psize)
> >  		fi->last_disk_size = psize;
> > -	up_write(&fi->i_sem);
> > +	spin_unlock(&fi->i_size_lock);
> > 
> >  	f2fs_put_rpages(cc);
> >  	f2fs_destroy_compress_ctx(cc);
> > diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
> > index cb41260ca941..5c9b072cf0de 100644
> > --- a/fs/f2fs/data.c
> > +++ b/fs/f2fs/data.c
> > @@ -2651,10 +2651,10 @@ int f2fs_write_single_data_page(struct page *page, int *submitted,
> >  	if (err) {
> >  		file_set_keep_isize(inode);
> >  	} else {
> > -		down_write(&F2FS_I(inode)->i_sem);
> > +		spin_lock(&F2FS_I(inode)->i_size_lock);
> >  		if (F2FS_I(inode)->last_disk_size < psize)
> >  			F2FS_I(inode)->last_disk_size = psize;
> > -		up_write(&F2FS_I(inode)->i_sem);
> > +		spin_unlock(&F2FS_I(inode)->i_size_lock);
> >  	}
> > 
> >  done:
> > diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
> > index 4a02edc2454b..1a8af2020e72 100644
> > --- a/fs/f2fs/f2fs.h
> > +++ b/fs/f2fs/f2fs.h
> > @@ -701,6 +701,7 @@ struct f2fs_inode_info {
> >  	struct task_struct *cp_task;	/* separate cp/wb IO stats*/
> >  	nid_t i_xattr_nid;		/* node id that contains xattrs */
> >  	loff_t	last_disk_size;		/* lastly written file size */
> > +	spinlock_t i_size_lock;		/* protect last_disk_size */
> > 
> >  #ifdef CONFIG_QUOTA
> >  	struct dquot *i_dquot[MAXQUOTAS];
> > @@ -2882,9 +2883,9 @@ static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
> >  	if (!f2fs_is_time_consistent(inode))
> >  		return false;
> > 
> > -	down_read(&F2FS_I(inode)->i_sem);
> > +	spin_lock(&F2FS_I(inode)->i_size_lock);
> >  	ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
> > -	up_read(&F2FS_I(inode)->i_sem);
> > +	spin_unlock(&F2FS_I(inode)->i_size_lock);
> > 
> >  	return ret;
> >  }
> > diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
> > index fdb492c2f248..56fe18fbb2ef 100644
> > --- a/fs/f2fs/file.c
> > +++ b/fs/f2fs/file.c
> > @@ -938,10 +938,10 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
> >  		if (err)
> >  			return err;
> > 
> > -		down_write(&F2FS_I(inode)->i_sem);
> > +		spin_lock(&F2FS_I(inode)->i_size_lock);
> >  		inode->i_mtime = inode->i_ctime = current_time(inode);
> >  		F2FS_I(inode)->last_disk_size = i_size_read(inode);
> > -		up_write(&F2FS_I(inode)->i_sem);
> > +		spin_unlock(&F2FS_I(inode)->i_size_lock);
> >  	}
> > 
> >  	__setattr_copy(inode, attr);
> > diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
> > index 0b16204d3b7d..2d0e5d1269f5 100644
> > --- a/fs/f2fs/super.c
> > +++ b/fs/f2fs/super.c
> > @@ -957,6 +957,7 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb)
> >  	/* Initialize f2fs-specific inode info */
> >  	atomic_set(&fi->dirty_pages, 0);
> >  	init_rwsem(&fi->i_sem);
> > +	spin_lock_init(&fi->i_size_lock);
> >  	INIT_LIST_HEAD(&fi->dirty_list);
> >  	INIT_LIST_HEAD(&fi->gdirty_list);
> >  	INIT_LIST_HEAD(&fi->inmem_ilist);
> > -- 
> > 2.18.0.rc1
> > 
> > 
> > 
> > 
> > > 
> > > thank you and regards,
> > > 	o.
> > > 
> > >> thank you and regards,
> > >> 	o.
> > >>
> > >>> Thanks,
> > >>>
> > >>>>
> > >>>> So it's probably not inode locking.
> > >>>>
> > >>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
> > >>>>> 16
> > >>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
> > >>>>> 16
> > >>>>>
> > >>>>> regards,
> > >>>>> 	o.
> > >>>>>
> > >>>>>> thank you,
> > >>>>>> 	o.
> > >>>>>>
> > >>>>>>>> Thanks,
> > >>>>>>>>
> > >>>>>>>>> [  246.758190]  r5:eff213b0 r4:da283c60
> > >>>>>>>>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> > >>>>>>>>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> > >>>>>>>>> [  246.758206]  r4:eff213b0
> > >>>>>>>>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> > >>>>>>>>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> > >>>>>>>>> [  246.758223]  r4:da645d80
> > >>>>>>>>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> > >>>>>>>>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> > >>>>>>>>> [  246.758246]  r4:da283c60
> > >>>>>>>>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> > >>>>>>>>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> > >>>>>>>>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> > >>>>>>>>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> > >>>>>>>>> [  246.758274]  r4:d9dc9848
> > >>>>>>>>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> > >>>>>>>>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> > >>>>>>>>> [  246.758289]  r4:da5a8800
> > >>>>>>>>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> > >>>>>>>>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> > >>>>>>>>> [  246.758305]  r4:d9dc9848
> > >>>>>>>>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> > >>>>>>>>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> > >>>>>>>>> [  246.758321]  r4:000031e6
> > >>>>>>>>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> > >>>>>>>>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> > >>>>>>>>> [  246.758343]  r4:d9dc9954
> > >>>>>>>>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> > >>>>>>>>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> > >>>>>>>>> [  246.758359]  r4:da5eb000
> > >>>>>>>>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> > >>>>>>>>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> > >>>>>>>>> [  246.758377]  r4:dabf3240
> > >>>>>>>>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> > >>>>>>>>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> > >>>>>>>>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> > >>>>>>>>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> > >>>>>>>>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> > >>>>>>>>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> > >>>>>>>>> [  246.758416]  r4:da5fe000
> > >>>>>>>>> .
> > >>>>>>>>>
> > >>>>>>>>
> > >>>>>>>>
> > >>>>>>>> _______________________________________________
> > >>>>>>>> Linux-f2fs-devel mailing list
> > >>>>>>>> Linux-f2fs-devel@lists.sourceforge.net
> > >>>>>>>> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> > >>>>>>>>
> > >>>> .
> > >>>>
> > > .
> > > 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Chao Yu]

On 2020/2/27 2:05, Ondřej Jirman wrote:
> On Wed, Feb 26, 2020 at 01:11:43PM +0100, megi xff wrote:
>> On Wed, Feb 26, 2020 at 09:58:03AM +0800, Chao Yu wrote:
>>> On 2020/2/25 20:27, Ondřej Jirman wrote:
>>>> So this time it just took several times longer to appear (8-20mins to the hang):
>>>>
>>>> https://megous.com/dl/tmp/dmesg1
>>>> https://megous.com/dl/tmp/dmesg2
>>>
>>> Alright, I still didn't see any possible deadlock in f2fs.
>>>
>>> Can you try below patch? I'd like to see whether spinlock can cause the same issue.
>>
>> Uptime 60 minutes and it didn't hang so far. I applied it on top of the previous
>> patch:
>>   
>>   https://megous.com/git/linux/log/?h=f2fs-debug-5.6

We don't need to move spinlock out of page lock coverage, since the new spinlock's
coverage is limited and it doesn't depend on other locks, so it's safe to call in
original place in where we update last_disk_size.

> 
> No issue after 7h uptime either. So I guess this patch solved it for some
> reason.

I hope so as well, I will send a formal patch for this.

Thanks,

> 
> regards,
> 	o.
> 
>> regards,
>> 	o.
>>
>>> From 3e9e8daf922eaa2c5db195ce278e89e10191c516 Mon Sep 17 00:00:00 2001
>>> From: Chao Yu <yuchao0@huawei.com>
>>> Date: Wed, 26 Feb 2020 09:53:03 +0800
>>> Subject: [PATCH] fix
>>>
>>> Signed-off-by: Chao Yu <yuchao0@huawei.com>
>>> ---
>>>  fs/f2fs/compress.c | 4 ++--
>>>  fs/f2fs/data.c     | 4 ++--
>>>  fs/f2fs/f2fs.h     | 5 +++--
>>>  fs/f2fs/file.c     | 4 ++--
>>>  fs/f2fs/super.c    | 1 +
>>>  5 files changed, 10 insertions(+), 8 deletions(-)
>>>
>>> diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
>>> index b4ff25dc55a9..6de0872ad881 100644
>>> --- a/fs/f2fs/compress.c
>>> +++ b/fs/f2fs/compress.c
>>> @@ -906,10 +906,10 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
>>>  	f2fs_put_dnode(&dn);
>>>  	f2fs_unlock_op(sbi);
>>>
>>> -	down_write(&fi->i_sem);
>>> +	spin_lock(&fi->i_size_lock);
>>>  	if (fi->last_disk_size < psize)
>>>  		fi->last_disk_size = psize;
>>> -	up_write(&fi->i_sem);
>>> +	spin_unlock(&fi->i_size_lock);
>>>
>>>  	f2fs_put_rpages(cc);
>>>  	f2fs_destroy_compress_ctx(cc);
>>> diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
>>> index cb41260ca941..5c9b072cf0de 100644
>>> --- a/fs/f2fs/data.c
>>> +++ b/fs/f2fs/data.c
>>> @@ -2651,10 +2651,10 @@ int f2fs_write_single_data_page(struct page *page, int *submitted,
>>>  	if (err) {
>>>  		file_set_keep_isize(inode);
>>>  	} else {
>>> -		down_write(&F2FS_I(inode)->i_sem);
>>> +		spin_lock(&F2FS_I(inode)->i_size_lock);
>>>  		if (F2FS_I(inode)->last_disk_size < psize)
>>>  			F2FS_I(inode)->last_disk_size = psize;
>>> -		up_write(&F2FS_I(inode)->i_sem);
>>> +		spin_unlock(&F2FS_I(inode)->i_size_lock);
>>>  	}
>>>
>>>  done:
>>> diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
>>> index 4a02edc2454b..1a8af2020e72 100644
>>> --- a/fs/f2fs/f2fs.h
>>> +++ b/fs/f2fs/f2fs.h
>>> @@ -701,6 +701,7 @@ struct f2fs_inode_info {
>>>  	struct task_struct *cp_task;	/* separate cp/wb IO stats*/
>>>  	nid_t i_xattr_nid;		/* node id that contains xattrs */
>>>  	loff_t	last_disk_size;		/* lastly written file size */
>>> +	spinlock_t i_size_lock;		/* protect last_disk_size */
>>>
>>>  #ifdef CONFIG_QUOTA
>>>  	struct dquot *i_dquot[MAXQUOTAS];
>>> @@ -2882,9 +2883,9 @@ static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
>>>  	if (!f2fs_is_time_consistent(inode))
>>>  		return false;
>>>
>>> -	down_read(&F2FS_I(inode)->i_sem);
>>> +	spin_lock(&F2FS_I(inode)->i_size_lock);
>>>  	ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
>>> -	up_read(&F2FS_I(inode)->i_sem);
>>> +	spin_unlock(&F2FS_I(inode)->i_size_lock);
>>>
>>>  	return ret;
>>>  }
>>> diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
>>> index fdb492c2f248..56fe18fbb2ef 100644
>>> --- a/fs/f2fs/file.c
>>> +++ b/fs/f2fs/file.c
>>> @@ -938,10 +938,10 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
>>>  		if (err)
>>>  			return err;
>>>
>>> -		down_write(&F2FS_I(inode)->i_sem);
>>> +		spin_lock(&F2FS_I(inode)->i_size_lock);
>>>  		inode->i_mtime = inode->i_ctime = current_time(inode);
>>>  		F2FS_I(inode)->last_disk_size = i_size_read(inode);
>>> -		up_write(&F2FS_I(inode)->i_sem);
>>> +		spin_unlock(&F2FS_I(inode)->i_size_lock);
>>>  	}
>>>
>>>  	__setattr_copy(inode, attr);
>>> diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
>>> index 0b16204d3b7d..2d0e5d1269f5 100644
>>> --- a/fs/f2fs/super.c
>>> +++ b/fs/f2fs/super.c
>>> @@ -957,6 +957,7 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb)
>>>  	/* Initialize f2fs-specific inode info */
>>>  	atomic_set(&fi->dirty_pages, 0);
>>>  	init_rwsem(&fi->i_sem);
>>> +	spin_lock_init(&fi->i_size_lock);
>>>  	INIT_LIST_HEAD(&fi->dirty_list);
>>>  	INIT_LIST_HEAD(&fi->gdirty_list);
>>>  	INIT_LIST_HEAD(&fi->inmem_ilist);
>>> -- 
>>> 2.18.0.rc1
>>>
>>>
>>>
>>>
>>>>
>>>> thank you and regards,
>>>> 	o.
>>>>
>>>>> thank you and regards,
>>>>> 	o.
>>>>>
>>>>>> Thanks,
>>>>>>
>>>>>>>
>>>>>>> So it's probably not inode locking.
>>>>>>>
>>>>>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep down_read | wc -l
>>>>>>>> 16
>>>>>>>> root@tbs2[/proc/sys/kernel] # dmesg | grep up_read | wc -l
>>>>>>>> 16
>>>>>>>>
>>>>>>>> regards,
>>>>>>>> 	o.
>>>>>>>>
>>>>>>>>> thank you,
>>>>>>>>> 	o.
>>>>>>>>>
>>>>>>>>>>> Thanks,
>>>>>>>>>>>
>>>>>>>>>>>> [  246.758190]  r5:eff213b0 r4:da283c60
>>>>>>>>>>>> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
>>>>>>>>>>>> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
>>>>>>>>>>>> [  246.758206]  r4:eff213b0
>>>>>>>>>>>> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
>>>>>>>>>>>> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
>>>>>>>>>>>> [  246.758223]  r4:da645d80
>>>>>>>>>>>> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
>>>>>>>>>>>> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
>>>>>>>>>>>> [  246.758246]  r4:da283c60
>>>>>>>>>>>> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
>>>>>>>>>>>> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
>>>>>>>>>>>> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
>>>>>>>>>>>> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
>>>>>>>>>>>> [  246.758274]  r4:d9dc9848
>>>>>>>>>>>> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
>>>>>>>>>>>> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
>>>>>>>>>>>> [  246.758289]  r4:da5a8800
>>>>>>>>>>>> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
>>>>>>>>>>>> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
>>>>>>>>>>>> [  246.758305]  r4:d9dc9848
>>>>>>>>>>>> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
>>>>>>>>>>>> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
>>>>>>>>>>>> [  246.758321]  r4:000031e6
>>>>>>>>>>>> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
>>>>>>>>>>>> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
>>>>>>>>>>>> [  246.758343]  r4:d9dc9954
>>>>>>>>>>>> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
>>>>>>>>>>>> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
>>>>>>>>>>>> [  246.758359]  r4:da5eb000
>>>>>>>>>>>> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
>>>>>>>>>>>> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
>>>>>>>>>>>> [  246.758377]  r4:dabf3240
>>>>>>>>>>>> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
>>>>>>>>>>>> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
>>>>>>>>>>>> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
>>>>>>>>>>>> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
>>>>>>>>>>>> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
>>>>>>>>>>>> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
>>>>>>>>>>>> [  246.758416]  r4:da5fe000
>>>>>>>>>>>> .
>>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>> _______________________________________________
>>>>>>>>>>> Linux-f2fs-devel mailing list
>>>>>>>>>>> Linux-f2fs-devel@lists.sourceforge.net
>>>>>>>>>>> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
>>>>>>>>>>>
>>>>>>> .
>>>>>>>
>>>> .
>>>>
> .
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

Hello,

On Thu, Feb 27, 2020 at 10:01:50AM +0800, Chao Yu wrote:
> On 2020/2/27 2:05, Ondřej Jirman wrote:
> > 
> > No issue after 7h uptime either. So I guess this patch solved it for some
> > reason.
> 
> I hope so as well, I will send a formal patch for this.

So I had it happen again, even with the patches. This time in f2fs_rename2:

regards,
	o.

 INFO: task ldconfig:620 blocked for more than 122 seconds.
      Not tainted 5.6.0-rc3-00469-g44d686977effa #48
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
ldconfig        D    0   620    441 0x00000001
Backtrace: 
[<c0913190>] (__schedule) from [<c0913834>] (schedule+0x78/0xf4)
 r10:e587a000 r9:00000000 r8:e587bd80 r7:ee687eb8 r6:00000002 r5:e587a000
 r4:ee8c8000
[<c09137bc>] (schedule) from [<c017ec6c>] (rwsem_down_write_slowpath+0x24c/0x4c0)
 r5:00000001 r4:ee687ea8
[<c017ea20>] (rwsem_down_write_slowpath) from [<c091652c>] (down_write+0x6c/0x70)
 r10:ee687d08 r9:e5131170 r8:ee0f0000 r7:ee687ea8 r6:e587be10 r5:ee685740
 r4:ee687ea8
[<c09164c0>] (down_write) from [<c04149ac>] (f2fs_rename2+0x1a0/0x1114)
 r5:ee685740 r4:ee685740
[<c041480c>] (f2fs_rename2) from [<c02ea5a8>] (vfs_rename+0x434/0x838)
 r10:c041480c r9:e5131170 r8:ee687d08 r7:ee685740 r6:eed21440 r5:00000000
 r4:eecb82a8
[<c02ea174>] (vfs_rename) from [<c02ed764>] (do_renameat2+0x310/0x494)
 r10:eea4c000 r9:00000000 r8:e587bf50 r7:eed21440 r6:00000000 r5:ffffffd9
 r4:eea4d000
[<c02ed454>] (do_renameat2) from [<c02eec20>] (sys_rename+0x34/0x3c)
 r10:00000026 r9:e587a000 r8:c0101204 r7:00000026 r6:00007458 r5:000dac90
 r4:00000003
[<c02eebec>] (sys_rename) from [<c0101000>] (ret_fast_syscall+0x0/0x54)
Exception stack(0xe587bfa8 to 0xe587bff0)
bfa0:                   00000003 000dac90 000dac90 be9e1540 0000c7d5 00000000
bfc0: 00000003 000dac90 00007458 00000026 0000c7d5 00000001 000cd60c be9e1534
bfe0: 00000025 be9e14cc 00014120 0001fd00
NMI backtrace for cpu 3
CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:600b0013 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0901a5c>] (nmi_cpu_backtrace+0x98/0xcc)
 r7:00000000 r6:00000003 r5:00000000 r4:00000003
[<c09019c4>] (nmi_cpu_backtrace) from [<c0901b84>] (nmi_trigger_cpumask_backtrace+0xf4/0x138)
 r5:c0e08498 r4:c010ea5c
[<c0901a90>] (nmi_trigger_cpumask_backtrace) from [<c0110334>] (arch_trigger_cpumask_backtrace+0x20/0x24)
 r7:0008908a r6:c0e08c50 r5:00007f84 r4:ef19b674
[<c0110314>] (arch_trigger_cpumask_backtrace) from [<c01de160>] (watchdog+0x334/0x540)
[<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
 r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
 r4:ef3a64c0
[<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
Exception stack(0xee85dfb0 to 0xee85dff8)
dfa0:                                     00000000 00000000 00000000 00000000
dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
 r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
 r4:ee806040
Sending NMI from CPU 3 to CPUs 0-2,4-7:
NMI backtrace for cpu 2
CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
PC is at arch_cpu_idle+0x48/0x4c
LR is at arch_cpu_idle+0x44/0x4c
pc : [<c0109848>]    lr : [<c0109844>]    psr: 60030013
sp : ef135f68  ip : ef135f78  fp : ef135f74
r10: 00000000  r9 : 00000000  r8 : c0d82830
r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000002  r4 : ef134000
r3 : c011ab00  r2 : ef672d70  r1 : 0004ea3c  r0 : 00000000
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
Control: 10c5387d  Table: 6d87406a  DAC: 00000051
CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
 r7:00000000 r6:00000007 r5:ef135f18 r4:00000002
[<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
[<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
 r5:ef135f18 r4:c0d83110
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef135f18 to 0xef135f60)
5f00:                                                       00000000 0004ea3c
5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
 r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
NMI backtrace for cpu 1
CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
PC is at arch_cpu_idle+0x48/0x4c
LR is at arch_cpu_idle+0x44/0x4c
pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
sp : ef133f68  ip : ef133f78  fp : ef133f74
r10: 00000000  r9 : 00000000  r8 : c0d82830
r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000001  r4 : ef132000
r3 : c011ab00  r2 : ef65ed70  r1 : 0004ed5c  r0 : 00000000
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
Control: 10c5387d  Table: 6de3806a  DAC: 00000051
CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
 r7:00000000 r6:00000007 r5:ef133f18 r4:00000001
[<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
[<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
 r5:ef133f18 r4:c0d83110
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef133f18 to 0xef133f60)
3f00:                                                       00000000 0004ed5c
3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
 r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
NMI backtrace for cpu 0
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
PC is at arch_cpu_idle+0x48/0x4c
LR is at arch_cpu_idle+0x44/0x4c
pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
sp : c0e01f08  ip : c0e01f18  fp : c0e01f14
r10: 00000000  r9 : 00000000  r8 : c0d82830
r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000000  r4 : c0e00000
r3 : c011ab00  r2 : ef64ad70  r1 : 002da988  r0 : 00000000
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
Control: 10c5387d  Table: 6cbc406a  DAC: 00000051
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
 r7:00000000 r6:00000007 r5:c0e01eb8 r4:00000000
[<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
[<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
 r5:c0e01eb8 r4:c0d83110
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xc0e01eb8 to 0xc0e01f00)
1ea0:                                                       00000000 002da988
1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
 r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
 r4:000000d1
[<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
[<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
 r5:c0d47f38 r4:c0ead1c0
[<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
[<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
NMI backtrace for cpu 5
CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
PC is at arch_cpu_idle+0x48/0x4c
LR is at arch_cpu_idle+0x44/0x4c
pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
sp : ef13bf68  ip : ef13bf78  fp : ef13bf74
r10: 00000000  r9 : 00000000  r8 : c0d82830
r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000005  r4 : ef13a000
r3 : c011ab00  r2 : ef6aed70  r1 : 00064604  r0 : 00000000
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
Control: 10c5387d  Table: 6dd8806a  DAC: 00000051
CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
 r7:00000000 r6:00000007 r5:ef13bf18 r4:00000005
[<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
[<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
 r5:ef13bf18 r4:c0d83110
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef13bf18 to 0xef13bf60)
bf00:                                                       00000000 00064604
bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
 r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
NMI backtrace for cpu 4
CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
PC is at arch_cpu_idle+0x48/0x4c
LR is at arch_cpu_idle+0x44/0x4c
pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
sp : ef139f68  ip : ef139f78  fp : ef139f74
r10: 00000000  r9 : 00000000  r8 : c0d82830
r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000004  r4 : ef138000
r3 : c011ab00  r2 : ef69ad70  r1 : 0007c518  r0 : 00000000
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
Control: 10c5387d  Table: 6def806a  DAC: 00000051
CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
 r7:00000000 r6:00000007 r5:ef139f18 r4:00000004
[<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
[<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
 r5:ef139f18 r4:c0d83110
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef139f18 to 0xef139f60)
9f00:                                                       00000000 0007c518
9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
 r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
NMI backtrace for cpu 7
CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
PC is at arch_cpu_idle+0x48/0x4c
LR is at arch_cpu_idle+0x44/0x4c
pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
sp : ef13ff68  ip : ef13ff78  fp : ef13ff74
r10: 00000000  r9 : 00000000  r8 : c0d82830
r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000007  r4 : ef13e000
r3 : c011ab00  r2 : ef6d6d70  r1 : 00028058  r0 : 00000000
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
Control: 10c5387d  Table: 6c5e006a  DAC: 00000051
CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
 r7:00000000 r6:00000007 r5:ef13ff18 r4:00000007
[<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
[<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
 r5:ef13ff18 r4:c0d83110
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef13ff18 to 0xef13ff60)
ff00:                                                       00000000 00028058
ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
 r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
NMI backtrace for cpu 6
CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
PC is at arch_cpu_idle+0x48/0x4c
LR is at arch_cpu_idle+0x44/0x4c
pc : [<c0109848>]    lr : [<c0109844>]    psr: 60010013
sp : ef13df68  ip : ef13df78  fp : ef13df74
r10: 00000000  r9 : 00000000  r8 : c0d82830
r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000006  r4 : ef13c000
r3 : c011ab00  r2 : ef6c2d70  r1 : 00081da8  r0 : 00000000
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
Control: 10c5387d  Table: 6565006a  DAC: 00000051
CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
 r7:00000000 r6:00000007 r5:ef13df18 r4:00000006
[<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
[<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
 r5:ef13df18 r4:c0d83110
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef13df18 to 0xef13df60)
df00:                                                       00000000 00081da8
df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
 r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
Kernel panic - not syncing: hung_task: blocked tasks
CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:0008908a r6:600b0093 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c0131fcc>] (panic+0x108/0x310)
 r7:0008908a r6:c0e08c50 r5:00000000 r4:c0eadbc8
[<c0131ec4>] (panic) from [<c01de16c>] (watchdog+0x340/0x540)
 r3:c0f0dd60 r2:00000001 r1:00000000 r0:c0b49e00
 r7:0008908a
[<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
 r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
 r4:ef3a64c0
[<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
Exception stack(0xee85dfb0 to 0xee85dff8)
dfa0:                                     00000000 00000000 00000000 00000000
dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
 r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
 r4:ee806040
CPU1: stopping
CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
 r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef133f18 to 0xef133f60)
3f00:                                                       00000000 000509b4
3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
 r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
CPU2: stopping
CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
 r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef135f18 to 0xef135f60)
5f00:                                                       00000000 0004f168
5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
 r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
CPU4: stopping
CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
 r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef139f18 to 0xef139f60)
9f00:                                                       00000000 0007c528
9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
 r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
CPU5: stopping
CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
 r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef13bf18 to 0xef13bf60)
bf00:                                                       00000000 00064644
bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
 r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
CPU6: stopping
CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
 r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef13df18 to 0xef13df60)
df00:                                                       00000000 00081fa8
df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
 r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
CPU7: stopping
CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
 r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xef13ff18 to 0xef13ff60)
ff00:                                                       00000000 00028068
ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
 r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
 r4:0000008d
[<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
[<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
 r5:00000051 r4:6f12806a
CPU0: stopping
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
Hardware name: Allwinner A83t board
Backtrace: 
[<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
 r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
[<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
[<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
 r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
[<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
 r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
 r4:c0e0565c r3:c0109844
[<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
Exception stack(0xc0e01eb8 to 0xc0e01f00)
1ea0:                                                       00000000 002dadd8
1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
 r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
[<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
[<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
[<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
 r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
 r4:000000d1
[<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
[<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
 r5:c0d47f38 r4:c0ead1c0
[<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
[<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
Rebooting in 3 seconds..
DRAM: 1024 MiB
Trying to boot from MMC1

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

On Fri, Mar 06, 2020 at 01:02:03PM +0100, megi xff wrote:
> Hello,
> 
> On Thu, Feb 27, 2020 at 10:01:50AM +0800, Chao Yu wrote:
> > On 2020/2/27 2:05, Ondřej Jirman wrote:
> > > 
> > > No issue after 7h uptime either. So I guess this patch solved it for some
> > > reason.
> > 
> > I hope so as well, I will send a formal patch for this.
> 
> So I had it happen again, even with the patches. This time in f2fs_rename2:

Also this time it was not harmless (it happened during package installation via
pacman), and introduced some corruption (I think installed files were just
incomplete, I was able to reinstall all affected packages, and that was enough
to clean this up):

ldconfig: /usr/lib/libuchardet.so.0 is not an ELF file - it has the wrong magic bytes at the start.

ldconfig: /usr/lib/libXt.so.6 is not an ELF file - it has the wrong magic bytes at the start.

ldconfig: /usr/lib/libXmu.so.6.2.0 is not an ELF file - it has the wrong magic bytes at the start.

ldconfig: /usr/lib/libmpv.so.1.107.0 is not an ELF file - it has the wrong magic bytes at the start.

ldconfig: /usr/lib/libcaca.so is not an ELF file - it has the wrong magic bytes at the start.

ldconfig: /usr/lib/libcdio_paranoia.so.2 is not an ELF file - it has the wrong magic bytes at the start.

ldconfig: /usr/lib/libcdio_paranoia.so.2.0.0 is not an ELF file - it has the wrong magic bytes at the start.

ldconfig: /usr/lib/libdvdnav.so.4.2.0 is not an ELF file - it has the wrong magic bytes at the start.

Previously it was probably not harmless either, because I remember having
zsh report .zhistory corruption. But I was using mostly reads (find /) to
stress the fs and reproduce the issue previously.

regards,
	o.

> regards,
> 	o.
> 
>  INFO: task ldconfig:620 blocked for more than 122 seconds.
>       Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> ldconfig        D    0   620    441 0x00000001
> Backtrace: 
> [<c0913190>] (__schedule) from [<c0913834>] (schedule+0x78/0xf4)
>  r10:e587a000 r9:00000000 r8:e587bd80 r7:ee687eb8 r6:00000002 r5:e587a000
>  r4:ee8c8000
> [<c09137bc>] (schedule) from [<c017ec6c>] (rwsem_down_write_slowpath+0x24c/0x4c0)
>  r5:00000001 r4:ee687ea8
> [<c017ea20>] (rwsem_down_write_slowpath) from [<c091652c>] (down_write+0x6c/0x70)
>  r10:ee687d08 r9:e5131170 r8:ee0f0000 r7:ee687ea8 r6:e587be10 r5:ee685740
>  r4:ee687ea8
> [<c09164c0>] (down_write) from [<c04149ac>] (f2fs_rename2+0x1a0/0x1114)
>  r5:ee685740 r4:ee685740
> [<c041480c>] (f2fs_rename2) from [<c02ea5a8>] (vfs_rename+0x434/0x838)
>  r10:c041480c r9:e5131170 r8:ee687d08 r7:ee685740 r6:eed21440 r5:00000000
>  r4:eecb82a8
> [<c02ea174>] (vfs_rename) from [<c02ed764>] (do_renameat2+0x310/0x494)
>  r10:eea4c000 r9:00000000 r8:e587bf50 r7:eed21440 r6:00000000 r5:ffffffd9
>  r4:eea4d000
> [<c02ed454>] (do_renameat2) from [<c02eec20>] (sys_rename+0x34/0x3c)
>  r10:00000026 r9:e587a000 r8:c0101204 r7:00000026 r6:00007458 r5:000dac90
>  r4:00000003
> [<c02eebec>] (sys_rename) from [<c0101000>] (ret_fast_syscall+0x0/0x54)
> Exception stack(0xe587bfa8 to 0xe587bff0)
> bfa0:                   00000003 000dac90 000dac90 be9e1540 0000c7d5 00000000
> bfc0: 00000003 000dac90 00007458 00000026 0000c7d5 00000001 000cd60c be9e1534
> bfe0: 00000025 be9e14cc 00014120 0001fd00
> NMI backtrace for cpu 3
> CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:600b0013 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0901a5c>] (nmi_cpu_backtrace+0x98/0xcc)
>  r7:00000000 r6:00000003 r5:00000000 r4:00000003
> [<c09019c4>] (nmi_cpu_backtrace) from [<c0901b84>] (nmi_trigger_cpumask_backtrace+0xf4/0x138)
>  r5:c0e08498 r4:c010ea5c
> [<c0901a90>] (nmi_trigger_cpumask_backtrace) from [<c0110334>] (arch_trigger_cpumask_backtrace+0x20/0x24)
>  r7:0008908a r6:c0e08c50 r5:00007f84 r4:ef19b674
> [<c0110314>] (arch_trigger_cpumask_backtrace) from [<c01de160>] (watchdog+0x334/0x540)
> [<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
>  r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
>  r4:ef3a64c0
> [<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> Exception stack(0xee85dfb0 to 0xee85dff8)
> dfa0:                                     00000000 00000000 00000000 00000000
> dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
>  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
>  r4:ee806040
> Sending NMI from CPU 3 to CPUs 0-2,4-7:
> NMI backtrace for cpu 2
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60030013
> sp : ef135f68  ip : ef135f78  fp : ef135f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000002  r4 : ef134000
> r3 : c011ab00  r2 : ef672d70  r1 : 0004ea3c  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6d87406a  DAC: 00000051
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef135f18 r4:00000002
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef135f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef135f18 to 0xef135f60)
> 5f00:                                                       00000000 0004ea3c
> 5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
> 5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
>  r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 1
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef133f68  ip : ef133f78  fp : ef133f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000001  r4 : ef132000
> r3 : c011ab00  r2 : ef65ed70  r1 : 0004ed5c  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6de3806a  DAC: 00000051
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef133f18 r4:00000001
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef133f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef133f18 to 0xef133f60)
> 3f00:                                                       00000000 0004ed5c
> 3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
> 3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 0
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : c0e01f08  ip : c0e01f18  fp : c0e01f14
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000000  r4 : c0e00000
> r3 : c011ab00  r2 : ef64ad70  r1 : 002da988  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6cbc406a  DAC: 00000051
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:c0e01eb8 r4:00000000
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:c0e01eb8 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xc0e01eb8 to 0xc0e01f00)
> 1ea0:                                                       00000000 002da988
> 1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
> 1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
>  r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
>  r4:000000d1
> [<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
> [<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
>  r5:c0d47f38 r4:c0ead1c0
> [<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
> [<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
> NMI backtrace for cpu 5
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef13bf68  ip : ef13bf78  fp : ef13bf74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000005  r4 : ef13a000
> r3 : c011ab00  r2 : ef6aed70  r1 : 00064604  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6dd8806a  DAC: 00000051
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13bf18 r4:00000005
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13bf18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13bf18 to 0xef13bf60)
> bf00:                                                       00000000 00064604
> bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
> bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 4
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef139f68  ip : ef139f78  fp : ef139f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000004  r4 : ef138000
> r3 : c011ab00  r2 : ef69ad70  r1 : 0007c518  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6def806a  DAC: 00000051
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef139f18 r4:00000004
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef139f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef139f18 to 0xef139f60)
> 9f00:                                                       00000000 0007c518
> 9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
> 9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 7
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef13ff68  ip : ef13ff78  fp : ef13ff74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000007  r4 : ef13e000
> r3 : c011ab00  r2 : ef6d6d70  r1 : 00028058  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6c5e006a  DAC: 00000051
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13ff18 r4:00000007
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13ff18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13ff18 to 0xef13ff60)
> ff00:                                                       00000000 00028058
> ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
> ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 6
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60010013
> sp : ef13df68  ip : ef13df78  fp : ef13df74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000006  r4 : ef13c000
> r3 : c011ab00  r2 : ef6c2d70  r1 : 00081da8  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6565006a  DAC: 00000051
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13df18 r4:00000006
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13df18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13df18 to 0xef13df60)
> df00:                                                       00000000 00081da8
> df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
> df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
>  r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> Kernel panic - not syncing: hung_task: blocked tasks
> CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:0008908a r6:600b0093 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0131fcc>] (panic+0x108/0x310)
>  r7:0008908a r6:c0e08c50 r5:00000000 r4:c0eadbc8
> [<c0131ec4>] (panic) from [<c01de16c>] (watchdog+0x340/0x540)
>  r3:c0f0dd60 r2:00000001 r1:00000000 r0:c0b49e00
>  r7:0008908a
> [<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
>  r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
>  r4:ef3a64c0
> [<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> Exception stack(0xee85dfb0 to 0xee85dff8)
> dfa0:                                     00000000 00000000 00000000 00000000
> dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
>  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
>  r4:ee806040
> CPU1: stopping
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef133f18 to 0xef133f60)
> 3f00:                                                       00000000 000509b4
> 3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
> 3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU2: stopping
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef135f18 to 0xef135f60)
> 5f00:                                                       00000000 0004f168
> 5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
> 5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
>  r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU4: stopping
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef139f18 to 0xef139f60)
> 9f00:                                                       00000000 0007c528
> 9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
> 9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU5: stopping
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13bf18 to 0xef13bf60)
> bf00:                                                       00000000 00064644
> bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
> bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU6: stopping
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13df18 to 0xef13df60)
> df00:                                                       00000000 00081fa8
> df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
> df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
>  r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU7: stopping
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13ff18 to 0xef13ff60)
> ff00:                                                       00000000 00028068
> ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
> ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU0: stopping
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xc0e01eb8 to 0xc0e01f00)
> 1ea0:                                                       00000000 002dadd8
> 1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
> 1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
>  r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
>  r4:000000d1
> [<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
> [<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
>  r5:c0d47f38 r4:c0ead1c0
> [<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
> [<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
> Rebooting in 3 seconds..
> DRAM: 1024 MiB
> Trying to boot from MMC1
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Chao Yu]

Hi,

Sorry for the delay.

On 2020/3/6 20:02, Ondřej Jirman wrote:
> Hello,
> 
> On Thu, Feb 27, 2020 at 10:01:50AM +0800, Chao Yu wrote:
>> On 2020/2/27 2:05, Ondřej Jirman wrote:
>>>
>>> No issue after 7h uptime either. So I guess this patch solved it for some
>>> reason.
>>
>> I hope so as well, I will send a formal patch for this.
> 
> So I had it happen again, even with the patches. This time in f2fs_rename2:

Oops, it looks previous fix patch just cover the root cause... :(

Did this issue still happen frequently? If it is, would you please apply patch
that prints log during down/up semaphore.

And once we revert compression support patch, this issue will disappear, right?

Btw, did you try other hardware which is not Arm v7?

Thanks,

> 
> regards,
> 	o.
> 
>  INFO: task ldconfig:620 blocked for more than 122 seconds.
>       Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> ldconfig        D    0   620    441 0x00000001
> Backtrace: 
> [<c0913190>] (__schedule) from [<c0913834>] (schedule+0x78/0xf4)
>  r10:e587a000 r9:00000000 r8:e587bd80 r7:ee687eb8 r6:00000002 r5:e587a000
>  r4:ee8c8000
> [<c09137bc>] (schedule) from [<c017ec6c>] (rwsem_down_write_slowpath+0x24c/0x4c0)
>  r5:00000001 r4:ee687ea8
> [<c017ea20>] (rwsem_down_write_slowpath) from [<c091652c>] (down_write+0x6c/0x70)
>  r10:ee687d08 r9:e5131170 r8:ee0f0000 r7:ee687ea8 r6:e587be10 r5:ee685740
>  r4:ee687ea8
> [<c09164c0>] (down_write) from [<c04149ac>] (f2fs_rename2+0x1a0/0x1114)
>  r5:ee685740 r4:ee685740
> [<c041480c>] (f2fs_rename2) from [<c02ea5a8>] (vfs_rename+0x434/0x838)
>  r10:c041480c r9:e5131170 r8:ee687d08 r7:ee685740 r6:eed21440 r5:00000000
>  r4:eecb82a8
> [<c02ea174>] (vfs_rename) from [<c02ed764>] (do_renameat2+0x310/0x494)
>  r10:eea4c000 r9:00000000 r8:e587bf50 r7:eed21440 r6:00000000 r5:ffffffd9
>  r4:eea4d000
> [<c02ed454>] (do_renameat2) from [<c02eec20>] (sys_rename+0x34/0x3c)
>  r10:00000026 r9:e587a000 r8:c0101204 r7:00000026 r6:00007458 r5:000dac90
>  r4:00000003
> [<c02eebec>] (sys_rename) from [<c0101000>] (ret_fast_syscall+0x0/0x54)
> Exception stack(0xe587bfa8 to 0xe587bff0)
> bfa0:                   00000003 000dac90 000dac90 be9e1540 0000c7d5 00000000
> bfc0: 00000003 000dac90 00007458 00000026 0000c7d5 00000001 000cd60c be9e1534
> bfe0: 00000025 be9e14cc 00014120 0001fd00
> NMI backtrace for cpu 3
> CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:600b0013 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0901a5c>] (nmi_cpu_backtrace+0x98/0xcc)
>  r7:00000000 r6:00000003 r5:00000000 r4:00000003
> [<c09019c4>] (nmi_cpu_backtrace) from [<c0901b84>] (nmi_trigger_cpumask_backtrace+0xf4/0x138)
>  r5:c0e08498 r4:c010ea5c
> [<c0901a90>] (nmi_trigger_cpumask_backtrace) from [<c0110334>] (arch_trigger_cpumask_backtrace+0x20/0x24)
>  r7:0008908a r6:c0e08c50 r5:00007f84 r4:ef19b674
> [<c0110314>] (arch_trigger_cpumask_backtrace) from [<c01de160>] (watchdog+0x334/0x540)
> [<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
>  r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
>  r4:ef3a64c0
> [<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> Exception stack(0xee85dfb0 to 0xee85dff8)
> dfa0:                                     00000000 00000000 00000000 00000000
> dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
>  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
>  r4:ee806040
> Sending NMI from CPU 3 to CPUs 0-2,4-7:
> NMI backtrace for cpu 2
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60030013
> sp : ef135f68  ip : ef135f78  fp : ef135f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000002  r4 : ef134000
> r3 : c011ab00  r2 : ef672d70  r1 : 0004ea3c  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6d87406a  DAC: 00000051
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef135f18 r4:00000002
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef135f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef135f18 to 0xef135f60)
> 5f00:                                                       00000000 0004ea3c
> 5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
> 5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
>  r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 1
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef133f68  ip : ef133f78  fp : ef133f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000001  r4 : ef132000
> r3 : c011ab00  r2 : ef65ed70  r1 : 0004ed5c  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6de3806a  DAC: 00000051
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef133f18 r4:00000001
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef133f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef133f18 to 0xef133f60)
> 3f00:                                                       00000000 0004ed5c
> 3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
> 3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 0
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : c0e01f08  ip : c0e01f18  fp : c0e01f14
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000000  r4 : c0e00000
> r3 : c011ab00  r2 : ef64ad70  r1 : 002da988  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6cbc406a  DAC: 00000051
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:c0e01eb8 r4:00000000
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:c0e01eb8 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xc0e01eb8 to 0xc0e01f00)
> 1ea0:                                                       00000000 002da988
> 1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
> 1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
>  r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
>  r4:000000d1
> [<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
> [<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
>  r5:c0d47f38 r4:c0ead1c0
> [<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
> [<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
> NMI backtrace for cpu 5
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef13bf68  ip : ef13bf78  fp : ef13bf74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000005  r4 : ef13a000
> r3 : c011ab00  r2 : ef6aed70  r1 : 00064604  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6dd8806a  DAC: 00000051
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13bf18 r4:00000005
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13bf18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13bf18 to 0xef13bf60)
> bf00:                                                       00000000 00064604
> bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
> bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 4
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef139f68  ip : ef139f78  fp : ef139f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000004  r4 : ef138000
> r3 : c011ab00  r2 : ef69ad70  r1 : 0007c518  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6def806a  DAC: 00000051
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef139f18 r4:00000004
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef139f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef139f18 to 0xef139f60)
> 9f00:                                                       00000000 0007c518
> 9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
> 9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 7
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef13ff68  ip : ef13ff78  fp : ef13ff74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000007  r4 : ef13e000
> r3 : c011ab00  r2 : ef6d6d70  r1 : 00028058  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6c5e006a  DAC: 00000051
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13ff18 r4:00000007
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13ff18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13ff18 to 0xef13ff60)
> ff00:                                                       00000000 00028058
> ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
> ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 6
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60010013
> sp : ef13df68  ip : ef13df78  fp : ef13df74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000006  r4 : ef13c000
> r3 : c011ab00  r2 : ef6c2d70  r1 : 00081da8  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6565006a  DAC: 00000051
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13df18 r4:00000006
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13df18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13df18 to 0xef13df60)
> df00:                                                       00000000 00081da8
> df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
> df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
>  r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> Kernel panic - not syncing: hung_task: blocked tasks
> CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:0008908a r6:600b0093 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0131fcc>] (panic+0x108/0x310)
>  r7:0008908a r6:c0e08c50 r5:00000000 r4:c0eadbc8
> [<c0131ec4>] (panic) from [<c01de16c>] (watchdog+0x340/0x540)
>  r3:c0f0dd60 r2:00000001 r1:00000000 r0:c0b49e00
>  r7:0008908a
> [<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
>  r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
>  r4:ef3a64c0
> [<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> Exception stack(0xee85dfb0 to 0xee85dff8)
> dfa0:                                     00000000 00000000 00000000 00000000
> dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
>  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
>  r4:ee806040
> CPU1: stopping
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef133f18 to 0xef133f60)
> 3f00:                                                       00000000 000509b4
> 3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
> 3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU2: stopping
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef135f18 to 0xef135f60)
> 5f00:                                                       00000000 0004f168
> 5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
> 5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
>  r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU4: stopping
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef139f18 to 0xef139f60)
> 9f00:                                                       00000000 0007c528
> 9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
> 9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU5: stopping
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13bf18 to 0xef13bf60)
> bf00:                                                       00000000 00064644
> bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
> bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU6: stopping
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13df18 to 0xef13df60)
> df00:                                                       00000000 00081fa8
> df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
> df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
>  r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU7: stopping
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13ff18 to 0xef13ff60)
> ff00:                                                       00000000 00028068
> ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
> ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU0: stopping
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xc0e01eb8 to 0xc0e01f00)
> 1ea0:                                                       00000000 002dadd8
> 1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
> 1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
>  r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
>  r4:000000d1
> [<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
> [<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
>  r5:c0d47f38 r4:c0ead1c0
> [<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
> [<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
> Rebooting in 3 seconds..
> DRAM: 1024 MiB
> Trying to boot from MMC1
> 
> .
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

Hello,

On Wed, Mar 11, 2020 at 05:02:10PM +0800, Chao Yu wrote:
> Hi,
> 
> Sorry for the delay.
> 
> On 2020/3/6 20:02, Ondřej Jirman wrote:
> > Hello,
> > 
> > On Thu, Feb 27, 2020 at 10:01:50AM +0800, Chao Yu wrote:
> >> On 2020/2/27 2:05, Ondřej Jirman wrote:
> >>>
> >>> No issue after 7h uptime either. So I guess this patch solved it for some
> >>> reason.
> >>
> >> I hope so as well, I will send a formal patch for this.
> > 
> > So I had it happen again, even with the patches. This time in f2fs_rename2:
> 
> Oops, it looks previous fix patch just cover the root cause... :(
> 
> Did this issue still happen frequently? If it is, would you please apply patch
> that prints log during down/up semaphore.

Not frequently. Just once. I couldn't afford FS corruption during update,
so I reverted the compression support shortly after.

But I wasn't stressing the system much with FS activity after applying the
initial fix.

> And once we revert compression support patch, this issue will disappear, right?

Yes, AFAIK. I reverted it and run a few cycles of install 500MiB worth of
packages, uninstall the packages with pacman. And it didn't re-occur. I never
saw any issues with compression support patch reverted.

> Btw, did you try other hardware which is not Arm v7?

Yes, but I didn't ever see it on anything else. Just on two 8 core cortexes A7.
(2 clusters)

regards,
	o.

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Chao Yu]

Hi,

On 2020/3/11 18:33, Ondřej Jirman wrote:
> Hello,
> 
> On Wed, Mar 11, 2020 at 05:02:10PM +0800, Chao Yu wrote:
>> Hi,
>>
>> Sorry for the delay.
>>
>> On 2020/3/6 20:02, Ondřej Jirman wrote:
>>> Hello,
>>>
>>> On Thu, Feb 27, 2020 at 10:01:50AM +0800, Chao Yu wrote:
>>>> On 2020/2/27 2:05, Ondřej Jirman wrote:
>>>>>
>>>>> No issue after 7h uptime either. So I guess this patch solved it for some
>>>>> reason.
>>>>
>>>> I hope so as well, I will send a formal patch for this.
>>>
>>> So I had it happen again, even with the patches. This time in f2fs_rename2:
>>
>> Oops, it looks previous fix patch just cover the root cause... :(
>>
>> Did this issue still happen frequently? If it is, would you please apply patch
>> that prints log during down/up semaphore.
> 
> Not frequently. Just once. I couldn't afford FS corruption during update,

Alright.

> so I reverted the compression support shortly after.

What I can see is that filesystem was just stuck, rather than image became
corrupted, I guess the condition is not such bad, anyway, it's okay to just
revert compression support for now to keep fs stable.

> 
> But I wasn't stressing the system much with FS activity after applying the
> initial fix.
> 
>> And once we revert compression support patch, this issue will disappear, right?
> 
> Yes, AFAIK. I reverted it and run a few cycles of install 500MiB worth of
> packages, uninstall the packages with pacman. And it didn't re-occur. I never
> saw any issues with compression support patch reverted.

Okay, compression support may increase stack usage during page writeback, it
shouldn't overflow the stack, otherwise it could cause panic in somewhere else,
but still can find any clue why this is related to compression support patch...

> 
>> Btw, did you try other hardware which is not Arm v7?
> 
> Yes, but I didn't ever see it on anything else. Just on two 8 core cortexes A7.
> (2 clusters)

Not sure, maybe this issue is related to arm v7 architecture.

Thanks,

> 
> regards,
> 	o.
> .
> 

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

On Wed, Mar 11, 2020 at 06:51:23PM +0800, Chao Yu wrote:
> Hi,
> 
> >> Oops, it looks previous fix patch just cover the root cause... :(
> >>
> >> Did this issue still happen frequently? If it is, would you please apply patch
> >> that prints log during down/up semaphore.
> > 
> > Not frequently. Just once. I couldn't afford FS corruption during update,
> 
> Alright.
> 
> > so I reverted the compression support shortly after.
> 
> What I can see is that filesystem was just stuck, rather than image became
> corrupted, I guess the condition is not such bad, anyway, it's okay to just
> revert compression support for now to keep fs stable.

Yes, to be precise, file writes were not completed and fs was stuck.
The system as a whole would probably become unbootable if this would
happen to core libraries necessary for systemd, or something like that,
but filesystem itself was not corrupted.

Re-writing the files was enough to recover the system.

I guess we'll see if there will be more reports after 5.6 is released,
or if it's just some quirk in my system.

I'll try to collect more information, once I'll have some time, to
get to the bottom of this.

thank you and regards,
	o.

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Jaegeuk Kim]

On 03/06, Ondřej Jirman wrote:
> Hello,
> 
> On Thu, Feb 27, 2020 at 10:01:50AM +0800, Chao Yu wrote:
> > On 2020/2/27 2:05, Ondřej Jirman wrote:
> > > 
> > > No issue after 7h uptime either. So I guess this patch solved it for some
> > > reason.
> > 
> > I hope so as well, I will send a formal patch for this.
> 
> So I had it happen again, even with the patches. This time in f2fs_rename2:

Hmm, I haven't seen this so far. Is it doable to see all the other tasks
together? And, may I offer to test with some bug fixes together that I sent
pull-request?

https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git/log/?h=dev

Thanks,

> 
> regards,
> 	o.
> 
>  INFO: task ldconfig:620 blocked for more than 122 seconds.
>       Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> ldconfig        D    0   620    441 0x00000001
> Backtrace: 
> [<c0913190>] (__schedule) from [<c0913834>] (schedule+0x78/0xf4)
>  r10:e587a000 r9:00000000 r8:e587bd80 r7:ee687eb8 r6:00000002 r5:e587a000
>  r4:ee8c8000
> [<c09137bc>] (schedule) from [<c017ec6c>] (rwsem_down_write_slowpath+0x24c/0x4c0)
>  r5:00000001 r4:ee687ea8
> [<c017ea20>] (rwsem_down_write_slowpath) from [<c091652c>] (down_write+0x6c/0x70)
>  r10:ee687d08 r9:e5131170 r8:ee0f0000 r7:ee687ea8 r6:e587be10 r5:ee685740
>  r4:ee687ea8
> [<c09164c0>] (down_write) from [<c04149ac>] (f2fs_rename2+0x1a0/0x1114)
>  r5:ee685740 r4:ee685740
> [<c041480c>] (f2fs_rename2) from [<c02ea5a8>] (vfs_rename+0x434/0x838)
>  r10:c041480c r9:e5131170 r8:ee687d08 r7:ee685740 r6:eed21440 r5:00000000
>  r4:eecb82a8
> [<c02ea174>] (vfs_rename) from [<c02ed764>] (do_renameat2+0x310/0x494)
>  r10:eea4c000 r9:00000000 r8:e587bf50 r7:eed21440 r6:00000000 r5:ffffffd9
>  r4:eea4d000
> [<c02ed454>] (do_renameat2) from [<c02eec20>] (sys_rename+0x34/0x3c)
>  r10:00000026 r9:e587a000 r8:c0101204 r7:00000026 r6:00007458 r5:000dac90
>  r4:00000003
> [<c02eebec>] (sys_rename) from [<c0101000>] (ret_fast_syscall+0x0/0x54)
> Exception stack(0xe587bfa8 to 0xe587bff0)
> bfa0:                   00000003 000dac90 000dac90 be9e1540 0000c7d5 00000000
> bfc0: 00000003 000dac90 00007458 00000026 0000c7d5 00000001 000cd60c be9e1534
> bfe0: 00000025 be9e14cc 00014120 0001fd00
> NMI backtrace for cpu 3
> CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:600b0013 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0901a5c>] (nmi_cpu_backtrace+0x98/0xcc)
>  r7:00000000 r6:00000003 r5:00000000 r4:00000003
> [<c09019c4>] (nmi_cpu_backtrace) from [<c0901b84>] (nmi_trigger_cpumask_backtrace+0xf4/0x138)
>  r5:c0e08498 r4:c010ea5c
> [<c0901a90>] (nmi_trigger_cpumask_backtrace) from [<c0110334>] (arch_trigger_cpumask_backtrace+0x20/0x24)
>  r7:0008908a r6:c0e08c50 r5:00007f84 r4:ef19b674
> [<c0110314>] (arch_trigger_cpumask_backtrace) from [<c01de160>] (watchdog+0x334/0x540)
> [<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
>  r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
>  r4:ef3a64c0
> [<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> Exception stack(0xee85dfb0 to 0xee85dff8)
> dfa0:                                     00000000 00000000 00000000 00000000
> dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
>  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
>  r4:ee806040
> Sending NMI from CPU 3 to CPUs 0-2,4-7:
> NMI backtrace for cpu 2
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60030013
> sp : ef135f68  ip : ef135f78  fp : ef135f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000002  r4 : ef134000
> r3 : c011ab00  r2 : ef672d70  r1 : 0004ea3c  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6d87406a  DAC: 00000051
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef135f18 r4:00000002
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef135f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef135f18 to 0xef135f60)
> 5f00:                                                       00000000 0004ea3c
> 5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
> 5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
>  r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 1
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef133f68  ip : ef133f78  fp : ef133f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000001  r4 : ef132000
> r3 : c011ab00  r2 : ef65ed70  r1 : 0004ed5c  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6de3806a  DAC: 00000051
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef133f18 r4:00000001
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef133f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef133f18 to 0xef133f60)
> 3f00:                                                       00000000 0004ed5c
> 3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
> 3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 0
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : c0e01f08  ip : c0e01f18  fp : c0e01f14
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000000  r4 : c0e00000
> r3 : c011ab00  r2 : ef64ad70  r1 : 002da988  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6cbc406a  DAC: 00000051
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:c0e01eb8 r4:00000000
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:c0e01eb8 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xc0e01eb8 to 0xc0e01f00)
> 1ea0:                                                       00000000 002da988
> 1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
> 1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
>  r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
>  r4:000000d1
> [<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
> [<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
>  r5:c0d47f38 r4:c0ead1c0
> [<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
> [<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
> NMI backtrace for cpu 5
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef13bf68  ip : ef13bf78  fp : ef13bf74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000005  r4 : ef13a000
> r3 : c011ab00  r2 : ef6aed70  r1 : 00064604  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6dd8806a  DAC: 00000051
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13bf18 r4:00000005
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13bf18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13bf18 to 0xef13bf60)
> bf00:                                                       00000000 00064604
> bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
> bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 4
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef139f68  ip : ef139f78  fp : ef139f74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000004  r4 : ef138000
> r3 : c011ab00  r2 : ef69ad70  r1 : 0007c518  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6def806a  DAC: 00000051
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef139f18 r4:00000004
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef139f18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef139f18 to 0xef139f60)
> 9f00:                                                       00000000 0007c518
> 9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
> 9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 7
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
> sp : ef13ff68  ip : ef13ff78  fp : ef13ff74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000007  r4 : ef13e000
> r3 : c011ab00  r2 : ef6d6d70  r1 : 00028058  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6c5e006a  DAC: 00000051
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13ff18 r4:00000007
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13ff18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13ff18 to 0xef13ff60)
> ff00:                                                       00000000 00028058
> ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
> ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> NMI backtrace for cpu 6
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> PC is at arch_cpu_idle+0x48/0x4c
> LR is at arch_cpu_idle+0x44/0x4c
> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60010013
> sp : ef13df68  ip : ef13df78  fp : ef13df74
> r10: 00000000  r9 : 00000000  r8 : c0d82830
> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000006  r4 : ef13c000
> r3 : c011ab00  r2 : ef6c2d70  r1 : 00081da8  r0 : 00000000
> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
> Control: 10c5387d  Table: 6565006a  DAC: 00000051
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>  r7:00000000 r6:00000007 r5:ef13df18 r4:00000006
> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>  r5:ef13df18 r4:c0d83110
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13df18 to 0xef13df60)
> df00:                                                       00000000 00081da8
> df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
> df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
>  r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> Kernel panic - not syncing: hung_task: blocked tasks
> CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:0008908a r6:600b0093 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c0131fcc>] (panic+0x108/0x310)
>  r7:0008908a r6:c0e08c50 r5:00000000 r4:c0eadbc8
> [<c0131ec4>] (panic) from [<c01de16c>] (watchdog+0x340/0x540)
>  r3:c0f0dd60 r2:00000001 r1:00000000 r0:c0b49e00
>  r7:0008908a
> [<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
>  r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
>  r4:ef3a64c0
> [<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> Exception stack(0xee85dfb0 to 0xee85dff8)
> dfa0:                                     00000000 00000000 00000000 00000000
> dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
>  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
>  r4:ee806040
> CPU1: stopping
> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef133f18 to 0xef133f60)
> 3f00:                                                       00000000 000509b4
> 3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
> 3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU2: stopping
> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef135f18 to 0xef135f60)
> 5f00:                                                       00000000 0004f168
> 5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
> 5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
>  r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU4: stopping
> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef139f18 to 0xef139f60)
> 9f00:                                                       00000000 0007c528
> 9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
> 9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
>  r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU5: stopping
> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13bf18 to 0xef13bf60)
> bf00:                                                       00000000 00064644
> bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
> bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU6: stopping
> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13df18 to 0xef13df60)
> df00:                                                       00000000 00081fa8
> df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
> df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
>  r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU7: stopping
> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xef13ff18 to 0xef13ff60)
> ff00:                                                       00000000 00028068
> ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
> ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
>  r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
>  r4:0000008d
> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>  r5:00000051 r4:6f12806a
> CPU0: stopping
> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
> Hardware name: Allwinner A83t board
> Backtrace: 
> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>  r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
>  r4:c0e0565c r3:c0109844
> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
> Exception stack(0xc0e01eb8 to 0xc0e01f00)
> 1ea0:                                                       00000000 002dadd8
> 1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
> 1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
>  r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>  r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
>  r4:000000d1
> [<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
> [<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
>  r5:c0d47f38 r4:c0ead1c0
> [<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
> [<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
> Rebooting in 3 seconds..
> DRAM: 1024 MiB
> Trying to boot from MMC1

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Chao Yu]

Hi all,

I've figure out a patch for this issue, could you please try below one?

f2fs: fix potential .flags overflow on 32bit architecture

Thanks,

On 2020-3-12 1:01, Jaegeuk Kim wrote:
> On 03/06, Ondřej Jirman wrote:
>> Hello,
>>
>> On Thu, Feb 27, 2020 at 10:01:50AM +0800, Chao Yu wrote:
>>> On 2020/2/27 2:05, Ondřej Jirman wrote:
>>>>
>>>> No issue after 7h uptime either. So I guess this patch solved it for some
>>>> reason.
>>>
>>> I hope so as well, I will send a formal patch for this.
>>
>> So I had it happen again, even with the patches. This time in f2fs_rename2:
>
> Hmm, I haven't seen this so far. Is it doable to see all the other tasks
> together? And, may I offer to test with some bug fixes together that I sent
> pull-request?
>
> https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git/log/?h=dev
>
> Thanks,
>
>>
>> regards,
>> 	o.
>>
>>  INFO: task ldconfig:620 blocked for more than 122 seconds.
>>       Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
>> ldconfig        D    0   620    441 0x00000001
>> Backtrace:
>> [<c0913190>] (__schedule) from [<c0913834>] (schedule+0x78/0xf4)
>>  r10:e587a000 r9:00000000 r8:e587bd80 r7:ee687eb8 r6:00000002 r5:e587a000
>>  r4:ee8c8000
>> [<c09137bc>] (schedule) from [<c017ec6c>] (rwsem_down_write_slowpath+0x24c/0x4c0)
>>  r5:00000001 r4:ee687ea8
>> [<c017ea20>] (rwsem_down_write_slowpath) from [<c091652c>] (down_write+0x6c/0x70)
>>  r10:ee687d08 r9:e5131170 r8:ee0f0000 r7:ee687ea8 r6:e587be10 r5:ee685740
>>  r4:ee687ea8
>> [<c09164c0>] (down_write) from [<c04149ac>] (f2fs_rename2+0x1a0/0x1114)
>>  r5:ee685740 r4:ee685740
>> [<c041480c>] (f2fs_rename2) from [<c02ea5a8>] (vfs_rename+0x434/0x838)
>>  r10:c041480c r9:e5131170 r8:ee687d08 r7:ee685740 r6:eed21440 r5:00000000
>>  r4:eecb82a8
>> [<c02ea174>] (vfs_rename) from [<c02ed764>] (do_renameat2+0x310/0x494)
>>  r10:eea4c000 r9:00000000 r8:e587bf50 r7:eed21440 r6:00000000 r5:ffffffd9
>>  r4:eea4d000
>> [<c02ed454>] (do_renameat2) from [<c02eec20>] (sys_rename+0x34/0x3c)
>>  r10:00000026 r9:e587a000 r8:c0101204 r7:00000026 r6:00007458 r5:000dac90
>>  r4:00000003
>> [<c02eebec>] (sys_rename) from [<c0101000>] (ret_fast_syscall+0x0/0x54)
>> Exception stack(0xe587bfa8 to 0xe587bff0)
>> bfa0:                   00000003 000dac90 000dac90 be9e1540 0000c7d5 00000000
>> bfc0: 00000003 000dac90 00007458 00000026 0000c7d5 00000001 000cd60c be9e1534
>> bfe0: 00000025 be9e14cc 00014120 0001fd00
>> NMI backtrace for cpu 3
>> CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:600b0013 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0901a5c>] (nmi_cpu_backtrace+0x98/0xcc)
>>  r7:00000000 r6:00000003 r5:00000000 r4:00000003
>> [<c09019c4>] (nmi_cpu_backtrace) from [<c0901b84>] (nmi_trigger_cpumask_backtrace+0xf4/0x138)
>>  r5:c0e08498 r4:c010ea5c
>> [<c0901a90>] (nmi_trigger_cpumask_backtrace) from [<c0110334>] (arch_trigger_cpumask_backtrace+0x20/0x24)
>>  r7:0008908a r6:c0e08c50 r5:00007f84 r4:ef19b674
>> [<c0110314>] (arch_trigger_cpumask_backtrace) from [<c01de160>] (watchdog+0x334/0x540)
>> [<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
>>  r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
>>  r4:ef3a64c0
>> [<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
>> Exception stack(0xee85dfb0 to 0xee85dff8)
>> dfa0:                                     00000000 00000000 00000000 00000000
>> dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
>> dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
>>  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
>>  r4:ee806040
>> Sending NMI from CPU 3 to CPUs 0-2,4-7:
>> NMI backtrace for cpu 2
>> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> PC is at arch_cpu_idle+0x48/0x4c
>> LR is at arch_cpu_idle+0x44/0x4c
>> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60030013
>> sp : ef135f68  ip : ef135f78  fp : ef135f74
>> r10: 00000000  r9 : 00000000  r8 : c0d82830
>> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000002  r4 : ef134000
>> r3 : c011ab00  r2 : ef672d70  r1 : 0004ea3c  r0 : 00000000
>> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
>> Control: 10c5387d  Table: 6d87406a  DAC: 00000051
>> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>>  r7:00000000 r6:00000007 r5:ef135f18 r4:00000002
>> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
>> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>>  r5:ef135f18 r4:c0d83110
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef135f18 to 0xef135f60)
>> 5f00:                                                       00000000 0004ea3c
>> 5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
>> 5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
>>  r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> NMI backtrace for cpu 1
>> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> PC is at arch_cpu_idle+0x48/0x4c
>> LR is at arch_cpu_idle+0x44/0x4c
>> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
>> sp : ef133f68  ip : ef133f78  fp : ef133f74
>> r10: 00000000  r9 : 00000000  r8 : c0d82830
>> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000001  r4 : ef132000
>> r3 : c011ab00  r2 : ef65ed70  r1 : 0004ed5c  r0 : 00000000
>> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
>> Control: 10c5387d  Table: 6de3806a  DAC: 00000051
>> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>>  r7:00000000 r6:00000007 r5:ef133f18 r4:00000001
>> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
>> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>>  r5:ef133f18 r4:c0d83110
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef133f18 to 0xef133f60)
>> 3f00:                                                       00000000 0004ed5c
>> 3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
>> 3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
>>  r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> NMI backtrace for cpu 0
>> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> PC is at arch_cpu_idle+0x48/0x4c
>> LR is at arch_cpu_idle+0x44/0x4c
>> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
>> sp : c0e01f08  ip : c0e01f18  fp : c0e01f14
>> r10: 00000000  r9 : 00000000  r8 : c0d82830
>> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000000  r4 : c0e00000
>> r3 : c011ab00  r2 : ef64ad70  r1 : 002da988  r0 : 00000000
>> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
>> Control: 10c5387d  Table: 6cbc406a  DAC: 00000051
>> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>>  r7:00000000 r6:00000007 r5:c0e01eb8 r4:00000000
>> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
>> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>>  r5:c0e01eb8 r4:c0d83110
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xc0e01eb8 to 0xc0e01f00)
>> 1ea0:                                                       00000000 002da988
>> 1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
>> 1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
>>  r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
>>  r4:000000d1
>> [<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
>> [<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
>>  r5:c0d47f38 r4:c0ead1c0
>> [<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
>> [<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
>> NMI backtrace for cpu 5
>> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> PC is at arch_cpu_idle+0x48/0x4c
>> LR is at arch_cpu_idle+0x44/0x4c
>> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
>> sp : ef13bf68  ip : ef13bf78  fp : ef13bf74
>> r10: 00000000  r9 : 00000000  r8 : c0d82830
>> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000005  r4 : ef13a000
>> r3 : c011ab00  r2 : ef6aed70  r1 : 00064604  r0 : 00000000
>> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
>> Control: 10c5387d  Table: 6dd8806a  DAC: 00000051
>> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>>  r7:00000000 r6:00000007 r5:ef13bf18 r4:00000005
>> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
>> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>>  r5:ef13bf18 r4:c0d83110
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef13bf18 to 0xef13bf60)
>> bf00:                                                       00000000 00064604
>> bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
>> bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
>>  r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> NMI backtrace for cpu 4
>> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> PC is at arch_cpu_idle+0x48/0x4c
>> LR is at arch_cpu_idle+0x44/0x4c
>> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
>> sp : ef139f68  ip : ef139f78  fp : ef139f74
>> r10: 00000000  r9 : 00000000  r8 : c0d82830
>> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000004  r4 : ef138000
>> r3 : c011ab00  r2 : ef69ad70  r1 : 0007c518  r0 : 00000000
>> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
>> Control: 10c5387d  Table: 6def806a  DAC: 00000051
>> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>>  r7:00000000 r6:00000007 r5:ef139f18 r4:00000004
>> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
>> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>>  r5:ef139f18 r4:c0d83110
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef139f18 to 0xef139f60)
>> 9f00:                                                       00000000 0007c518
>> 9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
>> 9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
>>  r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> NMI backtrace for cpu 7
>> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> PC is at arch_cpu_idle+0x48/0x4c
>> LR is at arch_cpu_idle+0x44/0x4c
>> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60000013
>> sp : ef13ff68  ip : ef13ff78  fp : ef13ff74
>> r10: 00000000  r9 : 00000000  r8 : c0d82830
>> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000007  r4 : ef13e000
>> r3 : c011ab00  r2 : ef6d6d70  r1 : 00028058  r0 : 00000000
>> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
>> Control: 10c5387d  Table: 6c5e006a  DAC: 00000051
>> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>>  r7:00000000 r6:00000007 r5:ef13ff18 r4:00000007
>> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
>> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>>  r5:ef13ff18 r4:c0d83110
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef13ff18 to 0xef13ff60)
>> ff00:                                                       00000000 00028058
>> ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
>> ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
>>  r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> NMI backtrace for cpu 6
>> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> PC is at arch_cpu_idle+0x48/0x4c
>> LR is at arch_cpu_idle+0x44/0x4c
>> pc : [<c0109848>]    lr : [<c0109844>]    psr: 60010013
>> sp : ef13df68  ip : ef13df78  fp : ef13df74
>> r10: 00000000  r9 : 00000000  r8 : c0d82830
>> r7 : c0e04ea4  r6 : c0e04e64  r5 : 00000006  r4 : ef13c000
>> r3 : c011ab00  r2 : ef6c2d70  r1 : 00081da8  r0 : 00000000
>> Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
>> Control: 10c5387d  Table: 6565006a  DAC: 00000051
>> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0109ae0>] (show_regs+0x1c/0x20)
>>  r7:00000000 r6:00000007 r5:ef13df18 r4:00000006
>> [<c0109ac4>] (show_regs) from [<c0901a8c>] (nmi_cpu_backtrace+0xc8/0xcc)
>> [<c09019c4>] (nmi_cpu_backtrace) from [<c010fca8>] (handle_IPI+0x64/0x37c)
>>  r5:ef13df18 r4:c0d83110
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef13df18 to 0xef13df60)
>> df00:                                                       00000000 00081da8
>> df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
>> df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
>>  r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> Kernel panic - not syncing: hung_task: blocked tasks
>> CPU: 3 PID: 53 Comm: khungtaskd Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:0008908a r6:600b0093 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c0131fcc>] (panic+0x108/0x310)
>>  r7:0008908a r6:c0e08c50 r5:00000000 r4:c0eadbc8
>> [<c0131ec4>] (panic) from [<c01de16c>] (watchdog+0x340/0x540)
>>  r3:c0f0dd60 r2:00000001 r1:00000000 r0:c0b49e00
>>  r7:0008908a
>> [<c01dde2c>] (watchdog) from [<c01564f4>] (kthread+0x144/0x170)
>>  r10:ef0f7e60 r9:ef3a64dc r8:00000000 r7:ee85c000 r6:00000000 r5:ee806040
>>  r4:ef3a64c0
>> [<c01563b0>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
>> Exception stack(0xee85dfb0 to 0xee85dff8)
>> dfa0:                                     00000000 00000000 00000000 00000000
>> dfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
>> dfe0: 00000000 00000000 00000000 00000000 00000013 00000000
>>  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b0
>>  r4:ee806040
>> CPU1: stopping
>> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef132000 r8:ef133f18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef133f18 to 0xef133f60)
>> 3f00:                                                       00000000 000509b4
>> 3f20: ef65ed70 c011ab00 ef132000 00000001 c0e04e64 c0e04ea4 c0d82830 00000000
>> 3f40: 00000000 ef133f74 ef133f78 ef133f68 c0109844 c0109848 60000013 ffffffff
>>  r9:ef132000 r8:c0d82830 r7:ef133f4c r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000001
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> CPU2: stopping
>> CPU: 2 PID: 0 Comm: swapper/2 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60030193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef134000 r8:ef135f18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef135f18 to 0xef135f60)
>> 5f00:                                                       00000000 0004f168
>> 5f20: ef672d70 c011ab00 ef134000 00000002 c0e04e64 c0e04ea4 c0d82830 00000000
>> 5f40: 00000000 ef135f74 ef135f78 ef135f68 c0109844 c0109848 60030013 ffffffff
>>  r9:ef134000 r8:c0d82830 r7:ef135f4c r6:ffffffff r5:60030013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000002
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> CPU4: stopping
>> CPU: 4 PID: 0 Comm: swapper/4 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef138000 r8:ef139f18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef139f18 to 0xef139f60)
>> 9f00:                                                       00000000 0007c528
>> 9f20: ef69ad70 c011ab00 ef138000 00000004 c0e04e64 c0e04ea4 c0d82830 00000000
>> 9f40: 00000000 ef139f74 ef139f78 ef139f68 c0109844 c0109848 60000013 ffffffff
>>  r9:ef138000 r8:c0d82830 r7:ef139f4c r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000004
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> CPU5: stopping
>> CPU: 5 PID: 0 Comm: swapper/5 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef13a000 r8:ef13bf18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef13bf18 to 0xef13bf60)
>> bf00:                                                       00000000 00064644
>> bf20: ef6aed70 c011ab00 ef13a000 00000005 c0e04e64 c0e04ea4 c0d82830 00000000
>> bf40: 00000000 ef13bf74 ef13bf78 ef13bf68 c0109844 c0109848 60000013 ffffffff
>>  r9:ef13a000 r8:c0d82830 r7:ef13bf4c r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000005
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> CPU6: stopping
>> CPU: 6 PID: 0 Comm: swapper/6 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60010193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef13c000 r8:ef13df18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef13df18 to 0xef13df60)
>> df00:                                                       00000000 00081fa8
>> df20: ef6c2d70 c011ab00 ef13c000 00000006 c0e04e64 c0e04ea4 c0d82830 00000000
>> df40: 00000000 ef13df74 ef13df78 ef13df68 c0109844 c0109848 60010013 ffffffff
>>  r9:ef13c000 r8:c0d82830 r7:ef13df4c r6:ffffffff r5:60010013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000006
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> CPU7: stopping
>> CPU: 7 PID: 0 Comm: swapper/7 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:ef13e000 r8:ef13ff18 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xef13ff18 to 0xef13ff60)
>> ff00:                                                       00000000 00028068
>> ff20: ef6d6d70 c011ab00 ef13e000 00000007 c0e04e64 c0e04ea4 c0d82830 00000000
>> ff40: 00000000 ef13ff74 ef13ff78 ef13ff68 c0109844 c0109848 60000013 ffffffff
>>  r9:ef13e000 r8:c0d82830 r7:ef13ff4c r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:00000000 r9:410fc075 r8:4000406a r7:c0ead5c0 r6:10c0387d r5:00000007
>>  r4:0000008d
>> [<c0165c7c>] (cpu_startup_entry) from [<c010f4c0>] (secondary_start_kernel+0x158/0x164)
>> [<c010f368>] (secondary_start_kernel) from [<4010280c>] (0x4010280c)
>>  r5:00000051 r4:6f12806a
>> CPU0: stopping
>> CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc3-00469-g44d686977effa #48
>> Hardware name: Allwinner A83t board
>> Backtrace:
>> [<c010db14>] (dump_backtrace) from [<c010de98>] (show_stack+0x20/0x24)
>>  r7:00000000 r6:60000193 r5:00000000 r4:c0e9ab90
>> [<c010de78>] (show_stack) from [<c08fabbc>] (dump_stack+0x98/0xac)
>> [<c08fab24>] (dump_stack) from [<c010ff88>] (handle_IPI+0x344/0x37c)
>>  r7:00000000 r6:00000004 r5:c0e05358 r4:c0ead5b0
>> [<c010fc44>] (handle_IPI) from [<c01023d0>] (gic_handle_irq+0x84/0x88)
>>  r10:00000000 r9:c0e00000 r8:c0e01eb8 r7:f0803000 r6:f0802000 r5:f080200c
>>  r4:c0e0565c r3:c0109844
>> [<c010234c>] (gic_handle_irq) from [<c0101acc>] (__irq_svc+0x6c/0x90)
>> Exception stack(0xc0e01eb8 to 0xc0e01f00)
>> 1ea0:                                                       00000000 002dadd8
>> 1ec0: ef64ad70 c011ab00 c0e00000 00000000 c0e04e64 c0e04ea4 c0d82830 00000000
>> 1ee0: 00000000 c0e01f14 c0e01f18 c0e01f08 c0109844 c0109848 60000013 ffffffff
>>  r9:c0e00000 r8:c0d82830 r7:c0e01eec r6:ffffffff r5:60000013 r4:c0109848
>> [<c0109800>] (arch_cpu_idle) from [<c0918444>] (default_idle_call+0x30/0x3c)
>> [<c0918414>] (default_idle_call) from [<c016595c>] (do_idle+0x218/0x290)
>> [<c0165744>] (do_idle) from [<c0165ca4>] (cpu_startup_entry+0x28/0x2c)
>>  r10:c0d47f38 r9:efffcd40 r8:00000089 r7:c0ead210 r6:00000000 r5:c0d47f38
>>  r4:000000d1
>> [<c0165c7c>] (cpu_startup_entry) from [<c091245c>] (rest_init+0xb4/0xbc)
>> [<c09123a8>] (rest_init) from [<c0d00c78>] (arch_call_rest_init+0x18/0x1c)
>>  r5:c0d47f38 r4:c0ead1c0
>> [<c0d00c60>] (arch_call_rest_init) from [<c0d013e8>] (start_kernel+0x6f4/0x714)
>> [<c0d00cf4>] (start_kernel) from [<00000000>] (0x0)
>> Rebooting in 3 seconds..
>> DRAM: 1024 MiB
>> Trying to boot from MMC1
>
>
> _______________________________________________
> Linux-f2fs-devel mailing list
> Linux-f2fs-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
>

Re: [f2fs-dev] Writes stoped working on f2fs after the compression support was added

[Ondřej Jirman]

On Mon, Feb 24, 2020 at 06:41:03PM +0800, Chao Yu wrote:
> On 2020/2/24 18:37, Chao Yu wrote:
> > Hi,
> > 
> > Thanks for the report.
> > 
> > On 2020/2/23 2:17, Ondřej Jirman wrote:
> >> Hello,
> >>
> >> I observe hung background f2fs task on 5.6-rc2+ shortly after boot, leading to
> >> panics. I use f2fs as rootfs. See below for stack trace. The reads continue to
> >> work (if I disable panic on hung task). But sync blocks, writes block, etc.
> >>
> >> It does not happen on all my f2fs filesystems, so it may depend on workload
> >> or my particular filesystem state. It happens on two separate devices though,
> >> both 32-bit, and doesn't happen on a 64-bit device. (might be a false lead,
> >> though)
> >>
> >> I went through the f2fs-for-5.6 tag/branch and reverted each patch right
> >> down to:
> >>
> >>   4c8ff7095bef64fc47e996a938f7d57f9e077da3 f2fs: support data compression
> >>
> >> I could still reproduce the issue.
> >>
> >> After reverting the data compression too, I could no longer reproduce the
> >> issue. Perhaps not very helpful, since that patch is huge.
> >>
> >> I tried to collect some information. Some ftrace logs, etc. Not sure what would
> >> help debug this. Let me know if I can help debug this more.
> >>
> >> Symptoms look the same as this issue:
> >>
> >>   https://www.mail-archive.com/linux-f2fs-devel@lists.sourceforge.net/msg15298.html
> >>
> >> ftrace: https://megous.com/dl/tmp/f2fs-debug-info.txt
> >>
> >> longer ftrace: https://megous.com/dl/tmp/f2fs-debug-info-full.txt
> > 
> > I checked the full trace log, however I didn't find any clue to troubleshot this issue.
> > 
> > [snip]
> > 
> >> dmesg:
> > 
> > Could you dump all other task stack info via "echo "t" > /proc/sysrq-trigger"?
> > 
> >>
> >> [  246.758021] INFO: task kworker/u16:1:58 blocked for more than 122 seconds.
> >> [  246.758040]       Not tainted 5.6.0-rc2-00590-g9983bdae4974e #11
> >> [  246.758044] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> >> [  246.758052] kworker/u16:1   D    0    58      2 0x00000000
> >> [  246.758090] Workqueue: writeback wb_workfn (flush-179:0)
> >> [  246.758099] Backtrace:
> >> [  246.758121] [<c0912b90>] (__schedule) from [<c0913234>] (schedule+0x78/0xf4)
> >> [  246.758130]  r10:da644000 r9:00000000 r8:da645a60 r7:da283e10 r6:00000002 r5:da644000
> >> [  246.758132]  r4:da4d3600
> >> [  246.758148] [<c09131bc>] (schedule) from [<c017ec74>] (rwsem_down_write_slowpath+0x24c/0x4c0)
> >> [  246.758152]  r5:00000001 r4:da283e00
> >> [  246.758161] [<c017ea28>] (rwsem_down_write_slowpath) from [<c0915f2c>] (down_write+0x6c/0x70)
> >> [  246.758167]  r10:da283e00 r9:da645d80 r8:d9ed0000 r7:00000001 r6:00000000 r5:eff213b0
> >> [  246.758169]  r4:da283e00
> >> [  246.758187] [<c0915ec0>] (down_write) from [<c0435b80>] (f2fs_write_single_data_page+0x608/0x7ac)
> > 
> > I'm not sure what is this semaphore, I suspect this is F2FS_I(inode)->i_sem, in order to make
> > sure of this, can you help to add below function, and use them to replace
> > all {down,up}_{write,read}(&.i_sem) invoking? then reproduce this issue and catch the log.
> 
> Sorry, just forgot attaching below function.
> 
> void inode_down_write(struct inode *inode)
> {
> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> 	down_write(&F2FS_I(inode)->i_sem);
> }
> 
> void inode_up_write(struct inode *inode)
> {
> 	up_write(&F2FS_I(inode)->i_sem);
> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> }
> 
> void inode_down_read(struct inode *inode)
> {
> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> 	down_read(&F2FS_I(inode)->i_sem);
> }
> 
> void inode_up_read(struct inode *inode)
> {
> 	up_read(&F2FS_I(inode)->i_sem);
> 	printk("%s from %pS\n", __func__, __builtin_return_address(0));
> }

The actual debug code:

https://megous.com/git/linux/commit/?h=f2fs-debug-5.6&id=2f152bf17c9c5fe1f7c8449cb690ad0739d3ad44

> > 
> > Thanks,
> > 
> >> [  246.758190]  r5:eff213b0 r4:da283c60
> >> [  246.758198] [<c0435578>] (f2fs_write_single_data_page) from [<c0435fd8>] (f2fs_write_cache_pages+0x2b4/0x7c4)
> >> [  246.758204]  r10:da645c28 r9:da283d60 r8:da283c60 r7:0000000f r6:da645d80 r5:00000001
> >> [  246.758206]  r4:eff213b0
> >> [  246.758214] [<c0435d24>] (f2fs_write_cache_pages) from [<c043682c>] (f2fs_write_data_pages+0x344/0x35c)
> >> [  246.758220]  r10:00000000 r9:d9ed002c r8:d9ed0000 r7:00000004 r6:da283d60 r5:da283c60
> >> [  246.758223]  r4:da645d80
> >> [  246.758238] [<c04364e8>] (f2fs_write_data_pages) from [<c0267ee8>] (do_writepages+0x3c/0xd4)
> >> [  246.758244]  r10:0000000a r9:c0e03d00 r8:00000c00 r7:c0264ddc r6:da645d80 r5:da283d60
> >> [  246.758246]  r4:da283c60
> >> [  246.758254] [<c0267eac>] (do_writepages) from [<c0310cbc>] (__writeback_single_inode+0x44/0x454)
> >> [  246.758259]  r7:da283d60 r6:da645eac r5:da645d80 r4:da283c60
> >> [  246.758266] [<c0310c78>] (__writeback_single_inode) from [<c03112d0>] (writeback_sb_inodes+0x204/0x4b0)
> >> [  246.758272]  r10:0000000a r9:c0e03d00 r8:da283cc8 r7:da283c60 r6:da645eac r5:da283d08
> >> [  246.758274]  r4:d9dc9848
> >> [  246.758281] [<c03110cc>] (writeback_sb_inodes) from [<c03115cc>] (__writeback_inodes_wb+0x50/0xe4)
> >> [  246.758287]  r10:da3797a8 r9:c0e03d00 r8:d9dc985c r7:da645eac r6:00000000 r5:d9dc9848
> >> [  246.758289]  r4:da5a8800
> >> [  246.758296] [<c031157c>] (__writeback_inodes_wb) from [<c03118f4>] (wb_writeback+0x294/0x338)
> >> [  246.758302]  r10:fffbf200 r9:da644000 r8:c0e04e64 r7:d9dc9848 r6:d9dc9874 r5:da645eac
> >> [  246.758305]  r4:d9dc9848
> >> [  246.758312] [<c0311660>] (wb_writeback) from [<c0312dac>] (wb_workfn+0x35c/0x54c)
> >> [  246.758318]  r10:da5f2005 r9:d9dc984c r8:d9dc9948 r7:d9dc9848 r6:00000000 r5:d9dc9954
> >> [  246.758321]  r4:000031e6
> >> [  246.758334] [<c0312a50>] (wb_workfn) from [<c014f2b8>] (process_one_work+0x214/0x544)
> >> [  246.758340]  r10:da5f2005 r9:00000200 r8:00000000 r7:da5f2000 r6:ef044400 r5:da5eb000
> >> [  246.758343]  r4:d9dc9954
> >> [  246.758350] [<c014f0a4>] (process_one_work) from [<c014f634>] (worker_thread+0x4c/0x574)
> >> [  246.758357]  r10:ef044400 r9:c0e03d00 r8:ef044418 r7:00000088 r6:ef044400 r5:da5eb014
> >> [  246.758359]  r4:da5eb000
> >> [  246.758368] [<c014f5e8>] (worker_thread) from [<c01564fc>] (kthread+0x144/0x170)
> >> [  246.758374]  r10:ec9e5e90 r9:dabf325c r8:da5eb000 r7:da644000 r6:00000000 r5:da5fe000
> >> [  246.758377]  r4:dabf3240
> >> [  246.758386] [<c01563b8>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)
> >> [  246.758391] Exception stack(0xda645fb0 to 0xda645ff8)
> >> [  246.758397] 5fa0:                                     00000000 00000000 00000000 00000000
> >> [  246.758402] 5fc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
> >> [  246.758407] 5fe0: 00000000 00000000 00000000 00000000 00000013 00000000
> >> [  246.758413]  r10:00000000 r9:00000000 r8:00000000 r7:00000000 r6:00000000 r5:c01563b8
> >> [  246.758416]  r4:da5fe000
> >> .
> >>
> > 
> > 
> > _______________________________________________
> > Linux-f2fs-devel mailing list
> > Linux-f2fs-devel@lists.sourceforge.net
> > https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
> >