[PATCH 1/2] btrfs-progs: convert: Make ext*_image file obey datacsum setting.

[PATCH 1/2] btrfs-progs: convert: Make ext*_image file obey datacsum setting.

[Qu Wenruo]

Before this patch, ext*_image is always set NODATACSUM inode flag.
However btrfs-convert will set normal file with DATACUSM flag by
default, and generate checksum for regular file extent.

Now, a regular file extent is shared by a btrfs file inode with DATACSUM
and ext*_image with NODATACSUM, and it has checksum in csum tree.
This will cause btrfsck complain about odd checksum, since ext*_image is
set NODATACSUM but has checksum generated from regular file extent.

This patch makes convert completely obey datacsum setting, meaning
btrfs-convert will generate csum for every file extent by default.

Reported-by: Tsutomu Itoh <t-itoh@jp.fujitsu.com>
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 btrfs-convert.c | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/btrfs-convert.c b/btrfs-convert.c
index 4dc33b3..d742307 100644
--- a/btrfs-convert.c
+++ b/btrfs-convert.c
@@ -1161,7 +1161,7 @@ static int create_image_file_range(struct btrfs_trans_handle *trans,
 				   struct btrfs_root *root, u64 objectid,
 				   struct btrfs_inode_item *inode,
 				   u64 start_byte, u64 end_byte,
-				   ext2_filsys ext2_fs)
+				   ext2_filsys ext2_fs, int datacsum)
 {
 	u32 blocksize = ext2_fs->blocksize;
 	u32 block = start_byte / blocksize;
@@ -1176,7 +1176,7 @@ static int create_image_file_range(struct btrfs_trans_handle *trans,
 		.disk_block	= 0,
 		.num_blocks	= 0,
 		.boundary	= (u64)-1,
-		.checksum 	= 0,
+		.checksum	= datacsum,
 		.errcode	= 0,
 	};
 	for (; start_byte < end_byte; block++, start_byte += blocksize) {
@@ -1191,7 +1191,7 @@ static int create_image_file_range(struct btrfs_trans_handle *trans,
 	if (data.num_blocks > 0) {
 		ret = record_file_blocks(trans, root, objectid, inode,
 					 data.first_block, data.disk_block,
-					 data.num_blocks, 0);
+					 data.num_blocks, datacsum);
 		if (ret)
 			goto fail;
 		data.first_block += data.num_blocks;
@@ -1199,7 +1199,7 @@ static int create_image_file_range(struct btrfs_trans_handle *trans,
 	if (last_block > data.first_block) {
 		ret = record_file_blocks(trans, root, objectid, inode,
 					 data.first_block, 0, last_block -
-					 data.first_block, 0);
+					 data.first_block, datacsum);
 		if (ret)
 			goto fail;
 	}
@@ -1210,7 +1210,7 @@ fail:
  * Create the ext2fs image file.
  */
 static int create_ext2_image(struct btrfs_root *root, ext2_filsys ext2_fs,
-			     const char *name)
+			     const char *name, int datacsum)
 {
 	int ret;
 	struct btrfs_key key;
@@ -1231,11 +1231,14 @@ static int create_ext2_image(struct btrfs_root *root, ext2_filsys ext2_fs,
 	u64 last_byte;
 	u64 first_free;
 	u64 total_bytes;
+	u64 flags = BTRFS_INODE_READONLY;
 	u32 sectorsize = root->sectorsize;
 
 	total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
 	first_free =  BTRFS_SUPER_INFO_OFFSET + sectorsize * 2 - 1;
 	first_free &= ~((u64)sectorsize - 1);
+	if (!datacsum)
+		flags |= BTRFS_INODE_NODATASUM;
 
 	memset(&btrfs_inode, 0, sizeof(btrfs_inode));
 	btrfs_set_stack_inode_generation(&btrfs_inode, 1);
@@ -1243,8 +1246,7 @@ static int create_ext2_image(struct btrfs_root *root, ext2_filsys ext2_fs,
 	btrfs_set_stack_inode_nlink(&btrfs_inode, 1);
 	btrfs_set_stack_inode_nbytes(&btrfs_inode, 0);
 	btrfs_set_stack_inode_mode(&btrfs_inode, S_IFREG | 0400);
-	btrfs_set_stack_inode_flags(&btrfs_inode, BTRFS_INODE_NODATASUM |
-				    BTRFS_INODE_READONLY);
+	btrfs_set_stack_inode_flags(&btrfs_inode,  flags);
 	btrfs_init_path(&path);
 	trans = btrfs_start_transaction(root, 1);
 	BUG_ON(!trans);
@@ -1271,6 +1273,12 @@ static int create_ext2_image(struct btrfs_root *root, ext2_filsys ext2_fs,
 					       key.objectid, sectorsize);
 		if (ret)
 			goto fail;
+		if (datacsum) {
+			ret = csum_disk_extent(trans, root, key.objectid,
+					       sectorsize);
+			if (ret)
+				goto fail;
+		}
 	}
 
 	while(1) {
@@ -1323,7 +1331,8 @@ next:
 		if (bytenr > last_byte) {
 			ret = create_image_file_range(trans, root, objectid,
 						      &btrfs_inode, last_byte,
-						      bytenr, ext2_fs);
+						      bytenr, ext2_fs,
+						      datacsum);
 			if (ret)
 				goto fail;
 		}
@@ -1346,7 +1355,8 @@ next:
 	if (total_bytes > last_byte) {
 		ret = create_image_file_range(trans, root, objectid,
 					      &btrfs_inode, last_byte,
-					      total_bytes, ext2_fs);
+					      total_bytes, ext2_fs,
+					      datacsum);
 		if (ret)
 			goto fail;
 	}
@@ -2374,7 +2384,7 @@ static int do_convert(const char *devname, int datacsum, int packing, int noxatt
 		fprintf(stderr, "unable to create subvol\n");
 		goto fail;
 	}
-	ret = create_ext2_image(ext2_root, ext2_fs, "image");
+	ret = create_ext2_image(ext2_root, ext2_fs, "image", datacsum);
 	if (ret) {
 		fprintf(stderr, "error during create_ext2_image %d\n", ret);
 		goto fail;
-- 
2.3.4

[PATCH 2/2] btrfs-progs: convert-test: Add test for converting ext* with regular file extent.

[Qu Wenruo]

Before previous patch, btrfs-convert will result fsck complain if there
is any regular file extent in newly converted btrfs.

Add test case for it.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 tests/convert-tests.sh | 85 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 55 insertions(+), 30 deletions(-)

diff --git a/tests/convert-tests.sh b/tests/convert-tests.sh
index f6b919d..5c4f22e 100644
--- a/tests/convert-tests.sh
+++ b/tests/convert-tests.sh
@@ -4,48 +4,73 @@
 # clean.
 #
 
-here=`pwd`
+unset top
+unset LANG
+LANG=C
+script_dir=$(dirname $(realpath $0))
+top=$(realpath $script_dir/../)
+TEST_DEV=${TEST_DEV:-}
+TEST_MNT=${TEST_MNT:-$top/tests/mnt}
+RESULT="$top/tests/convert-tests-results.txt"
+IMAGE="$script_dir/test.img"
 
-_fail()
-{
-	echo "$*" | tee -a convert-tests-results.txt
-	exit 1
-}
+source $top/tests/common
+export top
+export RESULT
+# For comprehensive convert test which needs write something into ext*
+export TEST_MNT
+export LANG
+
+rm -f $RESULT
+mkdir -p $TEST_MNT || _fail "unable to create mount point on $TEST_MNT"
+
+# test reply on btrfs-convert
+check_prereq btrfs-convert
+check_prereq btrfs
 
-rm -f convert-tests-results.txt
 
-test(){
+convert_test(){
 	echo "    [TEST]   $1"
 	nodesize=$2
 	shift 2
-	echo "creating ext image with: $*" >> convert-tests-results.txt
+	echo "creating ext image with: $*" >> $RESULT
 	# 256MB is the smallest acceptable btrfs image.
-	rm -f $here/test.img >> convert-tests-results.txt 2>&1 \
+	rm -f $IMAGE >> $RESULT 2>&1 \
 		|| _fail "could not remove test image file"
-	truncate -s 256M $here/test.img >> convert-tests-results.txt 2>&1 \
+	truncate -s 256M $IMAGE >> $RESULT 2>&1 \
 		|| _fail "could not create test image file"
-	$* -F $here/test.img >> convert-tests-results.txt 2>&1 \
+	$* -F $IMAGE >> $RESULT 2>&1 \
 		|| _fail "filesystem create failed"
-	$here/btrfs-convert -N "$nodesize" $here/test.img \
-			>> convert-tests-results.txt 2>&1 \
+
+	# write a file with regular file extent
+	$SUDO_HELPER mount $IMAGE $TEST_MNT
+	$SUDO_HELPER dd if=/dev/zero bs=$nodesize count=4 of=$TEST_MNT/test \
+		1>/dev/null 2>&1
+	$SUDO_HELPER umount $TEST_MNT
+
+	# do convert test
+	$top/btrfs-convert -N "$nodesize" $script_dir/test.img \
+			>> $RESULT 2>&1 \
 		|| _fail "btrfs-convert failed"
-	$here/btrfs check $here/test.img >> convert-tests-results.txt 2>&1 \
+	$top/btrfs check $script_dir/test.img >> $RESULT 2>&1 \
 		|| _fail "btrfs check detected errors"
 }
 
+setup_root_helper
+
 # btrfs-convert requires 4k blocksize.
-test "ext2 4k nodesize" 4096 mke2fs -b 4096
-test "ext3 4k nodesize" 4096 mke2fs -j -b 4096
-test "ext4 4k nodesize" 4096 mke2fs -t ext4 -b 4096
-test "ext2 8k nodesize" 8192 mke2fs -b 4096
-test "ext3 8k nodesize" 8192 mke2fs -j -b 4096
-test "ext4 8k nodesize" 8192 mke2fs -t ext4 -b 4096
-test "ext2 16k nodesize" 16384 mke2fs -b 4096
-test "ext3 16k nodesize" 16384 mke2fs -j -b 4096
-test "ext4 16k nodesize" 16384 mke2fs -t ext4 -b 4096
-test "ext2 32k nodesize" 32768 mke2fs -b 4096
-test "ext3 32k nodesize" 32768 mke2fs -j -b 4096
-test "ext4 32k nodesize" 32768 mke2fs -t ext4 -b 4096
-test "ext2 64k nodesize" 65536 mke2fs -b 4096
-test "ext3 64k nodesize" 65536 mke2fs -j -b 4096
-test "ext4 64k nodesize" 65536 mke2fs -t ext4 -b 4096
+convert_test "ext2 4k nodesize" 4096 mke2fs -b 4096
+convert_test "ext3 4k nodesize" 4096 mke2fs -j -b 4096
+convert_test "ext4 4k nodesize" 4096 mke2fs -t ext4 -b 4096
+convert_test "ext2 8k nodesize" 8192 mke2fs -b 4096
+convert_test "ext3 8k nodesize" 8192 mke2fs -j -b 4096
+convert_test "ext4 8k nodesize" 8192 mke2fs -t ext4 -b 4096
+convert_test "ext2 16k nodesize" 16384 mke2fs -b 4096
+convert_test "ext3 16k nodesize" 16384 mke2fs -j -b 4096
+convert_test "ext4 16k nodesize" 16384 mke2fs -t ext4 -b 4096
+convert_test "ext2 32k nodesize" 32768 mke2fs -b 4096
+convert_test "ext3 32k nodesize" 32768 mke2fs -j -b 4096
+convert_test "ext4 32k nodesize" 32768 mke2fs -t ext4 -b 4096
+convert_test "ext2 64k nodesize" 65536 mke2fs -b 4096
+convert_test "ext3 64k nodesize" 65536 mke2fs -j -b 4096
+convert_test "ext4 64k nodesize" 65536 mke2fs -t ext4 -b 4096
-- 
2.3.4

Re: [PATCH 2/2] btrfs-progs: convert-test: Add test for converting ext* with regular file extent.

[David Sterba]

On Thu, Apr 02, 2015 at 10:21:36AM +0800, Qu Wenruo wrote:
> Before previous patch, btrfs-convert will result fsck complain if there
> is any regular file extent in newly converted btrfs.
> 
> Add test case for it.

Please separate the changes that update generic code and the test
itself.

> +script_dir=$(dirname $(realpath $0))
> +top=$(realpath $script_dir/../)

Please use upper case names.

> +TEST_DEV=${TEST_DEV:-}
> +TEST_MNT=${TEST_MNT:-$top/tests/mnt}
> +RESULT="$top/tests/convert-tests-results.txt"

RESULTS

> +IMAGE="$script_dir/test.img"
>  
> -_fail()
> -{
> -	echo "$*" | tee -a convert-tests-results.txt
> -	exit 1
> -}
> +source $top/tests/common
> +export top
> +export RESULT
> +# For comprehensive convert test which needs write something into ext*
> +export TEST_MNT
> +export LANG
> +
> +rm -f $RESULT
> +mkdir -p $TEST_MNT || _fail "unable to create mount point on $TEST_MNT"
> +
> +# test reply on btrfs-convert
> +check_prereq btrfs-convert
> +check_prereq btrfs
>  
> -rm -f convert-tests-results.txt
>  
> -test(){
> +convert_test(){
>  	echo "    [TEST]   $1"
>  	nodesize=$2
>  	shift 2
> -	echo "creating ext image with: $*" >> convert-tests-results.txt
> +	echo "creating ext image with: $*" >> $RESULT
>  	# 256MB is the smallest acceptable btrfs image.
> -	rm -f $here/test.img >> convert-tests-results.txt 2>&1 \
> +	rm -f $IMAGE >> $RESULT 2>&1 \
>  		|| _fail "could not remove test image file"
> -	truncate -s 256M $here/test.img >> convert-tests-results.txt 2>&1 \
> +	truncate -s 256M $IMAGE >> $RESULT 2>&1 \
>  		|| _fail "could not create test image file"
> -	$* -F $here/test.img >> convert-tests-results.txt 2>&1 \
> +	$* -F $IMAGE >> $RESULT 2>&1 \
>  		|| _fail "filesystem create failed"
> -	$here/btrfs-convert -N "$nodesize" $here/test.img \
> -			>> convert-tests-results.txt 2>&1 \
> +
> +	# write a file with regular file extent
> +	$SUDO_HELPER mount $IMAGE $TEST_MNT
> +	$SUDO_HELPER dd if=/dev/zero bs=$nodesize count=4 of=$TEST_MNT/test \
> +		1>/dev/null 2>&1
> +	$SUDO_HELPER umount $TEST_MNT
> +
> +	# do convert test
> +	$top/btrfs-convert -N "$nodesize" $script_dir/test.img \

$IMAGE instead of "$script_dir/test.img"

> +			>> $RESULT 2>&1 \
>  		|| _fail "btrfs-convert failed"
> -	$here/btrfs check $here/test.img >> convert-tests-results.txt 2>&1 \

same here

> +	$top/btrfs check $script_dir/test.img >> $RESULT 2>&1 \

and here

>  		|| _fail "btrfs check detected errors"

Thanks.

Re: [PATCH 1/2] btrfs-progs: convert: Make ext*_image file obey datacsum setting.

[David Sterba]

On Thu, Apr 02, 2015 at 10:21:35AM +0800, Qu Wenruo wrote:
> Before this patch, ext*_image is always set NODATACSUM inode flag.
> However btrfs-convert will set normal file with DATACUSM flag by
> default, and generate checksum for regular file extent.
> 
> Now, a regular file extent is shared by a btrfs file inode with DATACSUM
> and ext*_image with NODATACSUM, and it has checksum in csum tree.
> This will cause btrfsck complain about odd checksum, since ext*_image is
> set NODATACSUM but has checksum generated from regular file extent.
> 
> This patch makes convert completely obey datacsum setting, meaning
> btrfs-convert will generate csum for every file extent by default.
> 
> Reported-by: Tsutomu Itoh <t-itoh@jp.fujitsu.com>
> Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>

Applied, thanks.

Re: [PATCH 2/2] btrfs-progs: convert-test: Add test for converting ext* with regular file extent.

[Qu Wenruo]

-------- Original Message  --------
Subject: Re: [PATCH 2/2] btrfs-progs: convert-test: Add test for 
converting ext* with regular file extent.
From: David Sterba <dsterba@suse.cz>
To: Qu Wenruo <quwenruo@cn.fujitsu.com>
Date: 2015年04月02日 23:45

> On Thu, Apr 02, 2015 at 10:21:36AM +0800, Qu Wenruo wrote:
>> Before previous patch, btrfs-convert will result fsck complain if there
>> is any regular file extent in newly converted btrfs.
>>
>> Add test case for it.
>
> Please separate the changes that update generic code and the test
> itself.
OK, I'll update it soon.
>
>> +script_dir=$(dirname $(realpath $0))
>> +top=$(realpath $script_dir/../)
>
> Please use upper case names.
Some fsck-tests use lower case names, I'll update them too.

Thanks,
Qu
>
>> +TEST_DEV=${TEST_DEV:-}
>> +TEST_MNT=${TEST_MNT:-$top/tests/mnt}
>> +RESULT="$top/tests/convert-tests-results.txt"
>
> RESULTS
>
>> +IMAGE="$script_dir/test.img"
>>
>> -_fail()
>> -{
>> -	echo "$*" | tee -a convert-tests-results.txt
>> -	exit 1
>> -}
>> +source $top/tests/common
>> +export top
>> +export RESULT
>> +# For comprehensive convert test which needs write something into ext*
>> +export TEST_MNT
>> +export LANG
>> +
>> +rm -f $RESULT
>> +mkdir -p $TEST_MNT || _fail "unable to create mount point on $TEST_MNT"
>> +
>> +# test reply on btrfs-convert
>> +check_prereq btrfs-convert
>> +check_prereq btrfs
>>
>> -rm -f convert-tests-results.txt
>>
>> -test(){
>> +convert_test(){
>>   	echo "    [TEST]   $1"
>>   	nodesize=$2
>>   	shift 2
>> -	echo "creating ext image with: $*" >> convert-tests-results.txt
>> +	echo "creating ext image with: $*" >> $RESULT
>>   	# 256MB is the smallest acceptable btrfs image.
>> -	rm -f $here/test.img >> convert-tests-results.txt 2>&1 \
>> +	rm -f $IMAGE >> $RESULT 2>&1 \
>>   		|| _fail "could not remove test image file"
>> -	truncate -s 256M $here/test.img >> convert-tests-results.txt 2>&1 \
>> +	truncate -s 256M $IMAGE >> $RESULT 2>&1 \
>>   		|| _fail "could not create test image file"
>> -	$* -F $here/test.img >> convert-tests-results.txt 2>&1 \
>> +	$* -F $IMAGE >> $RESULT 2>&1 \
>>   		|| _fail "filesystem create failed"
>> -	$here/btrfs-convert -N "$nodesize" $here/test.img \
>> -			>> convert-tests-results.txt 2>&1 \
>> +
>> +	# write a file with regular file extent
>> +	$SUDO_HELPER mount $IMAGE $TEST_MNT
>> +	$SUDO_HELPER dd if=/dev/zero bs=$nodesize count=4 of=$TEST_MNT/test \
>> +		1>/dev/null 2>&1
>> +	$SUDO_HELPER umount $TEST_MNT
>> +
>> +	# do convert test
>> +	$top/btrfs-convert -N "$nodesize" $script_dir/test.img \
>
> $IMAGE instead of "$script_dir/test.img"
>
>> +			>> $RESULT 2>&1 \
>>   		|| _fail "btrfs-convert failed"
>> -	$here/btrfs check $here/test.img >> convert-tests-results.txt 2>&1 \
>
> same here
>
>> +	$top/btrfs check $script_dir/test.img >> $RESULT 2>&1 \
>
> and here
>
>>   		|| _fail "btrfs check detected errors"
>
> Thanks.
>