fstests.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
* [PATCH] generic: check reflink multiple mmap write
@ 2019-10-10  4:14 Darrick J. Wong
  2019-10-10  6:49 ` Christoph Hellwig
  2019-10-11 21:41 ` [PATCH v2] " Darrick J. Wong
  0 siblings, 2 replies; 5+ messages in thread
From: Darrick J. Wong @ 2019-10-10  4:14 UTC (permalink / raw)
  To: Eryu Guan, Christoph Hellwig; +Cc: fstests

From: Darrick J. Wong <darrick.wong@oracle.com>

Add a test to make sure that we can handle multiple memory mappings to a
physical storage extent shared by multiple files, and that we can handle
the copy on write operation without error.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
 src/Makefile                |    2 -
 src/mmap-write-concurrent.c |  157 +++++++++++++++++++++++++++++++++++++++++++
 tests/generic/945           |   89 ++++++++++++++++++++++++
 tests/generic/945.out       |   25 +++++++
 tests/generic/group         |    1 
 5 files changed, 273 insertions(+), 1 deletion(-)
 create mode 100644 src/mmap-write-concurrent.c
 create mode 100755 tests/generic/945
 create mode 100644 tests/generic/945.out

diff --git a/src/Makefile b/src/Makefile
index ef7cfa63..5bc33e77 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -16,7 +16,7 @@ TARGETS = dirstress fill fill2 getpagesize holes lstat64 \
 	holetest t_truncate_self t_mmap_dio af_unix t_mmap_stale_pmd \
 	t_mmap_cow_race t_mmap_fallocate fsync-err t_mmap_write_ro \
 	t_ext4_dax_journal_corruption t_ext4_dax_inline_corruption \
-	t_ofd_locks t_locks_execve t_mmap_collision
+	t_ofd_locks t_locks_execve t_mmap_collision mmap-write-concurrent
 
 LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
 	preallo_rw_pattern_writer ftrunc trunc fs_perms testx looptest \
diff --git a/src/mmap-write-concurrent.c b/src/mmap-write-concurrent.c
new file mode 100644
index 00000000..0eccea0a
--- /dev/null
+++ b/src/mmap-write-concurrent.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0-or-newer
+/*
+ * Copyright (c) 2019 Oracle.
+ * All Rights Reserved.
+ *
+ * Create writable mappings to multiple files and write them all to test
+ * concurrent mmap writes to the same shared blocks.
+ */
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(int argc, char *argv[])
+{
+	struct stat *statbuf;
+	off_t offset;
+	size_t length;
+	char *endptr;
+	char **mappings;
+	int *fds;
+	char *buf;
+	int i;
+	int ret;
+
+	if (argc < 4) {
+		printf("Usage: %s offset len file [files...]\n", argv[0]);
+		return 1;
+	}
+
+	/* Parse mwrite offset. */
+	errno = 0;
+	offset = strtoul(argv[1], &endptr, 0);
+	if (errno) {
+		perror(argv[1]);
+		return 1;
+	}
+	if (*endptr != '\0') {
+		fprintf(stderr, "%s: not a proper file offset?\n", argv[1]);
+		return 1;
+	}
+
+	/* Parse mwrite length. */
+	errno = 0;
+	length = strtoul(argv[2], &endptr, 0);
+	if (errno) {
+		perror(argv[1]);
+		return 1;
+	}
+	if (*endptr != '\0') {
+		fprintf(stderr, "%s: not a proper file length?\n", argv[2]);
+		return 1;
+	}
+
+	mappings = calloc(argc - 3, sizeof(char *));
+	if (!mappings) {
+		perror("calloc maps");
+		return 1;
+	}
+
+	fds = calloc(argc - 3, sizeof(int));
+	if (!fds) {
+		perror("calloc fds");
+		return 1;
+	}
+
+	buf = malloc(length);
+	if (!buf) {
+		perror("malloc buf");
+		return 1;
+	}
+
+	statbuf = calloc(argc - 3, sizeof(struct stat));
+	if (!statbuf) {
+		perror("calloc statbuf");
+		return 1;
+	}
+
+	for (i = 0; i < argc - 3; i++) {
+		char *fname = argv[i + 3];
+
+		/* Open file, create mapping for the range we want. */
+		fds[i] = open(fname, O_RDWR);
+		if (fds[i] < 0) {
+			perror(fname);
+			return 1;
+		}
+
+		ret = fstat(fds[i], &statbuf[i]);
+		if (ret) {
+			perror(fname);
+			return 1;
+		}
+
+		if (offset + length > statbuf[i].st_size) {
+			fprintf(stderr, "%s: file must be %llu bytes\n",
+					fname,
+					(unsigned long long)offset + length);
+			return 1;
+		}
+
+		mappings[i] = mmap(NULL, statbuf[i].st_size,
+				PROT_READ | PROT_WRITE, MAP_SHARED,
+				fds[i], 0);
+		if (mappings[i] == MAP_FAILED) {
+			perror(fname);
+			return 1;
+		}
+
+		/*
+		 * Make sure the mapping for region we're going to write is
+		 * already populated in the page cache.
+		 */
+		memcpy(buf, mappings[i] + offset, length);
+	}
+
+	/* Dirty the same region in each file to test COW. */
+	for (i = 0; i < argc - 3; i++) {
+		memset(buf, 0x58 + i, length);
+		memcpy(mappings[i] + offset, buf, length);
+	}
+	for (i = 0; i < argc - 3; i++) {
+		ret = msync(mappings[i], offset + length, MS_SYNC);
+		if (ret) {
+			perror("msync");
+			return 1;
+		}
+	}
+
+	/* Close everything. */
+	for (i = 0; i < argc - 3; i++) {
+		ret = munmap(mappings[i], statbuf[i].st_size);
+		if (ret) {
+			perror("munmap");
+			return 1;
+		}
+
+		ret = close(fds[i]);
+		if (ret) {
+			perror("close");
+			return 1;
+		}
+	}
+
+	/* Free everything. */
+	free(statbuf);
+	free(buf);
+	free(fds);
+	free(mappings);
+
+	return 0;
+}
diff --git a/tests/generic/945 b/tests/generic/945
new file mode 100755
index 00000000..53e26966
--- /dev/null
+++ b/tests/generic/945
@@ -0,0 +1,89 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0-or-newer
+# Copyright (c) 2019, Oracle and/or its affiliates.  All Rights Reserved.
+#
+# FS QA Test No. 945
+#
+# Make sure that we can handle multiple mmap writers to the same file.
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1    # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -rf $tmp.* $testdir
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/reflink
+
+# real QA test starts here
+_supported_os Linux
+_supported_fs generic
+_require_command "$FILEFRAG_PROG" filefrag
+_require_test_reflink
+_require_cp_reflink
+
+rm -f $seqres.full
+
+compare() {
+	md5sum $testdir/file1 | _filter_test_dir
+	md5sum $testdir/file2 | _filter_test_dir
+	md5sum $testdir/file3 | _filter_test_dir
+	md5sum $testdir/file4 | _filter_test_dir
+
+	cmp -s $testdir/file1 $testdir/file2 || echo "Files 1-2 do not match"
+	cmp -s $testdir/file1 $testdir/file3 || echo "Files 1-3 do not match"
+	cmp -s $testdir/file1 $testdir/file4 || echo "Files 1-4 do not match"
+}
+
+testdir=$TEST_DIR/test-$seq
+rm -rf $testdir
+mkdir $testdir
+
+echo "Create the original files"
+filesz=$((65536 * 4))
+_pwrite_byte 0x61 0 $filesz $testdir/file1 >> $seqres.full
+_cp_reflink $testdir/file1 $testdir/file2 >> $seqres.full
+_cp_reflink $testdir/file1 $testdir/file3 >> $seqres.full
+_cp_reflink $testdir/file1 $testdir/file4 >> $seqres.full
+_test_cycle_mount
+
+echo "Compare files before cow"
+compare
+
+echo "mwrite all copies"
+off=$(( (filesz / 2) - 168 ))
+len=337
+./src/mmap-write-concurrent $off $len $testdir/file1 $testdir/file2 \
+		$testdir/file3 $testdir/file4
+
+echo "Compare files before remount"
+compare
+_test_cycle_mount
+
+echo "Compare files after remount"
+compare
+
+echo "Check for non-shared extents"
+$FILEFRAG_PROG -v $testdir/file1 $testdir/file2 $testdir/file3 \
+		$testdir/file4 | grep '^[[:space:]]*[0-9]*:' > $testdir/fiemap
+cat $testdir/fiemap >> $seqres.full
+grep -q 'shared' $testdir/fiemap || \
+		echo "Expected to find shared extents"
+
+grep -q -v 'shared' $testdir/fiemap || \
+		echo "Expected to find non-shared extents"
+
+# success, all done
+status=0
+exit
diff --git a/tests/generic/945.out b/tests/generic/945.out
new file mode 100644
index 00000000..ba79ddfc
--- /dev/null
+++ b/tests/generic/945.out
@@ -0,0 +1,25 @@
+QA output created by 945
+Create the original files
+Compare files before cow
+c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file1
+c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file2
+c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file3
+c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file4
+mwrite all copies
+Compare files before remount
+83f84225313027ef51fea4f86239d432  TEST_DIR/test-945/file1
+d9d86b794e7d626ccee90392518c9048  TEST_DIR/test-945/file2
+c0dae8b7541aad8ef4ed3b9ffd20e9a3  TEST_DIR/test-945/file3
+094457223ea2751265e617631f5f4aa9  TEST_DIR/test-945/file4
+Files 1-2 do not match
+Files 1-3 do not match
+Files 1-4 do not match
+Compare files after remount
+83f84225313027ef51fea4f86239d432  TEST_DIR/test-945/file1
+d9d86b794e7d626ccee90392518c9048  TEST_DIR/test-945/file2
+c0dae8b7541aad8ef4ed3b9ffd20e9a3  TEST_DIR/test-945/file3
+094457223ea2751265e617631f5f4aa9  TEST_DIR/test-945/file4
+Files 1-2 do not match
+Files 1-3 do not match
+Files 1-4 do not match
+Check for non-shared extents
diff --git a/tests/generic/group b/tests/generic/group
index 4584667f..f77c5b21 100644
--- a/tests/generic/group
+++ b/tests/generic/group
@@ -576,3 +576,4 @@
 715 dangerous_norepair
 716 dangerous_norepair
 720 dangerous_norepair
+945 auto quick rw clone

^ permalink raw reply related	[flat|nested] 5+ messages in thread

* Re: [PATCH] generic: check reflink multiple mmap write
  2019-10-10  4:14 [PATCH] generic: check reflink multiple mmap write Darrick J. Wong
@ 2019-10-10  6:49 ` Christoph Hellwig
  2019-10-11 21:41 ` [PATCH v2] " Darrick J. Wong
  1 sibling, 0 replies; 5+ messages in thread
From: Christoph Hellwig @ 2019-10-10  6:49 UTC (permalink / raw)
  To: Darrick J. Wong; +Cc: Eryu Guan, Christoph Hellwig, fstests

On Wed, Oct 09, 2019 at 09:14:40PM -0700, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> Add a test to make sure that we can handle multiple memory mappings to a
> physical storage extent shared by multiple files, and that we can handle
> the copy on write operation without error.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>

Thanks, this looks good:

Reviewed-by: Christoph Hellwig <hch@lst.de>

To further try fund DAX corner cases we should also try different
offsets for the different files, or different offsets in the same
file referring to the same blocks.  I can take a stab at that once
I find some time.

^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH v2] generic: check reflink multiple mmap write
  2019-10-10  4:14 [PATCH] generic: check reflink multiple mmap write Darrick J. Wong
  2019-10-10  6:49 ` Christoph Hellwig
@ 2019-10-11 21:41 ` Darrick J. Wong
  2019-10-13 12:11   ` Eryu Guan
  1 sibling, 1 reply; 5+ messages in thread
From: Darrick J. Wong @ 2019-10-11 21:41 UTC (permalink / raw)
  To: Eryu Guan, Christoph Hellwig; +Cc: fstests

From: Darrick J. Wong <darrick.wong@oracle.com>

Add a test to make sure that we can handle multiple memory mappings to a
physical storage extent shared by multiple files, and that we can handle
the copy on write operation without error.  Make sure we can also handle
mappings at different offsets in the page cache.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
v2: test at different offsets
---
 src/Makefile                |    2 -
 src/mmap-write-concurrent.c |  155 +++++++++++++++++++++++++++++++++++++++++++
 tests/generic/945           |  104 +++++++++++++++++++++++++++++
 tests/generic/945.out       |   31 +++++++++
 tests/generic/group         |    1 
 5 files changed, 292 insertions(+), 1 deletion(-)
 create mode 100644 src/mmap-write-concurrent.c
 create mode 100755 tests/generic/945
 create mode 100644 tests/generic/945.out

diff --git a/src/Makefile b/src/Makefile
index ef7cfa63..5bc33e77 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -16,7 +16,7 @@ TARGETS = dirstress fill fill2 getpagesize holes lstat64 \
 	holetest t_truncate_self t_mmap_dio af_unix t_mmap_stale_pmd \
 	t_mmap_cow_race t_mmap_fallocate fsync-err t_mmap_write_ro \
 	t_ext4_dax_journal_corruption t_ext4_dax_inline_corruption \
-	t_ofd_locks t_locks_execve t_mmap_collision
+	t_ofd_locks t_locks_execve t_mmap_collision mmap-write-concurrent
 
 LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
 	preallo_rw_pattern_writer ftrunc trunc fs_perms testx looptest \
diff --git a/src/mmap-write-concurrent.c b/src/mmap-write-concurrent.c
new file mode 100644
index 00000000..38364de8
--- /dev/null
+++ b/src/mmap-write-concurrent.c
@@ -0,0 +1,155 @@
+// SPDX-License-Identifier: GPL-2.0-or-newer
+/*
+ * Copyright (c) 2019 Oracle.
+ * All Rights Reserved.
+ *
+ * Create writable mappings to multiple files and write them all to test
+ * concurrent mmap writes to the same shared blocks.
+ */
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct file_info {
+	char			*mapping;
+	off_t			file_offset;
+	off_t			file_length;
+	int			fd;
+};
+
+int
+main(
+	int			argc,
+	char			*argv[])
+{
+	struct file_info	*fi;
+	size_t			length;
+	char			*endptr;
+	unsigned int		nr_files;
+	unsigned int		i;
+	char			*buf;
+	int			ret;
+
+	if (argc < 4) {
+		printf("Usage: %s len offset file [offset file]...\n", argv[0]);
+		return 1;
+	}
+
+	/* Parse mwrite length. */
+	errno = 0;
+	length = strtoul(argv[1], &endptr, 0);
+	if (errno) {
+		perror(argv[1]);
+		return 1;
+	}
+	if (*endptr != '\0') {
+		fprintf(stderr, "%s: not a proper file length?\n", argv[2]);
+		return 1;
+	}
+
+	/* Allocate file info */
+	nr_files = (argc - 2) / 2;
+
+	fi = calloc(nr_files, sizeof(struct file_info));
+	if (!fi) {
+		perror("calloc file info");
+		return 1;
+	}
+
+	buf = malloc(length);
+	if (!buf) {
+		perror("malloc buf");
+		return 1;
+	}
+
+	for (i = 0; i < nr_files; i++) {
+		struct stat	statbuf;
+		char		*offset = argv[((i + 1) * 2)];
+		char		*fname = argv[((i + 1) * 2) + 1];
+
+		/* Open file, create mapping for the range we want. */
+		fi[i].fd = open(fname, O_RDWR);
+		if (fi[i].fd < 0) {
+			perror(fname);
+			return 1;
+		}
+
+		/* Parse mwrite offset */
+		errno = 0;
+		fi[i].file_offset = strtoul(offset, &endptr, 0);
+		if (errno) {
+			perror(argv[1]);
+			return 1;
+		}
+
+		/* Remember file size */
+		ret = fstat(fi[i].fd, &statbuf);
+		if (ret) {
+			perror(fname);
+			return 1;
+		}
+		fi[i].file_length = statbuf.st_size;
+
+		if (fi[i].file_offset + length > fi[i].file_length) {
+			fprintf(stderr, "%s: file must be %llu bytes\n",
+				fname,
+				(unsigned long long)fi[i].file_offset + length);
+			return 1;
+		}
+
+		/* Create the mapping */
+		fi[i].mapping = mmap(NULL, fi[i].file_length,
+				PROT_READ | PROT_WRITE, MAP_SHARED,
+				fi[i].fd, 0);
+		if (fi[i].mapping == MAP_FAILED) {
+			perror(fname);
+			return 1;
+		}
+
+		/*
+		 * Make sure the mapping for region we're going to write is
+		 * already populated in the page cache.
+		 */
+		memcpy(buf, fi[i].mapping + fi[i].file_offset, length);
+	}
+
+	/* Dirty the same region in each file to test COW. */
+	for (i = 0; i < nr_files; i++) {
+		memset(buf, 0x62 + i, length);
+		memcpy(fi[i].mapping + fi[i].file_offset, buf, length);
+	}
+	for (i = 0; i < nr_files; i++) {
+		ret = msync(fi[i].mapping, fi[i].file_offset + length, MS_SYNC);
+		if (ret) {
+			perror("msync");
+			return 1;
+		}
+	}
+
+	/* Close everything. */
+	for (i = 0; i < nr_files; i++) {
+		ret = munmap(fi[i].mapping, fi[i].file_length);
+		if (ret) {
+			perror("munmap");
+			return 1;
+		}
+
+		ret = close(fi[i].fd);
+		if (ret) {
+			perror("close");
+			return 1;
+		}
+	}
+
+	/* Free everything. */
+	free(buf);
+	free(fi);
+
+	return 0;
+}
diff --git a/tests/generic/945 b/tests/generic/945
new file mode 100755
index 00000000..38b5883c
--- /dev/null
+++ b/tests/generic/945
@@ -0,0 +1,104 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0-or-newer
+# Copyright (c) 2019, Oracle and/or its affiliates.  All Rights Reserved.
+#
+# FS QA Test No. 945
+#
+# Make sure that we can handle multiple mmap writers to the same file.
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1    # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -rf $tmp.* $testdir
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/reflink
+
+# real QA test starts here
+_supported_os Linux
+_supported_fs generic
+_require_command "$FILEFRAG_PROG" filefrag
+_require_test_reflink
+_require_cp_reflink
+
+rm -f $seqres.full
+
+compare() {
+	for i in $(seq 1 8); do
+		md5sum $testdir/file$i | _filter_test_dir
+		echo $testdir/file$i >> $seqres.full
+		od -tx1 -Ad -c $testdir/file$i >> $seqres.full
+	done
+}
+
+testdir=$TEST_DIR/test-$seq
+rm -rf $testdir
+mkdir $testdir
+
+echo "Create the original files"
+blksz=65536
+filesz=$((blksz * 4))
+_pwrite_byte 0x61 0 $filesz $testdir/file1 >> $seqres.full
+_cp_reflink $testdir/file1 $testdir/file2 >> $seqres.full
+_cp_reflink $testdir/file1 $testdir/file3 >> $seqres.full
+_cp_reflink $testdir/file1 $testdir/file4 >> $seqres.full
+_reflink_range $testdir/file1 0 $testdir/file5 $blksz $filesz >> $seqres.full
+_reflink_range $testdir/file1 0 $testdir/file6 $((blksz * 2)) $filesz >> $seqres.full
+_reflink_range $testdir/file1 0 $testdir/file7 $((blksz * 3)) $filesz >> $seqres.full
+_reflink_range $testdir/file1 0 $testdir/file8 $((blksz * 4)) $filesz >> $seqres.full
+_test_cycle_mount
+
+echo "Compare files before cow" | tee -a $seqres.full
+compare
+
+echo "mwrite all copies" | tee -a $seqres.full
+off=$(( (filesz / 2) - 168 ))
+len=337
+./src/mmap-write-concurrent $len \
+		$off $testdir/file1 \
+		$off $testdir/file2 \
+		$off $testdir/file3 \
+		$off $testdir/file4 \
+		$((off + blksz)) $testdir/file5 \
+		$((off + (blksz * 2))) $testdir/file6 \
+		$((off + (blksz * 3))) $testdir/file7 \
+		$((off + (blksz * 4))) $testdir/file8 \
+		168 $testdir/file1 \
+		$((blksz - 168)) $testdir/file2 \
+		$((filesz - 777)) $testdir/file3 \
+		$(((blksz * 3) - 168)) $testdir/file4 \
+
+
+echo "Compare files before remount" | tee -a $seqres.full
+compare
+_test_cycle_mount
+
+echo "Compare files after remount" | tee -a $seqres.full
+compare
+
+echo "Check for non-shared extents" | tee -a $seqres.full
+$FILEFRAG_PROG -v $testdir/file1 $testdir/file2 $testdir/file3 $testdir/file4 \
+		  $testdir/file5 $testdir/file6 $testdir/file7 $testdir/file8 \
+		  | grep '^[[:space:]]*[0-9]*:' > $testdir/fiemap
+cat $testdir/fiemap >> $seqres.full
+grep -q 'shared' $testdir/fiemap || \
+		echo "Expected to find shared extents"
+
+grep -q -v 'shared' $testdir/fiemap || \
+		echo "Expected to find non-shared extents"
+
+# success, all done
+status=0
+exit
diff --git a/tests/generic/945.out b/tests/generic/945.out
new file mode 100644
index 00000000..ad1e21ff
--- /dev/null
+++ b/tests/generic/945.out
@@ -0,0 +1,31 @@
+QA output created by 945
+Create the original files
+Compare files before cow
+c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file1
+c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file2
+c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file3
+c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file4
+74e6b9b1a03fdf09293c089b002800f8  TEST_DIR/test-945/file5
+c14f20b97155e3fc11a17532d02ad9df  TEST_DIR/test-945/file6
+22eb46e0f4a3742c8d86346845b7bc80  TEST_DIR/test-945/file7
+4d292f06cec9d3f1bece4822cd5ef532  TEST_DIR/test-945/file8
+mwrite all copies
+Compare files before remount
+c1b46135a2620ae6da21bbfd4cbb3cba  TEST_DIR/test-945/file1
+16f0bc0f97c42d2ad903c519095b8126  TEST_DIR/test-945/file2
+eb71e3135ca2abf33bb9081e2b49a876  TEST_DIR/test-945/file3
+2678dfcb77bed4dc29e19836bef82e5b  TEST_DIR/test-945/file4
+2802d75dbee4f29d62124aa7b473edca  TEST_DIR/test-945/file5
+acd58cf3d33ef905e26800a0e049223c  TEST_DIR/test-945/file6
+1b68d203e5a1c1b45a9510bedcd1e126  TEST_DIR/test-945/file7
+9479709b697ced2e3a57c17bc1b97373  TEST_DIR/test-945/file8
+Compare files after remount
+c1b46135a2620ae6da21bbfd4cbb3cba  TEST_DIR/test-945/file1
+16f0bc0f97c42d2ad903c519095b8126  TEST_DIR/test-945/file2
+eb71e3135ca2abf33bb9081e2b49a876  TEST_DIR/test-945/file3
+2678dfcb77bed4dc29e19836bef82e5b  TEST_DIR/test-945/file4
+2802d75dbee4f29d62124aa7b473edca  TEST_DIR/test-945/file5
+acd58cf3d33ef905e26800a0e049223c  TEST_DIR/test-945/file6
+1b68d203e5a1c1b45a9510bedcd1e126  TEST_DIR/test-945/file7
+9479709b697ced2e3a57c17bc1b97373  TEST_DIR/test-945/file8
+Check for non-shared extents
diff --git a/tests/generic/group b/tests/generic/group
index 4584667f..f77c5b21 100644
--- a/tests/generic/group
+++ b/tests/generic/group
@@ -576,3 +576,4 @@
 715 dangerous_norepair
 716 dangerous_norepair
 720 dangerous_norepair
+945 auto quick rw clone

^ permalink raw reply related	[flat|nested] 5+ messages in thread

* Re: [PATCH v2] generic: check reflink multiple mmap write
  2019-10-11 21:41 ` [PATCH v2] " Darrick J. Wong
@ 2019-10-13 12:11   ` Eryu Guan
  2019-10-13 16:23     ` Darrick J. Wong
  0 siblings, 1 reply; 5+ messages in thread
From: Eryu Guan @ 2019-10-13 12:11 UTC (permalink / raw)
  To: Darrick J. Wong; +Cc: Christoph Hellwig, fstests

On Fri, Oct 11, 2019 at 02:41:20PM -0700, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> Add a test to make sure that we can handle multiple memory mappings to a
> physical storage extent shared by multiple files, and that we can handle
> the copy on write operation without error.  Make sure we can also handle
> mappings at different offsets in the page cache.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> ---
> v2: test at different offsets
> ---
>  src/Makefile                |    2 -
>  src/mmap-write-concurrent.c |  155 +++++++++++++++++++++++++++++++++++++++++++

I added an entry in .gitignore file.

>  tests/generic/945           |  104 +++++++++++++++++++++++++++++
>  tests/generic/945.out       |   31 +++++++++
>  tests/generic/group         |    1 
>  5 files changed, 292 insertions(+), 1 deletion(-)
>  create mode 100644 src/mmap-write-concurrent.c
>  create mode 100755 tests/generic/945
>  create mode 100644 tests/generic/945.out
> 
> diff --git a/src/Makefile b/src/Makefile
> index ef7cfa63..5bc33e77 100644
> --- a/src/Makefile
> +++ b/src/Makefile
> @@ -16,7 +16,7 @@ TARGETS = dirstress fill fill2 getpagesize holes lstat64 \
>  	holetest t_truncate_self t_mmap_dio af_unix t_mmap_stale_pmd \
>  	t_mmap_cow_race t_mmap_fallocate fsync-err t_mmap_write_ro \
>  	t_ext4_dax_journal_corruption t_ext4_dax_inline_corruption \
> -	t_ofd_locks t_locks_execve t_mmap_collision
> +	t_ofd_locks t_locks_execve t_mmap_collision mmap-write-concurrent
>  
>  LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
>  	preallo_rw_pattern_writer ftrunc trunc fs_perms testx looptest \
> diff --git a/src/mmap-write-concurrent.c b/src/mmap-write-concurrent.c
> new file mode 100644
> index 00000000..38364de8
> --- /dev/null
> +++ b/src/mmap-write-concurrent.c
> @@ -0,0 +1,155 @@
> +// SPDX-License-Identifier: GPL-2.0-or-newer
> +/*
> + * Copyright (c) 2019 Oracle.
> + * All Rights Reserved.
> + *
> + * Create writable mappings to multiple files and write them all to test
> + * concurrent mmap writes to the same shared blocks.
> + */
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +#include <stdio.h>
> +#include <errno.h>
> +#include <stdlib.h>
> +#include <string.h>
> +
> +struct file_info {
> +	char			*mapping;
> +	off_t			file_offset;
> +	off_t			file_length;
> +	int			fd;
> +};
> +
> +int
> +main(
> +	int			argc,
> +	char			*argv[])
> +{
> +	struct file_info	*fi;
> +	size_t			length;
> +	char			*endptr;
> +	unsigned int		nr_files;
> +	unsigned int		i;
> +	char			*buf;
> +	int			ret;
> +
> +	if (argc < 4) {
> +		printf("Usage: %s len offset file [offset file]...\n", argv[0]);
> +		return 1;
> +	}
> +
> +	/* Parse mwrite length. */
> +	errno = 0;
> +	length = strtoul(argv[1], &endptr, 0);
> +	if (errno) {
> +		perror(argv[1]);
> +		return 1;
> +	}
> +	if (*endptr != '\0') {
> +		fprintf(stderr, "%s: not a proper file length?\n", argv[2]);
> +		return 1;
> +	}
> +
> +	/* Allocate file info */
> +	nr_files = (argc - 2) / 2;
> +
> +	fi = calloc(nr_files, sizeof(struct file_info));
> +	if (!fi) {
> +		perror("calloc file info");
> +		return 1;
> +	}
> +
> +	buf = malloc(length);
> +	if (!buf) {
> +		perror("malloc buf");
> +		return 1;
> +	}
> +
> +	for (i = 0; i < nr_files; i++) {
> +		struct stat	statbuf;
> +		char		*offset = argv[((i + 1) * 2)];
> +		char		*fname = argv[((i + 1) * 2) + 1];
> +
> +		/* Open file, create mapping for the range we want. */
> +		fi[i].fd = open(fname, O_RDWR);
> +		if (fi[i].fd < 0) {
> +			perror(fname);
> +			return 1;
> +		}
> +
> +		/* Parse mwrite offset */
> +		errno = 0;
> +		fi[i].file_offset = strtoul(offset, &endptr, 0);
> +		if (errno) {
> +			perror(argv[1]);
> +			return 1;
> +		}
> +
> +		/* Remember file size */
> +		ret = fstat(fi[i].fd, &statbuf);
> +		if (ret) {
> +			perror(fname);
> +			return 1;
> +		}
> +		fi[i].file_length = statbuf.st_size;
> +
> +		if (fi[i].file_offset + length > fi[i].file_length) {
> +			fprintf(stderr, "%s: file must be %llu bytes\n",
> +				fname,
> +				(unsigned long long)fi[i].file_offset + length);
> +			return 1;
> +		}
> +
> +		/* Create the mapping */
> +		fi[i].mapping = mmap(NULL, fi[i].file_length,
> +				PROT_READ | PROT_WRITE, MAP_SHARED,
> +				fi[i].fd, 0);
> +		if (fi[i].mapping == MAP_FAILED) {
> +			perror(fname);
> +			return 1;
> +		}
> +
> +		/*
> +		 * Make sure the mapping for region we're going to write is
> +		 * already populated in the page cache.
> +		 */
> +		memcpy(buf, fi[i].mapping + fi[i].file_offset, length);
> +	}
> +
> +	/* Dirty the same region in each file to test COW. */
> +	for (i = 0; i < nr_files; i++) {
> +		memset(buf, 0x62 + i, length);
> +		memcpy(fi[i].mapping + fi[i].file_offset, buf, length);
> +	}
> +	for (i = 0; i < nr_files; i++) {
> +		ret = msync(fi[i].mapping, fi[i].file_offset + length, MS_SYNC);
> +		if (ret) {
> +			perror("msync");
> +			return 1;
> +		}
> +	}
> +
> +	/* Close everything. */
> +	for (i = 0; i < nr_files; i++) {
> +		ret = munmap(fi[i].mapping, fi[i].file_length);
> +		if (ret) {
> +			perror("munmap");
> +			return 1;
> +		}
> +
> +		ret = close(fi[i].fd);
> +		if (ret) {
> +			perror("close");
> +			return 1;
> +		}
> +	}
> +
> +	/* Free everything. */
> +	free(buf);
> +	free(fi);
> +
> +	return 0;
> +}
> diff --git a/tests/generic/945 b/tests/generic/945
> new file mode 100755
> index 00000000..38b5883c
> --- /dev/null
> +++ b/tests/generic/945
> @@ -0,0 +1,104 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0-or-newer
> +# Copyright (c) 2019, Oracle and/or its affiliates.  All Rights Reserved.
> +#
> +# FS QA Test No. 945
> +#
> +# Make sure that we can handle multiple mmap writers to the same file.
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +
> +here=`pwd`
> +tmp=/tmp/$$
> +status=1    # failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -rf $tmp.* $testdir
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/filter
> +. ./common/reflink
> +
> +# real QA test starts here
> +_supported_os Linux
> +_supported_fs generic
> +_require_command "$FILEFRAG_PROG" filefrag

Also added

_require_test_program "mmap-write-concurrent"

> +_require_test_reflink
> +_require_cp_reflink
> +
> +rm -f $seqres.full
> +
> +compare() {
> +	for i in $(seq 1 8); do
> +		md5sum $testdir/file$i | _filter_test_dir
> +		echo $testdir/file$i >> $seqres.full
> +		od -tx1 -Ad -c $testdir/file$i >> $seqres.full
> +	done
> +}
> +
> +testdir=$TEST_DIR/test-$seq
> +rm -rf $testdir
> +mkdir $testdir
> +
> +echo "Create the original files"
> +blksz=65536
> +filesz=$((blksz * 4))
> +_pwrite_byte 0x61 0 $filesz $testdir/file1 >> $seqres.full
> +_cp_reflink $testdir/file1 $testdir/file2 >> $seqres.full
> +_cp_reflink $testdir/file1 $testdir/file3 >> $seqres.full
> +_cp_reflink $testdir/file1 $testdir/file4 >> $seqres.full
> +_reflink_range $testdir/file1 0 $testdir/file5 $blksz $filesz >> $seqres.full
> +_reflink_range $testdir/file1 0 $testdir/file6 $((blksz * 2)) $filesz >> $seqres.full
> +_reflink_range $testdir/file1 0 $testdir/file7 $((blksz * 3)) $filesz >> $seqres.full
> +_reflink_range $testdir/file1 0 $testdir/file8 $((blksz * 4)) $filesz >> $seqres.full
> +_test_cycle_mount
> +
> +echo "Compare files before cow" | tee -a $seqres.full
> +compare
> +
> +echo "mwrite all copies" | tee -a $seqres.full
> +off=$(( (filesz / 2) - 168 ))
> +len=337
> +./src/mmap-write-concurrent $len \

And used "$here/src/mmap-write-concurrent .." here.

Thanks,
Eryu

> +		$off $testdir/file1 \
> +		$off $testdir/file2 \
> +		$off $testdir/file3 \
> +		$off $testdir/file4 \
> +		$((off + blksz)) $testdir/file5 \
> +		$((off + (blksz * 2))) $testdir/file6 \
> +		$((off + (blksz * 3))) $testdir/file7 \
> +		$((off + (blksz * 4))) $testdir/file8 \
> +		168 $testdir/file1 \
> +		$((blksz - 168)) $testdir/file2 \
> +		$((filesz - 777)) $testdir/file3 \
> +		$(((blksz * 3) - 168)) $testdir/file4 \
> +
> +
> +echo "Compare files before remount" | tee -a $seqres.full
> +compare
> +_test_cycle_mount
> +
> +echo "Compare files after remount" | tee -a $seqres.full
> +compare
> +
> +echo "Check for non-shared extents" | tee -a $seqres.full
> +$FILEFRAG_PROG -v $testdir/file1 $testdir/file2 $testdir/file3 $testdir/file4 \
> +		  $testdir/file5 $testdir/file6 $testdir/file7 $testdir/file8 \
> +		  | grep '^[[:space:]]*[0-9]*:' > $testdir/fiemap
> +cat $testdir/fiemap >> $seqres.full
> +grep -q 'shared' $testdir/fiemap || \
> +		echo "Expected to find shared extents"
> +
> +grep -q -v 'shared' $testdir/fiemap || \
> +		echo "Expected to find non-shared extents"
> +
> +# success, all done
> +status=0
> +exit
> diff --git a/tests/generic/945.out b/tests/generic/945.out
> new file mode 100644
> index 00000000..ad1e21ff
> --- /dev/null
> +++ b/tests/generic/945.out
> @@ -0,0 +1,31 @@
> +QA output created by 945
> +Create the original files
> +Compare files before cow
> +c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file1
> +c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file2
> +c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file3
> +c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file4
> +74e6b9b1a03fdf09293c089b002800f8  TEST_DIR/test-945/file5
> +c14f20b97155e3fc11a17532d02ad9df  TEST_DIR/test-945/file6
> +22eb46e0f4a3742c8d86346845b7bc80  TEST_DIR/test-945/file7
> +4d292f06cec9d3f1bece4822cd5ef532  TEST_DIR/test-945/file8
> +mwrite all copies
> +Compare files before remount
> +c1b46135a2620ae6da21bbfd4cbb3cba  TEST_DIR/test-945/file1
> +16f0bc0f97c42d2ad903c519095b8126  TEST_DIR/test-945/file2
> +eb71e3135ca2abf33bb9081e2b49a876  TEST_DIR/test-945/file3
> +2678dfcb77bed4dc29e19836bef82e5b  TEST_DIR/test-945/file4
> +2802d75dbee4f29d62124aa7b473edca  TEST_DIR/test-945/file5
> +acd58cf3d33ef905e26800a0e049223c  TEST_DIR/test-945/file6
> +1b68d203e5a1c1b45a9510bedcd1e126  TEST_DIR/test-945/file7
> +9479709b697ced2e3a57c17bc1b97373  TEST_DIR/test-945/file8
> +Compare files after remount
> +c1b46135a2620ae6da21bbfd4cbb3cba  TEST_DIR/test-945/file1
> +16f0bc0f97c42d2ad903c519095b8126  TEST_DIR/test-945/file2
> +eb71e3135ca2abf33bb9081e2b49a876  TEST_DIR/test-945/file3
> +2678dfcb77bed4dc29e19836bef82e5b  TEST_DIR/test-945/file4
> +2802d75dbee4f29d62124aa7b473edca  TEST_DIR/test-945/file5
> +acd58cf3d33ef905e26800a0e049223c  TEST_DIR/test-945/file6
> +1b68d203e5a1c1b45a9510bedcd1e126  TEST_DIR/test-945/file7
> +9479709b697ced2e3a57c17bc1b97373  TEST_DIR/test-945/file8
> +Check for non-shared extents
> diff --git a/tests/generic/group b/tests/generic/group
> index 4584667f..f77c5b21 100644
> --- a/tests/generic/group
> +++ b/tests/generic/group
> @@ -576,3 +576,4 @@
>  715 dangerous_norepair
>  716 dangerous_norepair
>  720 dangerous_norepair
> +945 auto quick rw clone

^ permalink raw reply	[flat|nested] 5+ messages in thread

* Re: [PATCH v2] generic: check reflink multiple mmap write
  2019-10-13 12:11   ` Eryu Guan
@ 2019-10-13 16:23     ` Darrick J. Wong
  0 siblings, 0 replies; 5+ messages in thread
From: Darrick J. Wong @ 2019-10-13 16:23 UTC (permalink / raw)
  To: Eryu Guan; +Cc: Christoph Hellwig, fstests

On Sun, Oct 13, 2019 at 08:11:37PM +0800, Eryu Guan wrote:
> On Fri, Oct 11, 2019 at 02:41:20PM -0700, Darrick J. Wong wrote:
> > From: Darrick J. Wong <darrick.wong@oracle.com>
> > 
> > Add a test to make sure that we can handle multiple memory mappings to a
> > physical storage extent shared by multiple files, and that we can handle
> > the copy on write operation without error.  Make sure we can also handle
> > mappings at different offsets in the page cache.
> > 
> > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > ---
> > v2: test at different offsets
> > ---
> >  src/Makefile                |    2 -
> >  src/mmap-write-concurrent.c |  155 +++++++++++++++++++++++++++++++++++++++++++
> 
> I added an entry in .gitignore file.

Thanks!

<snip>
> > +
> > +# real QA test starts here
> > +_supported_os Linux
> > +_supported_fs generic
> > +_require_command "$FILEFRAG_PROG" filefrag
> 
> Also added
> 
> _require_test_program "mmap-write-concurrent"
> 

> > +echo "mwrite all copies" | tee -a $seqres.full
> > +off=$(( (filesz / 2) - 168 ))
> > +len=337
> > +./src/mmap-write-concurrent $len \
> 
> And used "$here/src/mmap-write-concurrent .." here.

Old habits die hard, I guess. :(  Thanks for fixing those up on
commit...

--D

> Thanks,
> Eryu
> 
> > +		$off $testdir/file1 \
> > +		$off $testdir/file2 \
> > +		$off $testdir/file3 \
> > +		$off $testdir/file4 \
> > +		$((off + blksz)) $testdir/file5 \
> > +		$((off + (blksz * 2))) $testdir/file6 \
> > +		$((off + (blksz * 3))) $testdir/file7 \
> > +		$((off + (blksz * 4))) $testdir/file8 \
> > +		168 $testdir/file1 \
> > +		$((blksz - 168)) $testdir/file2 \
> > +		$((filesz - 777)) $testdir/file3 \
> > +		$(((blksz * 3) - 168)) $testdir/file4 \
> > +
> > +
> > +echo "Compare files before remount" | tee -a $seqres.full
> > +compare
> > +_test_cycle_mount
> > +
> > +echo "Compare files after remount" | tee -a $seqres.full
> > +compare
> > +
> > +echo "Check for non-shared extents" | tee -a $seqres.full
> > +$FILEFRAG_PROG -v $testdir/file1 $testdir/file2 $testdir/file3 $testdir/file4 \
> > +		  $testdir/file5 $testdir/file6 $testdir/file7 $testdir/file8 \
> > +		  | grep '^[[:space:]]*[0-9]*:' > $testdir/fiemap
> > +cat $testdir/fiemap >> $seqres.full
> > +grep -q 'shared' $testdir/fiemap || \
> > +		echo "Expected to find shared extents"
> > +
> > +grep -q -v 'shared' $testdir/fiemap || \
> > +		echo "Expected to find non-shared extents"
> > +
> > +# success, all done
> > +status=0
> > +exit
> > diff --git a/tests/generic/945.out b/tests/generic/945.out
> > new file mode 100644
> > index 00000000..ad1e21ff
> > --- /dev/null
> > +++ b/tests/generic/945.out
> > @@ -0,0 +1,31 @@
> > +QA output created by 945
> > +Create the original files
> > +Compare files before cow
> > +c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file1
> > +c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file2
> > +c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file3
> > +c946b71bb69c07daf25470742c967e7c  TEST_DIR/test-945/file4
> > +74e6b9b1a03fdf09293c089b002800f8  TEST_DIR/test-945/file5
> > +c14f20b97155e3fc11a17532d02ad9df  TEST_DIR/test-945/file6
> > +22eb46e0f4a3742c8d86346845b7bc80  TEST_DIR/test-945/file7
> > +4d292f06cec9d3f1bece4822cd5ef532  TEST_DIR/test-945/file8
> > +mwrite all copies
> > +Compare files before remount
> > +c1b46135a2620ae6da21bbfd4cbb3cba  TEST_DIR/test-945/file1
> > +16f0bc0f97c42d2ad903c519095b8126  TEST_DIR/test-945/file2
> > +eb71e3135ca2abf33bb9081e2b49a876  TEST_DIR/test-945/file3
> > +2678dfcb77bed4dc29e19836bef82e5b  TEST_DIR/test-945/file4
> > +2802d75dbee4f29d62124aa7b473edca  TEST_DIR/test-945/file5
> > +acd58cf3d33ef905e26800a0e049223c  TEST_DIR/test-945/file6
> > +1b68d203e5a1c1b45a9510bedcd1e126  TEST_DIR/test-945/file7
> > +9479709b697ced2e3a57c17bc1b97373  TEST_DIR/test-945/file8
> > +Compare files after remount
> > +c1b46135a2620ae6da21bbfd4cbb3cba  TEST_DIR/test-945/file1
> > +16f0bc0f97c42d2ad903c519095b8126  TEST_DIR/test-945/file2
> > +eb71e3135ca2abf33bb9081e2b49a876  TEST_DIR/test-945/file3
> > +2678dfcb77bed4dc29e19836bef82e5b  TEST_DIR/test-945/file4
> > +2802d75dbee4f29d62124aa7b473edca  TEST_DIR/test-945/file5
> > +acd58cf3d33ef905e26800a0e049223c  TEST_DIR/test-945/file6
> > +1b68d203e5a1c1b45a9510bedcd1e126  TEST_DIR/test-945/file7
> > +9479709b697ced2e3a57c17bc1b97373  TEST_DIR/test-945/file8
> > +Check for non-shared extents
> > diff --git a/tests/generic/group b/tests/generic/group
> > index 4584667f..f77c5b21 100644
> > --- a/tests/generic/group
> > +++ b/tests/generic/group
> > @@ -576,3 +576,4 @@
> >  715 dangerous_norepair
> >  716 dangerous_norepair
> >  720 dangerous_norepair
> > +945 auto quick rw clone

^ permalink raw reply	[flat|nested] 5+ messages in thread

end of thread, other threads:[~2019-10-13 16:24 UTC | newest]

Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-10-10  4:14 [PATCH] generic: check reflink multiple mmap write Darrick J. Wong
2019-10-10  6:49 ` Christoph Hellwig
2019-10-11 21:41 ` [PATCH v2] " Darrick J. Wong
2019-10-13 12:11   ` Eryu Guan
2019-10-13 16:23     ` Darrick J. Wong

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).