Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit 8fc46b4d03bc61c4e72b69c3bc59bd50c4943e03:

  btrace2fio: expand rate options, bug fixes (2014-09-18 17:23:55 +0200)

are available in the git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to c0b69b92fb155424946b19228da9be0924e9e96c:

  dedupe: if percentage is 100, don't go through random + math (2014-09-22 14:20:05 -0600)

----------------------------------------------------------------
Jens Axboe (2):
      Basic support for dedupe
      dedupe: if percentage is 100, don't go through random + math

 HOWTO            |   20 ++++++++++++++------
 cconv.c          |    2 ++
 fio.1            |    7 +++++++
 fio.h            |    3 +++
 init.c           |    7 ++++++-
 io_u.c           |   41 +++++++++++++++++++++++++++++++++++++----
 lib/rand.h       |    8 ++++++++
 options.c        |   22 ++++++++++++++++++++++
 server.h         |    2 +-
 thread_options.h |    6 ++++--
 10 files changed, 104 insertions(+), 14 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO b/HOWTO
index 73e58ff..23746ce 100644
--- a/HOWTO
+++ b/HOWTO
@@ -565,12 +565,20 @@ buffer_compress_chunk=int	See buffer_compress_percentage. This
 		alternate random and zeroed data throughout the IO
 		buffer.
 
-buffer_pattern=str	If set, fio will fill the io buffers with this pattern.
-		If not set, the contents of io buffers is defined by the other
-		options related to buffer contents. The setting can be any
-		pattern of bytes, and can be prefixed with 0x for hex values.
-		It may also be a string, where the string must then be
-		wrapped with "".
+buffer_pattern=str	If set, fio will fill the io buffers with this
+		pattern. If not set, the contents of io buffers is defined by
+		the other options related to buffer contents. The setting can
+		be any pattern of bytes, and can be prefixed with 0x for hex
+		values. It may also be a string, where the string must then
+		be wrapped with "".
+
+dedupe_percentage=int	If set, fio will generate this percentage of
+		identical buffers when writing. These buffers will be
+		naturally dedupable. The contents of the buffers depend on
+		what other buffer compression settings have been set. It's
+		possible to have the individual buffers either fully
+		compressible, or not at all. This option only controls the
+		distribution of unique buffers.
 
 nrfiles=int	Number of files to use for this job. Defaults to 1.
 
diff --git a/cconv.c b/cconv.c
index d4fb158..4a40ed0 100644
--- a/cconv.c
+++ b/cconv.c
@@ -241,6 +241,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
 	o->latency_percentile.u.f = fio_uint64_to_double(le64_to_cpu(top->latency_percentile.u.i));
 	o->compress_percentage = le32_to_cpu(top->compress_percentage);
 	o->compress_chunk = le32_to_cpu(top->compress_chunk);
+	o->dedupe_percentage = le32_to_cpu(top->dedupe_percentage);
 
 	o->trim_backlog = le64_to_cpu(top->trim_backlog);
 
@@ -401,6 +402,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->latency_percentile.u.i = __cpu_to_le64(fio_double_to_uint64(o->latency_percentile.u.f));
 	top->compress_percentage = cpu_to_le32(o->compress_percentage);
 	top->compress_chunk = cpu_to_le32(o->compress_chunk);
+	top->dedupe_percentage = cpu_to_le32(o->dedupe_percentage);
 
 	for (i = 0; i < DDIR_RWDIR_CNT; i++) {
 		top->bs[i] = cpu_to_le32(o->bs[i]);
diff --git a/fio.1 b/fio.1
index e3334bd..bc6c9fa 100644
--- a/fio.1
+++ b/fio.1
@@ -481,6 +481,13 @@ setting can be any pattern of bytes, and can be prefixed with 0x for hex
 values. It may also be a string, where the string must then be wrapped with
 "".
 .TP
+.BI dedupe_percentage \fR=\fPint
+If set, fio will generate this percentage of identical buffers when writing.
+These buffers will be naturally dedupable. The contents of the buffers depend
+on what other buffer compression settings have been set. It's possible to have
+the individual buffers either fully compressible, or not at all. This option
+only controls the distribution of unique buffers.
+.TP
 .BI nrfiles \fR=\fPint
 Number of files to use for this job.  Default: 1.
 .TP
diff --git a/fio.h b/fio.h
index dfbad6d..136b430 100644
--- a/fio.h
+++ b/fio.h
@@ -89,6 +89,7 @@ enum {
 	FIO_RAND_SEQ_RAND_WRITE_OFF,
 	FIO_RAND_SEQ_RAND_TRIM_OFF,
 	FIO_RAND_START_DELAY,
+	FIO_DEDUPE_OFF,
 	FIO_RAND_NR_OFFS,
 };
 
@@ -177,6 +178,8 @@ struct thread_data {
 	};
 
 	struct frand_state buf_state;
+	struct frand_state buf_state_prev;
+	struct frand_state dedupe_state;
 
 	unsigned int verify_batch;
 	unsigned int trim_batch;
diff --git a/init.c b/init.c
index 62c7dc2..5b0290d 100644
--- a/init.c
+++ b/init.c
@@ -836,7 +836,9 @@ static void td_fill_rand_seeds_internal(struct thread_data *td)
 void td_fill_rand_seeds(struct thread_data *td)
 {
 	if (td->o.allrand_repeatable) {
-		for (int i = 0; i < FIO_RAND_NR_OFFS; i++)
+		unsigned int i;
+
+		for (i = 0; i < FIO_RAND_NR_OFFS; i++)
 			td->rand_seeds[i] = FIO_RANDSEED * td->thread_number
 			       	+ i;
 	}
@@ -847,6 +849,9 @@ void td_fill_rand_seeds(struct thread_data *td)
 		td_fill_rand_seeds_internal(td);
 
 	init_rand_seed(&td->buf_state, td->rand_seeds[FIO_RAND_BUF_OFF]);
+	frand_copy(&td->buf_state_prev, &td->buf_state);
+
+	init_rand_seed(&td->dedupe_state, td->rand_seeds[FIO_DEDUPE_OFF]);
 }
 
 /*
diff --git a/io_u.c b/io_u.c
index 7cbdb91..eac871b 100644
--- a/io_u.c
+++ b/io_u.c
@@ -1828,6 +1828,34 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u)
 	}
 }
 
+/*
+ * See if we should reuse the last seed, if dedupe is enabled
+ */
+static struct frand_state *get_buf_state(struct thread_data *td)
+{
+	unsigned int v;
+	unsigned long r;
+
+	if (!td->o.dedupe_percentage)
+		return &td->buf_state;
+	else if (td->o.dedupe_percentage == 100)
+		return &td->buf_state_prev;
+
+	r = __rand(&td->dedupe_state);
+	v = 1 + (int) (100.0 * (r / (FRAND_MAX + 1.0)));
+
+	if (v <= td->o.dedupe_percentage)
+		return &td->buf_state_prev;
+
+	return &td->buf_state;
+}
+
+static void save_buf_state(struct thread_data *td, struct frand_state *rs)
+{
+	if (rs == &td->buf_state)
+		frand_copy(&td->buf_state_prev, rs);
+}
+
 void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write,
 		    unsigned int max_bs)
 {
@@ -1835,6 +1863,9 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write,
 		fill_buffer_pattern(td, buf, max_bs);
 	else if (!td->o.zero_buffers) {
 		unsigned int perc = td->o.compress_percentage;
+		struct frand_state *rs;
+
+		rs = get_buf_state(td);
 
 		if (perc) {
 			unsigned int seg = min_write;
@@ -1843,10 +1874,12 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write,
 			if (!seg)
 				seg = min_write;
 
-			fill_random_buf_percentage(&td->buf_state, buf,
-						perc, seg, max_bs);
-		} else
-			fill_random_buf(&td->buf_state, buf, max_bs);
+			fill_random_buf_percentage(rs, buf, perc, seg,max_bs);
+			save_buf_state(td, rs);
+		} else {
+			fill_random_buf(rs, buf, max_bs);
+			save_buf_state(td, rs);
+		}
 	} else
 		memset(buf, 0, max_bs);
 }
diff --git a/lib/rand.h b/lib/rand.h
index d62ebe5..8c35ab1 100644
--- a/lib/rand.h
+++ b/lib/rand.h
@@ -7,6 +7,14 @@ struct frand_state {
 	unsigned int s1, s2, s3;
 };
 
+static inline void frand_copy(struct frand_state *dst,
+			      struct frand_state *src)
+{
+	dst->s1 = src->s1;
+	dst->s2 = src->s2;
+	dst->s3 = src->s3;
+}
+
 static inline unsigned int __rand(struct frand_state *state)
 {
 #define TAUSWORTHE(s,a,b,c,d) ((s&c)<<d) ^ (((s <<a) ^ s)>>b)
diff --git a/options.c b/options.c
index ce95513..593f717 100644
--- a/options.c
+++ b/options.c
@@ -1050,6 +1050,16 @@ static int str_buffer_compress_cb(void *data, unsigned long long *il)
 	return 0;
 }
 
+static int str_dedupe_cb(void *data, unsigned long long *il)
+{
+	struct thread_data *td = data;
+
+	td->flags |= TD_F_COMPRESS;
+	td->o.dedupe_percentage = *il;
+	td->o.refill_buffers = 1;
+	return 0;
+}
+
 static int str_verify_pattern_cb(void *data, const char *input)
 {
 	struct thread_data *td = data;
@@ -3257,6 +3267,18 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.group	= FIO_OPT_G_IO_BUF,
 	},
 	{
+		.name	= "dedupe_percentage",
+		.lname	= "Dedupe percentage",
+		.type	= FIO_OPT_INT,
+		.cb	= str_dedupe_cb,
+		.maxval	= 100,
+		.minval	= 0,
+		.help	= "Percentage of buffers that are dedupable",
+		.interval = 1,
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_IO_BUF,
+	},
+	{
 		.name	= "clat_percentiles",
 		.lname	= "Completion latency percentiles",
 		.type	= FIO_OPT_BOOL,
diff --git a/server.h b/server.h
index cc4c5b4..1b131b9 100644
--- a/server.h
+++ b/server.h
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 35,
+	FIO_SERVER_VER			= 36,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,
diff --git a/thread_options.h b/thread_options.h
index e545a8f..a45d7b7 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -184,6 +184,7 @@ struct thread_options {
 	unsigned int buffer_pattern_bytes;
 	unsigned int compress_percentage;
 	unsigned int compress_chunk;
+	unsigned int dedupe_percentage;
 	unsigned int time_based;
 	unsigned int disable_lat;
 	unsigned int disable_clat;
@@ -403,8 +404,9 @@ struct thread_options_pack {
 	uint32_t scramble_buffers;
 	uint8_t buffer_pattern[MAX_PATTERN_SIZE];
 	uint32_t buffer_pattern_bytes;
-	unsigned int compress_percentage;
-	unsigned int compress_chunk;
+	uint32_t compress_percentage;
+	uint32_t compress_chunk;
+	uint32_t dedupe_percentage;
 	uint32_t time_based;
 	uint32_t disable_lat;
 	uint32_t disable_clat;