All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH 0/3] iolog: add version 3 with timestamp support
@ 2022-04-06 17:05 Mohamad Gebai
  2022-04-06 17:05 ` [PATCH 1/3] iolog: add version 3 to support timestamp-based replay Mohamad Gebai
                   ` (4 more replies)
  0 siblings, 5 replies; 8+ messages in thread
From: Mohamad Gebai @ 2022-04-06 17:05 UTC (permalink / raw)
  To: fio; +Cc: Mohamad Gebai

This set of patches introduces version 3 of the iolog replay. This
version adds support for timestamps on both iolog_read and iolog_write.
The 'wait' action is no longer supported with version 3, as the same
behavior can be achieved with timestamps, and it keeps the iolog file
easier to follow. This could also allow to potentially deprecate the
'wait' action eventially.

Mohamad Gebai (3):
  iolog: add version 3 to support timestamp-based replay
  iolog: add iolog_write for version 3
  iolog: update man page

 HOWTO.rst  |  29 +++++++++++++-
 blktrace.c |  17 ++------
 fio.h      |   4 +-
 iolog.c    | 114 +++++++++++++++++++++++++++++++++++++++++------------
 iolog.h    |   8 ++--
 5 files changed, 127 insertions(+), 45 deletions(-)

-- 
2.30.2


^ permalink raw reply	[flat|nested] 8+ messages in thread

* [PATCH 1/3] iolog: add version 3 to support timestamp-based replay
  2022-04-06 17:05 [PATCH 0/3] iolog: add version 3 with timestamp support Mohamad Gebai
@ 2022-04-06 17:05 ` Mohamad Gebai
  2022-04-06 17:05 ` [PATCH 2/3] iolog: add iolog_write for version 3 Mohamad Gebai
                   ` (3 subsequent siblings)
  4 siblings, 0 replies; 8+ messages in thread
From: Mohamad Gebai @ 2022-04-06 17:05 UTC (permalink / raw)
  To: fio; +Cc: Mohamad Gebai

 - Version 3 format looks as follows:

    timestamp filename action offset length

 - File actions must have a timestamp, including add.

 - 'wait' is not allowed with version 3 so we can leave all timing-related
 functions to timestamps.

Signed-off-by: Mohamad Gebai <mogeb@fb.com>
---
 blktrace.c | 17 ++--------
 fio.h      |  3 +-
 iolog.c    | 99 ++++++++++++++++++++++++++++++++++++++++++------------
 iolog.h    |  8 ++---
 4 files changed, 86 insertions(+), 41 deletions(-)

diff --git a/blktrace.c b/blktrace.c
index ead60130..619121c7 100644
--- a/blktrace.c
+++ b/blktrace.c
@@ -306,39 +306,28 @@ static bool handle_trace_flush(struct thread_data *td, struct blk_io_trace *t,
 }
 
 /*
  * We only care for queue traces, most of the others are side effects
  * due to internal workings of the block layer.
  */
 static bool queue_trace(struct thread_data *td, struct blk_io_trace *t,
 			 unsigned long *ios, unsigned long long *bs,
 			 struct file_cache *cache)
 {
-	unsigned long long *last_ttime = &td->io_log_blktrace_last_ttime;
+	unsigned long long *last_ttime = &td->io_log_last_ttime;
 	unsigned long long delay = 0;
 
 	if ((t->action & 0xffff) != __BLK_TA_QUEUE)
 		return false;
 
 	if (!(t->action & BLK_TC_ACT(BLK_TC_NOTIFY))) {
-		if (!*last_ttime || td->o.no_stall || t->time < *last_ttime)
-			delay = 0;
-		else if (td->o.replay_time_scale == 100)
-			delay = t->time - *last_ttime;
-		else {
-			double tmp = t->time - *last_ttime;
-			double scale;
-
-			scale = (double) 100.0 / (double) td->o.replay_time_scale;
-			tmp *= scale;
-			delay = tmp;
-		}
+		delay = delay_since_ttime(td, t->time);
 		*last_ttime = t->time;
 	}
 
 	t_bytes_align(&td->o, t);
 
 	if (t->action & BLK_TC_ACT(BLK_TC_NOTIFY))
 		return handle_trace_notify(t);
 	else if (t->action & BLK_TC_ACT(BLK_TC_DISCARD))
 		return handle_trace_discard(td, t, delay, ios, bs, cache);
 	else if (t->action & BLK_TC_ACT(BLK_TC_FLUSH))
@@ -415,21 +404,21 @@ static void depth_end(struct blk_io_trace *t, int *this_depth, int *depth)
 bool init_blktrace_read(struct thread_data *td, const char *filename, int need_swap)
 {
 	int old_state;
 
 	td->io_log_rfile = fopen(filename, "rb");
 	if (!td->io_log_rfile) {
 		td_verror(td, errno, "open blktrace file");
 		goto err;
 	}
 	td->io_log_blktrace_swap = need_swap;
-	td->io_log_blktrace_last_ttime = 0;
+	td->io_log_last_ttime = 0;
 	td->o.size = 0;
 
 	free_release_files(td);
 
 	old_state = td_bump_runstate(td, TD_SETTING_UP);
 
 	if (!read_blktrace(td)) {
 		goto err;
 	}
 
diff --git a/fio.h b/fio.h
index 776fb51f..8830ff34 100644
--- a/fio.h
+++ b/fio.h
@@ -424,24 +424,25 @@ struct thread_data {
 	struct flist_head io_hist_list;
 	unsigned long io_hist_len;
 
 	/*
 	 * For IO replaying
 	 */
 	struct flist_head io_log_list;
 	FILE *io_log_rfile;
 	unsigned int io_log_blktrace;
 	unsigned int io_log_blktrace_swap;
-	unsigned long long io_log_blktrace_last_ttime;
+	unsigned long long io_log_last_ttime;
 	unsigned int io_log_current;
 	unsigned int io_log_checkmark;
 	unsigned int io_log_highmark;
+	unsigned int io_log_version;
 	struct timespec io_log_highmark_time;
 
 	/*
 	 * For tracking/handling discards
 	 */
 	struct flist_head trim_list;
 	unsigned long trim_entries;
 
 	/*
 	 * for fileservice, how often to switch to a new file
diff --git a/iolog.c b/iolog.c
index 724ec1fe..be7eb007 100644
--- a/iolog.c
+++ b/iolog.c
@@ -24,20 +24,21 @@
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <arpa/inet.h>
 #include <sys/stat.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 
 static int iolog_flush(struct io_log *log);
 
 static const char iolog_ver2[] = "fio version 2 iolog";
+static const char iolog_ver3[] = "fio version 3 iolog";
 
 void queue_io_piece(struct thread_data *td, struct io_piece *ipo)
 {
 	flist_add_tail(&ipo->list, &td->io_log_list);
 	td->total_io_size += ipo->len;
 }
 
 void log_io_u(const struct thread_data *td, const struct io_u *io_u)
 {
 	if (!td->o.write_iolog_file)
@@ -109,63 +110,93 @@ static int ipo_special(struct thread_data *td, struct io_piece *ipo)
 	int ret;
 
 	/*
 	 * Not a special ipo
 	 */
 	if (ipo->ddir != DDIR_INVAL)
 		return 0;
 
 	f = td->files[ipo->fileno];
 
+	if (ipo->delay)
+		iolog_delay(td, ipo->delay);
+	if (fio_fill_issue_time(td)) {
+		struct timespec now;
+		fio_gettime(&now, NULL);
+		memcpy(&td->last_issue, &now, sizeof(now));
+	}
 	switch (ipo->file_action) {
 	case FIO_LOG_OPEN_FILE:
 		if (td->o.replay_redirect && fio_file_open(f)) {
 			dprint(FD_FILE, "iolog: ignoring re-open of file %s\n",
 					f->file_name);
 			break;
 		}
 		ret = td_io_open_file(td, f);
 		if (!ret)
 			break;
 		td_verror(td, ret, "iolog open file");
 		return -1;
 	case FIO_LOG_CLOSE_FILE:
 		td_io_close_file(td, f);
 		break;
 	case FIO_LOG_UNLINK_FILE:
 		td_io_unlink_file(td, f);
 		break;
+	case FIO_LOG_ADD_FILE:
+		/*
+		 * Nothing to do
+		 */
+		break;
 	default:
 		log_err("fio: bad file action %d\n", ipo->file_action);
 		break;
 	}
 
 	return 1;
 }
 
-static bool read_iolog2(struct thread_data *td);
+static bool read_iolog(struct thread_data *td);
+
+unsigned long long delay_since_ttime(const struct thread_data *td,
+	       unsigned long long time)
+{
+	double tmp;
+	double scale;
+	const unsigned long long *last_ttime = &td->io_log_last_ttime;
+
+	if (!*last_ttime || td->o.no_stall || time < *last_ttime)
+		return 0;
+	else if (td->o.replay_time_scale == 100)
+		return time - *last_ttime;
+
+
+	scale = (double) 100.0 / (double) td->o.replay_time_scale;
+	tmp = time - *last_ttime;
+	return tmp * scale;
+}
 
 int read_iolog_get(struct thread_data *td, struct io_u *io_u)
 {
 	struct io_piece *ipo;
 	unsigned long elapsed;
 
 	while (!flist_empty(&td->io_log_list)) {
 		int ret;
 
 		if (td->o.read_iolog_chunked) {
 			if (td->io_log_checkmark == td->io_log_current) {
 				if (td->io_log_blktrace) {
 					if (!read_blktrace(td))
 						return 1;
 				} else {
-					if (!read_iolog2(td))
+					if (!read_iolog(td))
 						return 1;
 				}
 			}
 			td->io_log_current--;
 		}
 		ipo = flist_first_entry(&td->io_log_list, struct io_piece, list);
 		flist_del(&ipo->list);
 		remove_trim_entry(td, ipo);
 
 		ret = ipo_special(td, ipo);
@@ -381,28 +412,34 @@ int64_t iolog_items_to_fetch(struct thread_data *td)
 	} else
 		items_to_fetch = 0;
 
 	td->io_log_highmark = td->io_log_current + items_to_fetch;
 	td->io_log_checkmark = (td->io_log_highmark + 1) / 2;
 	fio_gettime(&td->io_log_highmark_time, NULL);
 
 	return items_to_fetch;
 }
 
+#define io_act(_td, _r) (((_td)->io_log_version == 3 && (r) == 5) || \
+					((_td)->io_log_version == 2 && (r) == 4))
+#define file_act(_td, _r) (((_td)->io_log_version == 3 && (r) == 3) || \
+					((_td)->io_log_version == 2 && (r) == 2))
+
 /*
- * Read version 2 iolog data. It is enhanced to include per-file logging,
+ * Read version 2 and 3 iolog data. It is enhanced to include per-file logging,
  * syncs, etc.
  */
-static bool read_iolog2(struct thread_data *td)
+static bool read_iolog(struct thread_data *td)
 {
 	unsigned long long offset;
 	unsigned int bytes;
+	unsigned long long delay = 0;
 	int reads, writes, waits, fileno = 0, file_action = 0; /* stupid gcc */
 	char *rfname, *fname, *act;
 	char *str, *p;
 	enum fio_ddir rw;
 	bool realloc = false;
 	int64_t items_to_fetch = 0;
 	int syncs;
 
 	if (td->o.read_iolog_chunked) {
 		items_to_fetch = iolog_items_to_fetch(td);
@@ -414,29 +451,43 @@ static bool read_iolog2(struct thread_data *td)
 	 * Read in the read iolog and store it, reuse the infrastructure
 	 * for doing verifications.
 	 */
 	str = malloc(4096);
 	rfname = fname = malloc(256+16);
 	act = malloc(256+16);
 
 	syncs = reads = writes = waits = 0;
 	while ((p = fgets(str, 4096, td->io_log_rfile)) != NULL) {
 		struct io_piece *ipo;
-		int r;
+		int r = 0;
+		unsigned long long ttime;
 
-		r = sscanf(p, "%256s %256s %llu %u", rfname, act, &offset,
-									&bytes);
+		if (td->io_log_version == 3) {
+			r = sscanf(p, "%llu %256s %256s %llu %u", &ttime, rfname, act,
+							&offset, &bytes);
+			delay = delay_since_ttime(td, ttime);
+			td->io_log_last_ttime = ttime;
+			/*
+			 * "wait" is not allowed for version 3
+			 */
+			if (!strcmp(act, "wait")) {
+				log_err("iolog: ignoring wait command with"
+					" version 3 for file %s\n", fname);
+				continue;
+			}
+		} else /* version 2 */
+			r = sscanf(p, "%256s %256s %llu %u", rfname, act, &offset, &bytes);
 
 		if (td->o.replay_redirect)
 			fname = td->o.replay_redirect;
 
-		if (r == 4) {
+		if (io_act(td, r)) {
 			/*
 			 * Check action first
 			 */
 			if (!strcmp(act, "wait"))
 				rw = DDIR_WAIT;
 			else if (!strcmp(act, "read"))
 				rw = DDIR_READ;
 			else if (!strcmp(act, "write"))
 				rw = DDIR_WRITE;
 			else if (!strcmp(act, "sync"))
@@ -444,45 +495,44 @@ static bool read_iolog2(struct thread_data *td)
 			else if (!strcmp(act, "datasync"))
 				rw = DDIR_DATASYNC;
 			else if (!strcmp(act, "trim"))
 				rw = DDIR_TRIM;
 			else {
 				log_err("fio: bad iolog file action: %s\n",
 									act);
 				continue;
 			}
 			fileno = get_fileno(td, fname);
-		} else if (r == 2) {
+		} else if (file_act(td, r)) {
 			rw = DDIR_INVAL;
 			if (!strcmp(act, "add")) {
 				if (td->o.replay_redirect &&
 				    get_fileno(td, fname) != -1) {
 					dprint(FD_FILE, "iolog: ignoring"
 						" re-add of file %s\n", fname);
 				} else {
 					fileno = add_file(td, fname, td->subjob_number, 1);
 					file_action = FIO_LOG_ADD_FILE;
 				}
-				continue;
 			} else if (!strcmp(act, "open")) {
 				fileno = get_fileno(td, fname);
 				file_action = FIO_LOG_OPEN_FILE;
 			} else if (!strcmp(act, "close")) {
 				fileno = get_fileno(td, fname);
 				file_action = FIO_LOG_CLOSE_FILE;
 			} else {
 				log_err("fio: bad iolog file action: %s\n",
 									act);
 				continue;
 			}
 		} else {
-			log_err("bad iolog2: %s\n", p);
+			log_err("bad iolog%d: %s\n", td->io_log_version, p);
 			continue;
 		}
 
 		if (rw == DDIR_READ)
 			reads++;
 		else if (rw == DDIR_WRITE) {
 			/*
 			 * Don't add a write for ro mode
 			 */
 			if (read_only)
@@ -499,20 +549,22 @@ static bool read_iolog2(struct thread_data *td)
 			log_err("bad ddir: %d\n", rw);
 			continue;
 		}
 
 		/*
 		 * Make note of file
 		 */
 		ipo = calloc(1, sizeof(*ipo));
 		init_ipo(ipo);
 		ipo->ddir = rw;
+		if (td->io_log_version == 3)
+			ipo->delay = delay;
 		if (rw == DDIR_WAIT) {
 			ipo->delay = offset;
 		} else {
 			if (td->o.replay_scale)
 				ipo->offset = offset / td->o.replay_scale;
 			else
 				ipo->offset = offset;
 			ipo_bytes_align(td->o.replay_align, ipo);
 
 			ipo->len = bytes;
@@ -643,32 +695,36 @@ static bool init_iolog_read(struct thread_data *td, char *fname)
 
 	p = fgets(buffer, sizeof(buffer), f);
 	if (!p) {
 		td_verror(td, errno, "iolog read");
 		log_err("fio: unable to read iolog\n");
 		fclose(f);
 		return false;
 	}
 
 	/*
-	 * version 2 of the iolog stores a specific string as the
+	 * versions 2 and 3 of the iolog store a specific string as the
 	 * first line, check for that
 	 */
-	if (!strncmp(iolog_ver2, buffer, strlen(iolog_ver2))) {
-		free_release_files(td);
-		td->io_log_rfile = f;
-		return read_iolog2(td);
+	if (!strncmp(iolog_ver2, buffer, strlen(iolog_ver2)))
+		td->io_log_version = 2;
+	else if (!strncmp(iolog_ver3, buffer, strlen(iolog_ver3)))
+		td->io_log_version = 3;
+	else {
+		log_err("fio: iolog version 1 is no longer supported\n");
+		fclose(f);
+		return false;
 	}
 
-	log_err("fio: iolog version 1 is no longer supported\n");
-	fclose(f);
-	return false;
+	free_release_files(td);
+	td->io_log_rfile = f;
+	return read_iolog(td);
 }
 
 /*
  * Set up a log for storing io patterns.
  */
 static bool init_iolog_write(struct thread_data *td)
 {
 	struct fio_file *ff;
 	FILE *f;
 	unsigned int i;
diff --git a/iolog.h b/iolog.h
index a3986309..62cbd1b0 100644
--- a/iolog.h
+++ b/iolog.h
@@ -220,24 +220,22 @@ struct io_piece {
 	struct flist_head trim_list;
 	union {
 		int fileno;
 		struct fio_file *file;
 	};
 	unsigned long long offset;
 	unsigned short numberio;
 	unsigned long len;
 	unsigned int flags;
 	enum fio_ddir ddir;
-	union {
-		unsigned long delay;
-		unsigned int file_action;
-	};
+	unsigned long delay;
+	unsigned int file_action;
 };
 
 /*
  * Log exports
  */
 enum file_log_act {
 	FIO_LOG_ADD_FILE,
 	FIO_LOG_OPEN_FILE,
 	FIO_LOG_CLOSE_FILE,
 	FIO_LOG_UNLINK_FILE,
@@ -252,20 +250,22 @@ extern void log_io_piece(struct thread_data *, struct io_u *);
 extern void unlog_io_piece(struct thread_data *, struct io_u *);
 extern void trim_io_piece(const struct io_u *);
 extern void queue_io_piece(struct thread_data *, struct io_piece *);
 extern void prune_io_piece_log(struct thread_data *);
 extern void write_iolog_close(struct thread_data *);
 int64_t iolog_items_to_fetch(struct thread_data *td);
 extern int iolog_compress_init(struct thread_data *, struct sk_out *);
 extern void iolog_compress_exit(struct thread_data *);
 extern size_t log_chunk_sizes(struct io_log *);
 extern int init_io_u_buffers(struct thread_data *);
+extern unsigned long long delay_since_ttime(const struct thread_data *,
+					     unsigned long long);
 
 #ifdef CONFIG_ZLIB
 extern int iolog_file_inflate(const char *);
 #endif
 
 /*
  * Logging
  */
 struct log_params {
 	struct thread_data *td;
-- 
2.30.2


^ permalink raw reply related	[flat|nested] 8+ messages in thread

* [PATCH 2/3] iolog: add iolog_write for version 3
  2022-04-06 17:05 [PATCH 0/3] iolog: add version 3 with timestamp support Mohamad Gebai
  2022-04-06 17:05 ` [PATCH 1/3] iolog: add version 3 to support timestamp-based replay Mohamad Gebai
@ 2022-04-06 17:05 ` Mohamad Gebai
  2022-04-06 17:05 ` [PATCH 3/3] iolog: update man page Mohamad Gebai
                   ` (2 subsequent siblings)
  4 siblings, 0 replies; 8+ messages in thread
From: Mohamad Gebai @ 2022-04-06 17:05 UTC (permalink / raw)
  To: fio; +Cc: Mohamad Gebai

Signed-off-by: Mohamad Gebai <mogeb@fb.com>
---
 fio.h   |  1 +
 iolog.c | 15 ++++++++++++---
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/fio.h b/fio.h
index 8830ff34..de7eca79 100644
--- a/fio.h
+++ b/fio.h
@@ -425,20 +425,21 @@ struct thread_data {
 	unsigned long io_hist_len;
 
 	/*
 	 * For IO replaying
 	 */
 	struct flist_head io_log_list;
 	FILE *io_log_rfile;
 	unsigned int io_log_blktrace;
 	unsigned int io_log_blktrace_swap;
 	unsigned long long io_log_last_ttime;
+	struct timespec io_log_start_time;
 	unsigned int io_log_current;
 	unsigned int io_log_checkmark;
 	unsigned int io_log_highmark;
 	unsigned int io_log_version;
 	struct timespec io_log_highmark_time;
 
 	/*
 	 * For tracking/handling discards
 	 */
 	struct flist_head trim_list;
diff --git a/iolog.c b/iolog.c
index be7eb007..b7ab08be 100644
--- a/iolog.c
+++ b/iolog.c
@@ -34,46 +34,54 @@ static const char iolog_ver2[] = "fio version 2 iolog";
 static const char iolog_ver3[] = "fio version 3 iolog";
 
 void queue_io_piece(struct thread_data *td, struct io_piece *ipo)
 {
 	flist_add_tail(&ipo->list, &td->io_log_list);
 	td->total_io_size += ipo->len;
 }
 
 void log_io_u(const struct thread_data *td, const struct io_u *io_u)
 {
+	struct timespec now;
+
 	if (!td->o.write_iolog_file)
 		return;
 
-	fprintf(td->iolog_f, "%s %s %llu %llu\n", io_u->file->file_name,
+	fio_gettime(&now, NULL);
+	fprintf(td->iolog_f, "%lu %s %s %llu %llu\n", utime_since_now(&td->io_log_start_time),
+						io_u->file->file_name,
 						io_ddir_name(io_u->ddir),
 						io_u->offset, io_u->buflen);
+
 }
 
 void log_file(struct thread_data *td, struct fio_file *f,
 	      enum file_log_act what)
 {
 	const char *act[] = { "add", "open", "close" };
+	struct timespec now;
 
 	assert(what < 3);
 
 	if (!td->o.write_iolog_file)
 		return;
 
 
 	/*
 	 * this happens on the pre-open/close done before the job starts
 	 */
 	if (!td->iolog_f)
 		return;
 
-	fprintf(td->iolog_f, "%s %s\n", f->file_name, act[what]);
+	fio_gettime(&now, NULL);
+	fprintf(td->iolog_f, "%lu %s %s\n", utime_since_now(&td->io_log_start_time),
+						f->file_name, act[what]);
 }
 
 static void iolog_delay(struct thread_data *td, unsigned long delay)
 {
 	uint64_t usec = utime_since_now(&td->last_issue);
 	unsigned long orig_delay = delay;
 	uint64_t this_delay;
 	struct timespec ts;
 
 	if (delay < td->time_offset) {
@@ -734,25 +742,26 @@ static bool init_iolog_write(struct thread_data *td)
 		perror("fopen write iolog");
 		return false;
 	}
 
 	/*
 	 * That's it for writing, setup a log buffer and we're done.
 	  */
 	td->iolog_f = f;
 	td->iolog_buf = malloc(8192);
 	setvbuf(f, td->iolog_buf, _IOFBF, 8192);
+	fio_gettime(&td->io_log_start_time, NULL);
 
 	/*
 	 * write our version line
 	 */
-	if (fprintf(f, "%s\n", iolog_ver2) < 0) {
+	if (fprintf(f, "%s\n", iolog_ver3) < 0) {
 		perror("iolog init\n");
 		return false;
 	}
 
 	/*
 	 * add all known files
 	 */
 	for_each_file(td, ff, i)
 		log_file(td, ff, FIO_LOG_ADD_FILE);
 
-- 
2.30.2


^ permalink raw reply related	[flat|nested] 8+ messages in thread

* [PATCH 3/3] iolog: update man page
  2022-04-06 17:05 [PATCH 0/3] iolog: add version 3 with timestamp support Mohamad Gebai
  2022-04-06 17:05 ` [PATCH 1/3] iolog: add version 3 to support timestamp-based replay Mohamad Gebai
  2022-04-06 17:05 ` [PATCH 2/3] iolog: add iolog_write for version 3 Mohamad Gebai
@ 2022-04-06 17:05 ` Mohamad Gebai
  2022-04-06 23:50 ` [PATCH 0/3] iolog: add version 3 with timestamp support Jens Axboe
  2022-04-07  1:37 ` Jens Axboe
  4 siblings, 0 replies; 8+ messages in thread
From: Mohamad Gebai @ 2022-04-06 17:05 UTC (permalink / raw)
  To: fio; +Cc: Mohamad Gebai

Signed-off-by: Mohamad Gebai <mogeb@fb.com>
---
 HOWTO.rst | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/HOWTO.rst b/HOWTO.rst
index 0978879c..fffab1ac 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -4391,33 +4391,60 @@ The `filename` is given as an absolute path. The `action` can be one of these:
 The file I/O action format::
 
     filename action offset length
 
 The `filename` is given as an absolute path, and has to have been added and
 opened before it can be used with this format. The `offset` and `length` are
 given in bytes. The `action` can be one of these:
 
 **wait**
 	   Wait for `offset` microseconds. Everything below 100 is discarded.
-	   The time is relative to the previous `wait` statement.
+	   The time is relative to the previous `wait` statement. Note that
+	   action `wait` is not allowed as of version 3, as the same behavior
+	   can be achieved using timestamps.
 **read**
 	   Read `length` bytes beginning from `offset`.
 **write**
 	   Write `length` bytes beginning from `offset`.
 **sync**
 	   :manpage:`fsync(2)` the file.
 **datasync**
 	   :manpage:`fdatasync(2)` the file.
 **trim**
 	   Trim the given file from the given `offset` for `length` bytes.
 
 
+Trace file format v3
+~~~~~~~~~~~~~~~~~~~~
+
+The third version of the trace file format was added in fio version 3.30.  It
+forces each action to have a timestamp associated with it.
+
+The first line of the trace file has to be::
+
+    fio version 3 iolog
+
+Following this can be lines in two different formats, which are described below.
+
+The file management format::
+
+    timestamp filename action
+
+The file I/O action format::
+
+    timestamp filename action offset length
+
+The `timestamp` is relative to the beginning of the run (ie starts at 0). The
+`filename`, `action`, `offset` and `length`  are identical to version 2, except
+that version 3 does not allow the `wait` action.
+
+
 I/O Replay - Merging Traces
 ---------------------------
 
 Colocation is a common practice used to get the most out of a machine.
 Knowing which workloads play nicely with each other and which ones don't is
 a much harder task. While fio can replay workloads concurrently via multiple
 jobs, it leaves some variability up to the scheduler making results harder to
 reproduce. Merging is a way to make the order of events consistent.
 
 Merging is integrated into I/O replay and done when a
-- 
2.30.2


^ permalink raw reply related	[flat|nested] 8+ messages in thread

* Re: [PATCH 0/3] iolog: add version 3 with timestamp support
  2022-04-06 17:05 [PATCH 0/3] iolog: add version 3 with timestamp support Mohamad Gebai
                   ` (2 preceding siblings ...)
  2022-04-06 17:05 ` [PATCH 3/3] iolog: update man page Mohamad Gebai
@ 2022-04-06 23:50 ` Jens Axboe
  2022-04-07  0:59   ` Damien Le Moal
  2022-04-07  1:37 ` Jens Axboe
  4 siblings, 1 reply; 8+ messages in thread
From: Jens Axboe @ 2022-04-06 23:50 UTC (permalink / raw)
  To: Mohamad Gebai, fio

On 4/6/22 11:05 AM, Mohamad Gebai wrote:
> This set of patches introduces version 3 of the iolog replay. This
> version adds support for timestamps on both iolog_read and iolog_write.
> The 'wait' action is no longer supported with version 3, as the same
> behavior can be achieved with timestamps, and it keeps the iolog file
> easier to follow. This could also allow to potentially deprecate the
> 'wait' action eventially.

Not sure if it's just for me, but patch 1 isn't there. I do however see
it on the mailing list, so maybe it is just me?

-- 
Jens Axboe


^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH 0/3] iolog: add version 3 with timestamp support
  2022-04-06 23:50 ` [PATCH 0/3] iolog: add version 3 with timestamp support Jens Axboe
@ 2022-04-07  0:59   ` Damien Le Moal
  2022-04-07  1:26     ` Jens Axboe
  0 siblings, 1 reply; 8+ messages in thread
From: Damien Le Moal @ 2022-04-07  0:59 UTC (permalink / raw)
  To: Jens Axboe, Mohamad Gebai, fio

On 4/7/22 08:50, Jens Axboe wrote:
> On 4/6/22 11:05 AM, Mohamad Gebai wrote:
>> This set of patches introduces version 3 of the iolog replay. This
>> version adds support for timestamps on both iolog_read and iolog_write.
>> The 'wait' action is no longer supported with version 3, as the same
>> behavior can be achieved with timestamps, and it keeps the iolog file
>> easier to follow. This could also allow to potentially deprecate the
>> 'wait' action eventially.
> 
> Not sure if it's just for me, but patch 1 isn't there. I do however see
> it on the mailing list, so maybe it is just me?

I did receive it from the list... Junk folder ?
vger seems to have bad days recently. Lots of weird behavior.



-- 
Damien Le Moal
Western Digital Research

^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH 0/3] iolog: add version 3 with timestamp support
  2022-04-07  0:59   ` Damien Le Moal
@ 2022-04-07  1:26     ` Jens Axboe
  0 siblings, 0 replies; 8+ messages in thread
From: Jens Axboe @ 2022-04-07  1:26 UTC (permalink / raw)
  To: Damien Le Moal, Mohamad Gebai, fio

On 4/6/22 6:59 PM, Damien Le Moal wrote:
> On 4/7/22 08:50, Jens Axboe wrote:
>> On 4/6/22 11:05 AM, Mohamad Gebai wrote:
>>> This set of patches introduces version 3 of the iolog replay. This
>>> version adds support for timestamps on both iolog_read and iolog_write.
>>> The 'wait' action is no longer supported with version 3, as the same
>>> behavior can be achieved with timestamps, and it keeps the iolog file
>>> easier to follow. This could also allow to potentially deprecate the
>>> 'wait' action eventially.
>>
>> Not sure if it's just for me, but patch 1 isn't there. I do however see
>> it on the mailing list, so maybe it is just me?
> 
> I did receive it from the list... Junk folder ?

Indeed, it was in there.

> vger seems to have bad days recently. Lots of weird behavior.

Yep vger seems like a shit show these days, it's really messing with my
workflow.

-- 
Jens Axboe


^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH 0/3] iolog: add version 3 with timestamp support
  2022-04-06 17:05 [PATCH 0/3] iolog: add version 3 with timestamp support Mohamad Gebai
                   ` (3 preceding siblings ...)
  2022-04-06 23:50 ` [PATCH 0/3] iolog: add version 3 with timestamp support Jens Axboe
@ 2022-04-07  1:37 ` Jens Axboe
  4 siblings, 0 replies; 8+ messages in thread
From: Jens Axboe @ 2022-04-07  1:37 UTC (permalink / raw)
  To: Mohamad Gebai, fio

On 4/6/22 11:05 AM, Mohamad Gebai wrote:
> This set of patches introduces version 3 of the iolog replay. This
> version adds support for timestamps on both iolog_read and iolog_write.
> The 'wait' action is no longer supported with version 3, as the same
> behavior can be achieved with timestamps, and it keeps the iolog file
> easier to follow. This could also allow to potentially deprecate the
> 'wait' action eventially.

Change looks good to me, but it would be nice with some actual commit
messages. #1 has one, but it's formatted pretty strange. 2 and 3 don't
have any. At the very least, there should be a basic description of
"why" the change is made in the commit message.

Can you send a v2 with proper commit messages, and #1 formatted that
isn't a strange list?

Thanks!

-- 
Jens Axboe


^ permalink raw reply	[flat|nested] 8+ messages in thread

end of thread, other threads:[~2022-04-07  1:42 UTC | newest]

Thread overview: 8+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-04-06 17:05 [PATCH 0/3] iolog: add version 3 with timestamp support Mohamad Gebai
2022-04-06 17:05 ` [PATCH 1/3] iolog: add version 3 to support timestamp-based replay Mohamad Gebai
2022-04-06 17:05 ` [PATCH 2/3] iolog: add iolog_write for version 3 Mohamad Gebai
2022-04-06 17:05 ` [PATCH 3/3] iolog: update man page Mohamad Gebai
2022-04-06 23:50 ` [PATCH 0/3] iolog: add version 3 with timestamp support Jens Axboe
2022-04-07  0:59   ` Damien Le Moal
2022-04-07  1:26     ` Jens Axboe
2022-04-07  1:37 ` Jens Axboe

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.