All of lore.kernel.org
 help / color / mirror / Atom feed
* Recent changes (master)
@ 2021-09-17 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2021-09-17 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 2686fc2279c0e1272a48657dc62c16059a672da9:

  t/io_uring: add switch -O for O_DIRECT vs buffered (2021-09-15 06:51:01 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to f3057d268d98aeb638b659a284d087fb0c2f654c:

  t/io_uring: fix bandwidth calculation (2021-09-16 11:41:06 -0600)

----------------------------------------------------------------
Erwan Velu (1):
      t/io_uring: Reporting bandwidth

Jens Axboe (2):
      Merge branch 'bwps' of https://github.com/ErwanAliasr1/fio
      t/io_uring: fix bandwidth calculation

 t/io_uring.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/t/io_uring.c b/t/io_uring.c
index 0acbf0b4..1adb8789 100644
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -727,6 +727,7 @@ int main(int argc, char *argv[])
 		unsigned long this_reap = 0;
 		unsigned long this_call = 0;
 		unsigned long rpc = 0, ipc = 0;
+		unsigned long iops, bw;
 
 		sleep(1);
 		for (j = 0; j < nthreads; j++) {
@@ -740,8 +741,13 @@ int main(int argc, char *argv[])
 		} else
 			rpc = ipc = -1;
 		file_depths(fdepths);
-		printf("IOPS=%lu, IOS/call=%ld/%ld, inflight=(%s)\n",
-				this_done - done, rpc, ipc, fdepths);
+		iops = this_done - done;
+		if (bs > 1048576)
+			bw = iops * (bs / 1048576);
+		else
+			bw = iops / (1048576 / bs);
+		printf("IOPS=%lu, BW=%luMiB/s, IOS/call=%ld/%ld, inflight=(%s)\n",
+				iops, bw, rpc, ipc, fdepths);
 		done = this_done;
 		calls = this_call;
 		reap = this_reap;

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-04-20 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-04-20 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 420415dd1180c14ec0f55f65a05e57ea1fd85f9f:

  fio: ioengine flag cleanup (2024-04-18 12:36:01 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to c948ee34afde7eda14adf82512772b03f6fb1d69:

  Merge branch 'master' of https://github.com/celestinechen/fio (2024-04-19 13:51:05 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'master' of https://github.com/celestinechen/fio

celestinechen (1):
      FIO with fsync option issues more DDIR_SYNC commands than expected

 io_u.c      | 2 --
 ioengines.c | 4 ++++
 2 files changed, 4 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/io_u.c b/io_u.c
index 83895893..a499ff07 100644
--- a/io_u.c
+++ b/io_u.c
@@ -2113,7 +2113,6 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
 	if (ddir_sync(ddir)) {
 		if (io_u->error)
 			goto error;
-		td->last_was_sync = true;
 		if (f) {
 			f->first_write = -1ULL;
 			f->last_write = -1ULL;
@@ -2123,7 +2122,6 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
 		return;
 	}
 
-	td->last_was_sync = false;
 	td->last_ddir = ddir;
 
 	if (!io_u->error && ddir_rw(ddir)) {
diff --git a/ioengines.c b/ioengines.c
index 87cc2286..6b81dc77 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -436,6 +436,8 @@ enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u)
 			io_u_mark_depth(td, 1);
 			td->ts.total_io_u[io_u->ddir]++;
 		}
+
+		td->last_was_sync = ddir_sync(io_u->ddir);
 	} else if (ret == FIO_Q_QUEUED) {
 		td->io_u_queued++;
 
@@ -445,6 +447,8 @@ enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u)
 
 		if (td->io_u_queued >= td->o.iodepth_batch)
 			td_io_commit(td);
+
+		td->last_was_sync = ddir_sync(io_u->ddir);
 	}
 
 	if (!td_ioengine_flagged(td, FIO_SYNCIO) &&

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-04-19 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-04-19 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit e0bb44a5b2a67695f0b940772c70f678b323ec54:

  t/io_uring: only calculate per-file depth if we have files (2024-04-17 16:34:47 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 420415dd1180c14ec0f55f65a05e57ea1fd85f9f:

  fio: ioengine flag cleanup (2024-04-18 12:36:01 -0400)

----------------------------------------------------------------
Chana-Zaks-wdc (2):
      Don break too early in readwrite mode
      test: add test for readwrite issue

Vincent Fu (3):
      fio: use thread flag count for ioengine flag shift
      fio: remove compile time assertion
      fio: ioengine flag cleanup

 backend.c                 |  5 ++++
 fio.c                     |  2 --
 fio.h                     |  4 +--
 io_u.c                    | 11 +++++++
 ioengines.h               | 73 ++++++++++++++++++++++++++++++-----------------
 libfio.c                  |  1 +
 t/jobs/t0032-43063a1c.fio | 12 ++++++++
 7 files changed, 78 insertions(+), 30 deletions(-)
 create mode 100644 t/jobs/t0032-43063a1c.fio

---

Diff of recent changes:

diff --git a/backend.c b/backend.c
index fb7dc68a..fe03eab3 100644
--- a/backend.c
+++ b/backend.c
@@ -977,6 +977,11 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)
 	*/
 	if (td_write(td) && td_random(td) && td->o.norandommap)
 		total_bytes = max(total_bytes, (uint64_t) td->o.io_size);
+
+	/* Don't break too early if io_size > size */
+	if (td_rw(td) && !td_random(td))
+		total_bytes = max(total_bytes, (uint64_t)td->o.io_size);
+
 	/*
 	 * If verify_backlog is enabled, we'll run the verify in this
 	 * handler as well. For that case, we may need up to twice the
diff --git a/fio.c b/fio.c
index f19db1be..3d6ce597 100644
--- a/fio.c
+++ b/fio.c
@@ -27,8 +27,6 @@ int main(int argc, char *argv[], char *envp[])
 {
 	int ret = 1;
 
-	compiletime_assert(TD_NR <= TD_ENG_FLAG_SHIFT, "TD_ENG_FLAG_SHIFT");
-
 	if (initialize_fio(envp))
 		return 1;
 
diff --git a/fio.h b/fio.h
index fc3e3ece..7d9927a0 100644
--- a/fio.h
+++ b/fio.h
@@ -706,8 +706,8 @@ enum {
 	TD_NR,
 };
 
-#define TD_ENG_FLAG_SHIFT	18
-#define TD_ENG_FLAG_MASK	((1ULL << 18) - 1)
+#define TD_ENG_FLAG_SHIFT	(__TD_F_LAST)
+#define TD_ENG_FLAG_MASK	((1ULL << (__TD_F_LAST)) - 1)
 
 static inline void td_set_ioengine_flags(struct thread_data *td)
 {
diff --git a/io_u.c b/io_u.c
index 09e5f15a..83895893 100644
--- a/io_u.c
+++ b/io_u.c
@@ -360,6 +360,17 @@ static int get_next_seq_offset(struct thread_data *td, struct fio_file *f,
 		loop_cache_invalidate(td, f);
 	}
 
+	/*
+	 * If we reach the end for a rw-io-size based run, reset us back to 0
+	 * and invalidate the cache, if we need to.
+	 */
+	if (td_rw(td) && o->io_size > o->size) {
+		if (f->last_pos[ddir] >= f->io_size + get_start_offset(td, f)) {
+			f->last_pos[ddir] = f->file_offset;
+			loop_cache_invalidate(td, f);
+		}
+        }
+
 	if (f->last_pos[ddir] < f->real_file_size) {
 		uint64_t pos;
 
diff --git a/ioengines.h b/ioengines.h
index e43650f7..4fe9bb98 100644
--- a/ioengines.h
+++ b/ioengines.h
@@ -72,33 +72,54 @@ struct ioengine_ops {
 	struct fio_option *options;
 };
 
-enum fio_ioengine_flags {
-	FIO_SYNCIO	= 1 << 0,	/* io engine has synchronous ->queue */
-	FIO_RAWIO	= 1 << 1,	/* some sort of direct/raw io */
-	FIO_DISKLESSIO	= 1 << 2,	/* no disk involved */
-	FIO_NOEXTEND	= 1 << 3,	/* engine can't extend file */
-	FIO_NODISKUTIL  = 1 << 4,	/* diskutil can't handle filename */
-	FIO_UNIDIR	= 1 << 5,	/* engine is uni-directional */
-	FIO_NOIO	= 1 << 6,	/* thread does only pseudo IO */
-	FIO_PIPEIO	= 1 << 7,	/* input/output no seekable */
-	FIO_BARRIER	= 1 << 8,	/* engine supports barriers */
-	FIO_MEMALIGN	= 1 << 9,	/* engine wants aligned memory */
-	FIO_BIT_BASED	= 1 << 10,	/* engine uses a bit base (e.g. uses Kbit as opposed to KB) */
-	FIO_FAKEIO	= 1 << 11,	/* engine pretends to do IO */
-	FIO_NOSTATS	= 1 << 12,	/* don't do IO stats */
-	FIO_NOFILEHASH	= 1 << 13,	/* doesn't hash the files for lookup later. */
-	FIO_ASYNCIO_SYNC_TRIM
-			= 1 << 14,	/* io engine has async ->queue except for trim */
-	FIO_NO_OFFLOAD	= 1 << 15,	/* no async offload */
-	FIO_ASYNCIO_SETS_ISSUE_TIME
-			= 1 << 16,	/* async ioengine with commit function that sets issue_time */
-	FIO_SKIPPABLE_IOMEM_ALLOC
-			= 1 << 17,	/* skip iomem_alloc & iomem_free if job sets mem/iomem */
-	FIO_RO_NEEDS_RW_OPEN
-			= 1 << 18,	/* open files in rw mode even if we have a read job; only
+enum {
+	__FIO_SYNCIO = 0,		/* io engine has synchronous ->queue */
+	__FIO_RAWIO,			/* some sort of direct/raw io */
+	__FIO_DISKLESSIO,		/* no disk involved */
+	__FIO_NOEXTEND,			/* engine can't extend file */
+	__FIO_NODISKUTIL,		/* diskutil can't handle filename */
+	__FIO_UNIDIR,			/* engine is uni-directional */
+	__FIO_NOIO,			/* thread does only pseudo IO */
+	__FIO_PIPEIO,			/* input/output no seekable */
+	__FIO_BARRIER,			/* engine supports barriers */
+	__FIO_MEMALIGN,			/* engine wants aligned memory */
+	__FIO_BIT_BASED,		/* engine uses a bit base (e.g. uses Kbit as opposed to
+					   KB) */
+	__FIO_FAKEIO,			/* engine pretends to do IO */
+	__FIO_NOSTATS,			/* don't do IO stats */
+	__FIO_NOFILEHASH,		/* doesn't hash the files for lookup later. */
+	__FIO_ASYNCIO_SYNC_TRIM,	/* io engine has async ->queue except for trim */
+	__FIO_NO_OFFLOAD,		/* no async offload */
+	__FIO_ASYNCIO_SETS_ISSUE_TIME,	/* async ioengine with commit function that sets
+					   issue_time */
+	__FIO_SKIPPABLE_IOMEM_ALLOC,	/* skip iomem_alloc & iomem_free if job sets mem/iomem */
+	__FIO_RO_NEEDS_RW_OPEN,		/* open files in rw mode even if we have a read job; only
 					   affects ioengines using generic_open_file */
-	FIO_MULTI_RANGE_TRIM
-			= 1 << 19,	/* ioengine supports trim with more than one range */
+	__FIO_MULTI_RANGE_TRIM,		/* ioengine supports trim with more than one range */
+	__FIO_IOENGINE_F_LAST,		/* not a real bit; used to count number of bits */
+};
+
+enum fio_ioengine_flags {
+	FIO_SYNCIO			= 1 << __FIO_SYNCIO,
+	FIO_RAWIO			= 1 << __FIO_RAWIO,
+	FIO_DISKLESSIO			= 1 << __FIO_DISKLESSIO,
+	FIO_NOEXTEND			= 1 << __FIO_NOEXTEND,
+	FIO_NODISKUTIL  		= 1 << __FIO_NODISKUTIL,
+	FIO_UNIDIR			= 1 << __FIO_UNIDIR,
+	FIO_NOIO			= 1 << __FIO_NOIO,
+	FIO_PIPEIO			= 1 << __FIO_PIPEIO,
+	FIO_BARRIER			= 1 << __FIO_BARRIER,
+	FIO_MEMALIGN			= 1 << __FIO_MEMALIGN,
+	FIO_BIT_BASED			= 1 << __FIO_BIT_BASED,
+	FIO_FAKEIO			= 1 << __FIO_FAKEIO,
+	FIO_NOSTATS			= 1 << __FIO_NOSTATS,
+	FIO_NOFILEHASH			= 1 << __FIO_NOFILEHASH,
+	FIO_ASYNCIO_SYNC_TRIM		= 1 << __FIO_ASYNCIO_SYNC_TRIM,
+	FIO_NO_OFFLOAD			= 1 << __FIO_NO_OFFLOAD,
+	FIO_ASYNCIO_SETS_ISSUE_TIME	= 1 << __FIO_ASYNCIO_SETS_ISSUE_TIME,
+	FIO_SKIPPABLE_IOMEM_ALLOC	= 1 << __FIO_SKIPPABLE_IOMEM_ALLOC,
+	FIO_RO_NEEDS_RW_OPEN		= 1 << __FIO_RO_NEEDS_RW_OPEN,
+	FIO_MULTI_RANGE_TRIM		= 1 << __FIO_MULTI_RANGE_TRIM,
 };
 
 /*
diff --git a/libfio.c b/libfio.c
index 5c433277..d0c6bf8f 100644
--- a/libfio.c
+++ b/libfio.c
@@ -378,6 +378,7 @@ int initialize_fio(char *envp[])
 	compiletime_assert((offsetof(struct jobs_eta, m_rate) % 8) == 0, "m_rate");
 
 	compiletime_assert(__TD_F_LAST <= TD_ENG_FLAG_SHIFT, "TD_ENG_FLAG_SHIFT");
+	compiletime_assert((__TD_F_LAST + __FIO_IOENGINE_F_LAST) <= 8*sizeof(((struct thread_data *)0)->flags), "td->flags");
 	compiletime_assert(BSSPLIT_MAX <= ZONESPLIT_MAX, "bsssplit/zone max");
 
 	err = endian_check();
diff --git a/t/jobs/t0032-43063a1c.fio b/t/jobs/t0032-43063a1c.fio
new file mode 100644
index 00000000..db998e5b
--- /dev/null
+++ b/t/jobs/t0032-43063a1c.fio
@@ -0,0 +1,12 @@
+# Expected results: max offset is ~1280K
+# Buggy result: max offset is ~640K
+#
+
+[global]
+ioengine=null
+size=1280K
+io_size=2560k
+bs=128K
+
+[test1]
+rw=rw

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-04-18 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-04-18 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 5b347a1d4e5fd81b6a1b515843b834c39fb3463d:

  Merge branch 'directory-operation' of https://github.com/friendy-su/fio (2024-04-16 11:04:58 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to e0bb44a5b2a67695f0b940772c70f678b323ec54:

  t/io_uring: only calculate per-file depth if we have files (2024-04-17 16:34:47 -0600)

----------------------------------------------------------------
Ankit Kumar (1):
      ioengines: bump up FIO_IOOPS_VERSION

Jens Axboe (2):
      Merge branch 'ioops_ver' of https://github.com/ankit-sam/fio
      t/io_uring: only calculate per-file depth if we have files

Vincent Fu (1):
      github: add reminders to PR template

 .github/PULL_REQUEST_TEMPLATE.md | 7 +++++++
 ioengines.h                      | 2 +-
 t/io_uring.c                     | 4 +++-
 3 files changed, 11 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 4d98a694..6cead5b3 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -6,3 +6,10 @@ Please confirm that your commit message(s) follow these guidelines:
    aren't something like a URL at 72-74 chars.
 4. Empty line
 5. Signed-off-by: Real Name <real@email.com>
+
+Reminders:
+
+1. If you modify struct thread_options, also make corresponding changes in
+   cconv.c and bump FIO_SERVER_VER in server.h
+2. If you change the ioengine interface (hooks, flags, etc), remember to bump
+   FIO_IOOPS_VERSION in ioengines.h.
diff --git a/ioengines.h b/ioengines.h
index 2fd7f52c..e43650f7 100644
--- a/ioengines.h
+++ b/ioengines.h
@@ -9,7 +9,7 @@
 #include "zbd_types.h"
 #include "fdp.h"
 
-#define FIO_IOOPS_VERSION	33
+#define FIO_IOOPS_VERSION	34
 
 #ifndef CONFIG_DYNAMIC_ENGINES
 #define FIO_STATIC	static
diff --git a/t/io_uring.c b/t/io_uring.c
index 18e8b38e..aa6e09e9 100644
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -974,7 +974,9 @@ static int setup_ring(struct submitter *s)
 	for (i = 0; i < p.sq_entries; i++)
 		sring->array[i] = i;
 
-	s->per_file_depth = (depth + s->nr_files - 1) / s->nr_files;
+	s->per_file_depth = INT_MAX;
+	if (s->nr_files)
+		s->per_file_depth = (depth + s->nr_files - 1) / s->nr_files;
 	return 0;
 }
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-04-17 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-04-17 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 8bcd4cb42931996575cb4430f3d630810f213434:

  Revert "ioengines: Make td_io_queue print log_err when got error " (2024-04-15 12:26:56 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 5b347a1d4e5fd81b6a1b515843b834c39fb3463d:

  Merge branch 'directory-operation' of https://github.com/friendy-su/fio (2024-04-16 11:04:58 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      Merge branch 'directory-operation' of https://github.com/friendy-su/fio

friendy-su (1):
      engines/fileoperations: add more description for file/directory operation engines

 HOWTO.rst | 111 ++++++++++++++++++++++++++++++++++++++++----------------
 fio.1     | 123 ++++++++++++++++++++++++++++++++++++++++++++++----------------
 2 files changed, 171 insertions(+), 63 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 25fdfbc4..2f108e36 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -1992,7 +1992,9 @@ I/O engine
 
 .. option:: ioengine=str
 
-	Defines how the job issues I/O to the file. The following types are defined:
+	fio supports 2 kinds of performance measurement: I/O and file/directory operation.
+
+	I/O engines define how the job issues I/O to the file. The following types are defined:
 
 		**sync**
 			Basic :manpage:`read(2)` or :manpage:`write(2)`
@@ -2177,36 +2179,6 @@ I/O engine
 			absolute or relative. See :file:`engines/skeleton_external.c` for
 			details of writing an external I/O engine.
 
-		**filecreate**
-			Simply create the files and do no I/O to them.  You still need to
-			set  `filesize` so that all the accounting still occurs, but no
-			actual I/O will be done other than creating the file.
-
-		**filestat**
-			Simply do stat() and do no I/O to the file. You need to set 'filesize'
-			and 'nrfiles', so that files will be created.
-			This engine is to measure file lookup and meta data access.
-
-		**filedelete**
-			Simply delete the files by unlink() and do no I/O to them. You need to set 'filesize'
-			and 'nrfiles', so that the files will be created.
-			This engine is to measure file delete.
-
-		**dircreate**
-			Simply create the directories and do no I/O to them.  You still need to
-			set  `filesize` so that all the accounting still occurs, but no
-			actual I/O will be done other than creating the directories.
-
-		**dirstat**
-			Simply do stat() and do no I/O to the directories. You need to set 'filesize'
-			and 'nrfiles', so that directories will be created.
-			This engine is to measure directory lookup and meta data access.
-
-		**dirdelete**
-			Simply delete the directories by rmdir() and do no I/O to them. You need to set 'filesize'
-			and 'nrfiles', so that the directories will be created.
-			This engine is to measure directory delete.
-
 		**libpmem**
 			Read and write using mmap I/O to a file on a filesystem
 			mounted with DAX on a persistent memory device through the PMDK
@@ -2276,6 +2248,50 @@ I/O engine
 			several instances to access the same device or file
 			simultaneously, but allow it for threads.
 
+	File/directory operation engines define how the job operates file or directory. The
+	following types are defined:
+
+		**filecreate**
+			Simply create the files and do no I/O to them.  You still need to
+			set  `filesize` so that all the accounting still occurs, but no
+			actual I/O will be done other than creating the file.
+			Example job file: filecreate-ioengine.fio.
+
+		**filestat**
+			Simply do stat() and do no I/O to the file. You need to set 'filesize'
+			and 'nrfiles', so that files will be created.
+			This engine is to measure file lookup and meta data access.
+			Example job file: filestat-ioengine.fio.
+
+		**filedelete**
+			Simply delete the files by unlink() and do no I/O to them. You need to set 'filesize'
+			and 'nrfiles', so that the files will be created.
+			This engine is to measure file delete.
+			Example job file: filedelete-ioengine.fio.
+
+		**dircreate**
+			Simply create the directories and do no I/O to them.  You still need to
+			set  `filesize` so that all the accounting still occurs, but no
+			actual I/O will be done other than creating the directories.
+			Example job file: dircreate-ioengine.fio.
+
+		**dirstat**
+			Simply do stat() and do no I/O to the directories. You need to set 'filesize'
+			and 'nrfiles', so that directories will be created.
+			This engine is to measure directory lookup and meta data access.
+			Example job file: dirstat-ioengine.fio.
+
+		**dirdelete**
+			Simply delete the directories by rmdir() and do no I/O to them. You need to set 'filesize'
+			and 'nrfiles', so that the directories will be created.
+			This engine is to measure directory delete.
+			Example job file: dirdelete-ioengine.fio.
+
+		For file and directory operation engines, there is no I/O throughput, then the
+		statistics data in report have different meanings. The meaningful output indexes are: 'iops' and 'clat'.
+		'bw' is meaningless. Refer to section: "Interpreting the output" for more details.
+
+
 I/O engine specific parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -4583,6 +4599,21 @@ writes in the example above).  In the order listed, they denote:
                 commit if available) functions were completed to when the I/O's
                 completion was reaped by fio.
 
+		For file and directory operation engines, **clat** denotes the time
+		to complete one file or directory operation.
+
+		  **filecreate engine**:the time cost to create a new file
+
+		  **filestat engine**:	the time cost to look up an existing file
+
+		  **filedelete engine**:the time cost to delete a file
+
+		  **dircreate engine**:	the time cost to create a new directory
+
+		  **dirstat engine**:	the time cost to look up an existing directory
+
+		  **dirdelete engine**:	the time cost to delete a directory
+
 **lat**
 		Total latency. Same names as slat and clat, this denotes the time from
 		when fio created the I/O unit to completion of the I/O operation.
@@ -4601,12 +4632,30 @@ writes in the example above).  In the order listed, they denote:
 		are on the same disk, since they are then competing for disk
 		access.
 
+		For file and directory operation engines, **bw** is meaningless.
+
 **iops**
 		IOPS statistics based on measurements from discrete intervals.
 		For details see the description for bw above. See
 		:option:`iopsavgtime` to control the duration of the intervals.
 		Same values reported here as for bw except for percentage.
 
+		For file and directory operation engines, **iops** is the most
+		fundamental index to denote the performance.
+		It means how many files or directories can be operated per second.
+
+		  **filecreate engine**:number of files can be created per second
+
+		  **filestat engine**:	number of files can be looked up per second
+
+		  **filedelete engine**:number of files can be deleted per second
+
+		  **dircreate engine**:	number of directories can be created per second
+
+		  **dirstat engine**:	number of directories can be looked up per second
+
+		  **dirdelete engine**:	number of directories can be deleted per second
+
 **lat (nsec/usec/msec)**
 		The distribution of I/O completion latencies. This is the time from when
 		I/O leaves fio and when it gets completed. Unlike the separate
diff --git a/fio.1 b/fio.1
index 545bb872..5fd3603d 100644
--- a/fio.1
+++ b/fio.1
@@ -1794,8 +1794,9 @@ started on the result.
 .SS "I/O engine"
 .TP
 .BI ioengine \fR=\fPstr
-Defines how the job issues I/O to the file. The following types are defined:
-.RS
+fio supports 2 kinds of performance measurement: I/O and file/directory operation.
+
+I/O engines define how the job issues I/O to the file. The following types are defined:
 .RS
 .TP
 .B sync
@@ -1989,36 +1990,6 @@ ioengine `foo.o' in `/tmp'. The path can be either
 absolute or relative. See `engines/skeleton_external.c' in the fio source for
 details of writing an external I/O engine.
 .TP
-.B filecreate
-Simply create the files and do no I/O to them.  You still need to set
-\fBfilesize\fR so that all the accounting still occurs, but no actual I/O will be
-done other than creating the file.
-.TP
-.B filestat
-Simply do stat() and do no I/O to the file. You need to set 'filesize'
-and 'nrfiles', so that files will be created.
-This engine is to measure file lookup and meta data access.
-.TP
-.B filedelete
-Simply delete files by unlink() and do no I/O to the file. You need to set 'filesize'
-and 'nrfiles', so that files will be created.
-This engine is to measure file delete.
-.TP
-.B dircreate
-Simply create the directories and do no I/O to them.  You still need to set
-\fBfilesize\fR so that all the accounting still occurs, but no actual I/O will be
-done other than creating the directories.
-.TP
-.B dirstat
-Simply do stat() and do no I/O to the directory. You need to set 'filesize'
-and 'nrfiles', so that directories will be created.
-This engine is to measure directory lookup and meta data access.
-.TP
-.B dirdelete
-Simply delete directories by unlink() and do no I/O to the directory. You need to set 'filesize'
-and 'nrfiles', so that directories will be created.
-This engine is to measure directory delete.
-.TP
 .B libpmem
 Read and write using mmap I/O to a file on a filesystem
 mounted with DAX on a persistent memory device through the PMDK
@@ -2079,6 +2050,55 @@ instance is used per process, so all jobs setting option \fBthread\fR will share
 a single instance (with one queue per thread) and must specify compatible
 options. Note that some drivers don't allow several instances to access the same
 device or file simultaneously, but allow it for threads.
+.TP
+.RE
+.P
+File/directory operation engines define how the job operates file or directory.
+The following types are defined:
+.RS
+.TP
+.B filecreate
+Simply create the files and do no I/O to them.  You still need to
+set  \fBfilesize\fP so that all the accounting still occurs, but no
+actual I/O will be done other than creating the file.
+Example job file: filecreate-ioengine.fio.
+.TP
+.B filestat
+Simply do stat() and do no I/O to the file. You need to set \fBfilesize\fP
+and \fBnrfiles\fP, so that files will be created.
+This engine is to measure file lookup and meta data access.
+Example job file: filestat-ioengine.fio.
+.TP
+.B filedelete
+Simply delete the files by unlink() and do no I/O to them. You need to set \fBfilesize\fP
+and \fBnrfiles\fP, so that the files will be created.
+This engine is to measure file delete.
+Example job file: filedelete-ioengine.fio.
+.TP
+.B dircreate
+Simply create the directories and do no I/O to them.  You still need to
+set  \fBfilesize\fP so that all the accounting still occurs, but no
+actual I/O will be done other than creating the directories.
+Example job file: dircreate-ioengine.fio.
+.TP
+.B dirstat
+Simply do stat() and do no I/O to the directories. You need to set \fBfilesize\fP
+and \fBnrfiles\fP, so that directories will be created.
+This engine is to measure directory lookup and meta data access.
+Example job file: dirstat-ioengine.fio.
+.TP
+.B dirdelete
+Simply delete the directories by rmdir() and do no I/O to them. You need to set \fBfilesize\fP
+and \fBnrfiles\fP, so that the directories will be created.
+This engine is to measure directory delete.
+.TP
+.RE
+.P
+For file and directory operation engines, there is no I/O throughput, then the statistics \
+data in report have different meanings. The meaningful output indexes are: \fBiops\fP and \fBclat\fP. \
+\fBbw\fP is meaningless. Refer to section: "Interpreting the output" for more details.
+.RE
+.P
 .SS "I/O engine specific parameters"
 In addition, there are some parameters which are only valid when a specific
 \fBioengine\fR is in use. These are used identically to normal parameters,
@@ -4236,6 +4256,24 @@ submission to completion of the I/O pieces. For sync I/O, clat will
 usually be equal (or very close) to 0, as the time from submit to
 complete is basically just CPU time (I/O has already been done, see slat
 explanation).
+
+For file and directory operation engines, \fBclat\fP denotes the time
+to complete one file or directory operation.
+.RS
+.TP
+\fBfilecreate engine\fP:\tthe time cost to create a new file
+.TP
+\fBfilestat engine\fP:\tthe time cost to look up an existing file
+.TP
+\fBfiledelete engine\fP:\tthe time cost to delete a file
+.TP
+\fBdircreate engine\fP:\tthe time cost to create a new directory
+.TP
+\fBdirstat engine\fP:\tthe time cost to look up an existing directory
+.TP
+\fBdirdelete engine\fP:\tthe time cost to delete a directory
+.TP
+.RE
 .TP
 .B lat
 Total latency. Same names as slat and clat, this denotes the time from
@@ -4250,12 +4288,33 @@ xlat stats, but also includes the number of samples taken (\fIsamples\fR) and an
 approximate percentage of total aggregate bandwidth this thread received in its
 group (\fIper\fR). This last value is only really useful if the threads in this
 group are on the same disk, since they are then competing for disk access.
+
+For file and directory operation engines, \fBbw\fR is meaningless.
 .TP
 .B iops
 IOPS statistics based on measurements from discrete intervals.
 For details see the description for \fBbw\fR above. See
 \fBiopsavgtime\fR to control the duration of the intervals.
 Same values reported here as for \fBbw\fR except for percentage.
+
+For file and directory operation engines, \fBiops\fP is the most
+fundamental index to denote the performance.
+It means how many files or directories can be operated per second.
+.RS
+.TP
+\fBfilecreate engine\fP:\tnumber of files can be created per second
+.TP
+\fBfilestat engine\fP:\tnumber of files can be looked up per second
+.TP
+\fBfiledelete engine\fP:\tnumber of files can be deleted per second
+.TP
+\fBdircreate engine\fP:\tnumber of directories can be created per second
+.TP
+\fBdirstat engine\fP:\tnumber of directories can be looked up per second
+.TP
+\fBdirdelete engine\fP:\tnumber of directories can be deleted per second
+.TP
+.RE
 .TP
 .B lat (nsec/usec/msec)
 The distribution of I/O completion latencies. This is the time from when

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-04-16 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-04-16 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 4eef23f627d103d7092b4141bd6b0c8f95309ee9:

  howto: fix zonemode formatting (2024-04-02 11:10:58 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 8bcd4cb42931996575cb4430f3d630810f213434:

  Revert "ioengines: Make td_io_queue print log_err when got error " (2024-04-15 12:26:56 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Revert "ioengines: Make td_io_queue print log_err when got error "

 ioengines.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/ioengines.c b/ioengines.c
index 5dd4355d..87cc2286 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -413,7 +413,7 @@ enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u)
 	if (io_u->error == EINVAL && td->io_issues[io_u->ddir & 1] == 1 &&
 	    td->o.odirect) {
 
-		log_err("fio: first direct IO errored. File system may not "
+		log_info("fio: first direct IO errored. File system may not "
 			 "support direct IO, or iomem_align= is bad, or "
 			 "invalid block size. Try setting direct=0.\n");
 	}
@@ -421,7 +421,7 @@ enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u)
 	if (zbd_unaligned_write(io_u->error) &&
 	    td->io_issues[io_u->ddir & 1] == 1 &&
 	    td->o.zone_mode != ZONE_MODE_ZBD) {
-		log_err("fio: first I/O failed. If %s is a zoned block device, consider --zonemode=zbd\n",
+		log_info("fio: first I/O failed. If %s is a zoned block device, consider --zonemode=zbd\n",
 			 io_u->file->file_name);
 	}
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-04-03 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-04-03 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 9213e16d98b0e9d2f8d4f7e760ed0fd45c8960f6:

  Fio 3.37 (2024-03-26 15:13:51 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 4eef23f627d103d7092b4141bd6b0c8f95309ee9:

  howto: fix zonemode formatting (2024-04-02 11:10:58 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      howto: fix zonemode formatting

 HOWTO.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 6a204072..25fdfbc4 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -985,14 +985,14 @@ Target file/device
 
 		**none**
 				The :option:`zonerange`, :option:`zonesize`,
-				:option `zonecapacity` and option:`zoneskip`
+				:option:`zonecapacity` and :option:`zoneskip`
 				parameters are ignored.
 		**strided**
 				I/O happens in a single zone until
 				:option:`zonesize` bytes have been transferred.
 				After that number of bytes has been
 				transferred processing of the next zone
-				starts. :option `zonecapacity` is ignored.
+				starts. :option:`zonecapacity` is ignored.
 		**zbd**
 				Zoned block device mode. I/O happens
 				sequentially in each zone, even if random I/O

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-27 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-27 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit b2403d413ee734e8835539319d8bc3429a0777ac:

  Merge branch 'delete-instead-of-unlink' of https://github.com/edigaryev/fio (2024-03-25 10:45:13 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 9213e16d98b0e9d2f8d4f7e760ed0fd45c8960f6:

  Fio 3.37 (2024-03-26 15:13:51 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Fio 3.37

 FIO-VERSION-GEN | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/FIO-VERSION-GEN b/FIO-VERSION-GEN
index cf8dbb0e..be0d7620 100755
--- a/FIO-VERSION-GEN
+++ b/FIO-VERSION-GEN
@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=FIO-VERSION-FILE
-DEF_VER=fio-3.36
+DEF_VER=fio-3.37
 
 LF='
 '

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-26 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-26 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 2b03792ceb7ed00bd50db5b59486fab902295df8:

  Merge branch 'issue-1735' of https://github.com/yygcode/fio (2024-03-22 10:39:37 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to b2403d413ee734e8835539319d8bc3429a0777ac:

  Merge branch 'delete-instead-of-unlink' of https://github.com/edigaryev/fio (2024-03-25 10:45:13 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'delete-instead-of-unlink' of https://github.com/edigaryev/fio

Nikolay Edigaryev (1):
      docs: use "delete" term instead of "unlink", which is less common

 HOWTO.rst | 4 ++--
 fio.1     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index fb067fe5..6a204072 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -971,13 +971,13 @@ Target file/device
 
 .. option:: unlink=bool
 
-	Unlink the job files when done. Not the default, as repeated runs of that
+	Unlink (delete) the job files when done. Not the default, as repeated runs of that
 	job would then waste time recreating the file set again and again. Default:
 	false.
 
 .. option:: unlink_each_loop=bool
 
-	Unlink job files after each iteration or loop.  Default: false.
+	Unlink (delete) job files after each iteration or loop.  Default: false.
 
 .. option:: zonemode=str
 
diff --git a/fio.1 b/fio.1
index 63375c62..545bb872 100644
--- a/fio.1
+++ b/fio.1
@@ -749,12 +749,12 @@ same data multiple times. Thus it will not work on non-seekable I/O engines
 (e.g. network, splice). Default: false.
 .TP
 .BI unlink \fR=\fPbool
-Unlink the job files when done. Not the default, as repeated runs of that
+Unlink (delete) the job files when done. Not the default, as repeated runs of that
 job would then waste time recreating the file set again and again. Default:
 false.
 .TP
 .BI unlink_each_loop \fR=\fPbool
-Unlink job files after each iteration or loop. Default: false.
+Unlink (delete) job files after each iteration or loop. Default: false.
 .TP
 .BI zonemode \fR=\fPstr
 Accepted values are:

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-23 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-23 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 20f42c101f7876648705a4fb8a9e2a647dc936ce:

  t/run-fio-tests: restrict t0031 to Linux only (2024-03-21 08:36:14 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 2b03792ceb7ed00bd50db5b59486fab902295df8:

  Merge branch 'issue-1735' of https://github.com/yygcode/fio (2024-03-22 10:39:37 -0400)

----------------------------------------------------------------
Vincent Fu (4):
      engines/fileoperations: remove extra blank lines
      engines/fileoperations: use local var for ioengine data
      examples: fiograph plots for dir operation ioengines
      Merge branch 'issue-1735' of https://github.com/yygcode/fio

friendy-su (1):
      ioengines: implement dircreate, dirstat, dirdelete engines to fileoperations.c

yonggang.yyg (1):
      iolog: fix disk stats issue

 HOWTO.rst                       |  15 +++++
 engines/fileoperations.c        | 119 ++++++++++++++++++++++++++++++++++++++--
 examples/dircreate-ioengine.fio |  25 +++++++++
 examples/dircreate-ioengine.png | Bin 0 -> 42659 bytes
 examples/dirdelete-ioengine.fio |  18 ++++++
 examples/dirdelete-ioengine.png | Bin 0 -> 45530 bytes
 examples/dirstat-ioengine.fio   |  18 ++++++
 examples/dirstat-ioengine.png   | Bin 0 -> 33597 bytes
 fio.1                           |  15 +++++
 iolog.c                         |   2 +
 10 files changed, 206 insertions(+), 6 deletions(-)
 create mode 100644 examples/dircreate-ioengine.fio
 create mode 100644 examples/dircreate-ioengine.png
 create mode 100644 examples/dirdelete-ioengine.fio
 create mode 100644 examples/dirdelete-ioengine.png
 create mode 100644 examples/dirstat-ioengine.fio
 create mode 100644 examples/dirstat-ioengine.png

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 4c8ac331..fb067fe5 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2192,6 +2192,21 @@ I/O engine
 			and 'nrfiles', so that the files will be created.
 			This engine is to measure file delete.
 
+		**dircreate**
+			Simply create the directories and do no I/O to them.  You still need to
+			set  `filesize` so that all the accounting still occurs, but no
+			actual I/O will be done other than creating the directories.
+
+		**dirstat**
+			Simply do stat() and do no I/O to the directories. You need to set 'filesize'
+			and 'nrfiles', so that directories will be created.
+			This engine is to measure directory lookup and meta data access.
+
+		**dirdelete**
+			Simply delete the directories by rmdir() and do no I/O to them. You need to set 'filesize'
+			and 'nrfiles', so that the directories will be created.
+			This engine is to measure directory delete.
+
 		**libpmem**
 			Read and write using mmap I/O to a file on a filesystem
 			mounted with DAX on a persistent memory device through the PMDK
diff --git a/engines/fileoperations.c b/engines/fileoperations.c
index 1db60da1..c52f0900 100644
--- a/engines/fileoperations.c
+++ b/engines/fileoperations.c
@@ -1,8 +1,8 @@
 /*
- * fileoperations engine
+ * file/directory operations engine
  *
- * IO engine that doesn't do any IO, just operates files and tracks the latency
- * of the file operation.
+ * IO engine that doesn't do any IO, just operates files/directories
+ * and tracks the latency of the operation.
  */
 #include <stdio.h>
 #include <stdlib.h>
@@ -15,9 +15,15 @@
 #include "../optgroup.h"
 #include "../oslib/statx.h"
 
+enum fio_engine {
+	UNKNOWN_OP_ENGINE = 0,
+	FILE_OP_ENGINE = 1,
+	DIR_OP_ENGINE = 2,
+};
 
 struct fc_data {
 	enum fio_ddir stat_ddir;
+	enum fio_engine op_engine;
 };
 
 struct filestat_options {
@@ -61,11 +67,30 @@ static struct fio_option options[] = {
 	},
 };
 
+static int setup_dirs(struct thread_data *td)
+{
+	int ret = 0;
+	int i;
+	struct fio_file *f;
+
+	for_each_file(td, f, i) {
+		dprint(FD_FILE, "setup directory %s\n", f->file_name);
+		ret = fio_mkdir(f->file_name, 0700);
+		if ((ret && errno != EEXIST)) {
+			log_err("create directory %s failed with %d\n",
+				f->file_name, errno);
+			break;
+		}
+		ret = 0;
+	}
+	return ret;
+}
 
 static int open_file(struct thread_data *td, struct fio_file *f)
 {
 	struct timespec start;
 	int do_lat = !td->o.disable_lat;
+	struct fc_data *fcd = td->io_ops_data;
 
 	dprint(FD_FILE, "fd open %s\n", f->file_name);
 
@@ -81,7 +106,14 @@ static int open_file(struct thread_data *td, struct fio_file *f)
 	if (do_lat)
 		fio_gettime(&start, NULL);
 
-	f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
+	if (fcd->op_engine == FILE_OP_ENGINE)
+		f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
+	else if (fcd->op_engine == DIR_OP_ENGINE)
+		f->fd = fio_mkdir(f->file_name, S_IFDIR);
+	else {
+		log_err("fio: unknown file/directory operation engine\n");
+		return 1;
+	}
 
 	if (f->fd == -1) {
 		char buf[FIO_VERROR_SIZE];
@@ -174,11 +206,11 @@ static int stat_file(struct thread_data *td, struct fio_file *f)
 	return 0;
 }
 
-
 static int delete_file(struct thread_data *td, struct fio_file *f)
 {
 	struct timespec start;
 	int do_lat = !td->o.disable_lat;
+	struct fc_data *fcd = td->io_ops_data;
 	int ret;
 
 	dprint(FD_FILE, "fd delete %s\n", f->file_name);
@@ -195,7 +227,14 @@ static int delete_file(struct thread_data *td, struct fio_file *f)
 	if (do_lat)
 		fio_gettime(&start, NULL);
 
-	ret = unlink(f->file_name);
+	if (fcd->op_engine == FILE_OP_ENGINE)
+		ret = unlink(f->file_name);
+	else if (fcd->op_engine == DIR_OP_ENGINE)
+		ret = rmdir(f->file_name);
+	else {
+		log_err("fio: unknown file/directory operation engine\n");
+		return 1;
+	}
 
 	if (ret == -1) {
 		char buf[FIO_VERROR_SIZE];
@@ -250,6 +289,17 @@ static int init(struct thread_data *td)
 	else if (td_write(td))
 		data->stat_ddir = DDIR_WRITE;
 
+	data->op_engine = UNKNOWN_OP_ENGINE;
+
+	if (!strncmp(td->o.ioengine, "file", 4)) {
+		data->op_engine = FILE_OP_ENGINE;
+		dprint(FD_FILE, "Operate engine type: file\n");
+	}
+	if (!strncmp(td->o.ioengine, "dir", 3)) {
+		data->op_engine = DIR_OP_ENGINE;
+		dprint(FD_FILE, "Operate engine type: directory\n");
+	}
+
 	td->io_ops_data = data;
 	return 0;
 }
@@ -261,6 +311,12 @@ static void cleanup(struct thread_data *td)
 	free(data);
 }
 
+static int remove_dir(struct thread_data *td, struct fio_file *f)
+{
+	dprint(FD_FILE, "remove directory %s\n", f->file_name);
+	return rmdir(f->file_name);
+}
+
 static struct ioengine_ops ioengine_filecreate = {
 	.name		= "filecreate",
 	.version	= FIO_IOOPS_VERSION,
@@ -302,12 +358,60 @@ static struct ioengine_ops ioengine_filedelete = {
 				FIO_NOSTATS | FIO_NOFILEHASH,
 };
 
+static struct ioengine_ops ioengine_dircreate = {
+	.name		= "dircreate",
+	.version	= FIO_IOOPS_VERSION,
+	.init		= init,
+	.cleanup	= cleanup,
+	.queue		= queue_io,
+	.get_file_size	= get_file_size,
+	.open_file	= open_file,
+	.close_file	= generic_close_file,
+	.unlink_file    = remove_dir,
+	.flags		= FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+				FIO_NOSTATS | FIO_NOFILEHASH,
+};
+
+static struct ioengine_ops ioengine_dirstat = {
+	.name		= "dirstat",
+	.version	= FIO_IOOPS_VERSION,
+	.setup		= setup_dirs,
+	.init		= init,
+	.cleanup	= cleanup,
+	.queue		= queue_io,
+	.invalidate	= invalidate_do_nothing,
+	.get_file_size	= generic_get_file_size,
+	.open_file	= stat_file,
+	.unlink_file	= remove_dir,
+	.flags		=  FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+				FIO_NOSTATS | FIO_NOFILEHASH,
+	.options	= options,
+	.option_struct_size = sizeof(struct filestat_options),
+};
+
+static struct ioengine_ops ioengine_dirdelete = {
+	.name		= "dirdelete",
+	.version	= FIO_IOOPS_VERSION,
+	.setup		= setup_dirs,
+	.init		= init,
+	.invalidate	= invalidate_do_nothing,
+	.cleanup	= cleanup,
+	.queue		= queue_io,
+	.get_file_size	= get_file_size,
+	.open_file	= delete_file,
+	.unlink_file	= remove_dir,
+	.flags		= FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+				FIO_NOSTATS | FIO_NOFILEHASH,
+};
 
 static void fio_init fio_fileoperations_register(void)
 {
 	register_ioengine(&ioengine_filecreate);
 	register_ioengine(&ioengine_filestat);
 	register_ioengine(&ioengine_filedelete);
+	register_ioengine(&ioengine_dircreate);
+	register_ioengine(&ioengine_dirstat);
+	register_ioengine(&ioengine_dirdelete);
 }
 
 static void fio_exit fio_fileoperations_unregister(void)
@@ -315,4 +419,7 @@ static void fio_exit fio_fileoperations_unregister(void)
 	unregister_ioengine(&ioengine_filecreate);
 	unregister_ioengine(&ioengine_filestat);
 	unregister_ioengine(&ioengine_filedelete);
+	unregister_ioengine(&ioengine_dircreate);
+	unregister_ioengine(&ioengine_dirstat);
+	unregister_ioengine(&ioengine_dirdelete);
 }
diff --git a/examples/dircreate-ioengine.fio b/examples/dircreate-ioengine.fio
new file mode 100644
index 00000000..c89d9e4d
--- /dev/null
+++ b/examples/dircreate-ioengine.fio
@@ -0,0 +1,25 @@
+# Example dircreate job
+#
+# create_on_open is needed so that the open happens during the run and not the
+# setup.
+#
+# openfiles needs to be set so that you do not exceed the maximum allowed open
+# files.
+#
+# filesize needs to be set to a non zero value so fio will actually run, but the
+# IO will not really be done and the write latency numbers will only reflect the
+# open times.
+[global]
+create_on_open=1
+nrfiles=30
+ioengine=dircreate
+fallocate=none
+filesize=4k
+openfiles=1
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]
diff --git a/examples/dircreate-ioengine.png b/examples/dircreate-ioengine.png
new file mode 100644
index 00000000..da1a8c40
Binary files /dev/null and b/examples/dircreate-ioengine.png differ
diff --git a/examples/dirdelete-ioengine.fio b/examples/dirdelete-ioengine.fio
new file mode 100644
index 00000000..4e5b1e2c
--- /dev/null
+++ b/examples/dirdelete-ioengine.fio
@@ -0,0 +1,18 @@
+# Example dirdelete job
+
+# 'filedelete' engine only do 'rmdir(dirname)'.
+# 'filesize' must be set, then directories will be created at setup stage.
+# 'unlink' is better set to 0, since the directory is deleted in measurement.
+# the options disabled completion latency output such as 'disable_clat' and 'gtod_reduce' must not set.
+[global]
+ioengine=dirdelete
+filesize=4k
+nrfiles=200
+unlink=0
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]
diff --git a/examples/dirdelete-ioengine.png b/examples/dirdelete-ioengine.png
new file mode 100644
index 00000000..af246195
Binary files /dev/null and b/examples/dirdelete-ioengine.png differ
diff --git a/examples/dirstat-ioengine.fio b/examples/dirstat-ioengine.fio
new file mode 100644
index 00000000..1322dd28
--- /dev/null
+++ b/examples/dirstat-ioengine.fio
@@ -0,0 +1,18 @@
+# Example dirstat job
+
+# 'dirstat' engine only do 'stat(dirname)', file will not be open().
+# 'filesize' must be set, then files will be created at setup stage.
+
+[global]
+ioengine=dirstat
+numjobs=10
+filesize=4k
+nrfiles=5
+thread
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]
diff --git a/examples/dirstat-ioengine.png b/examples/dirstat-ioengine.png
new file mode 100644
index 00000000..14b948ba
Binary files /dev/null and b/examples/dirstat-ioengine.png differ
diff --git a/fio.1 b/fio.1
index 09c6b621..63375c62 100644
--- a/fio.1
+++ b/fio.1
@@ -2004,6 +2004,21 @@ Simply delete files by unlink() and do no I/O to the file. You need to set 'file
 and 'nrfiles', so that files will be created.
 This engine is to measure file delete.
 .TP
+.B dircreate
+Simply create the directories and do no I/O to them.  You still need to set
+\fBfilesize\fR so that all the accounting still occurs, but no actual I/O will be
+done other than creating the directories.
+.TP
+.B dirstat
+Simply do stat() and do no I/O to the directory. You need to set 'filesize'
+and 'nrfiles', so that directories will be created.
+This engine is to measure directory lookup and meta data access.
+.TP
+.B dirdelete
+Simply delete directories by unlink() and do no I/O to the directory. You need to set 'filesize'
+and 'nrfiles', so that directories will be created.
+This engine is to measure directory delete.
+.TP
 .B libpmem
 Read and write using mmap I/O to a file on a filesystem
 mounted with DAX on a persistent memory device through the PMDK
diff --git a/iolog.c b/iolog.c
index 251e9d7f..96af4f33 100644
--- a/iolog.c
+++ b/iolog.c
@@ -814,6 +814,8 @@ bool init_iolog(struct thread_data *td)
 	if (!ret)
 		td_verror(td, EINVAL, "failed initializing iolog");
 
+	init_disk_util(td);
+
 	return ret;
 }
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-22 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-22 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 140c58beeee44a10358a817c7699b66c5c7290f9:

  test: add the test for regrow logs with asynchronous I/O replay (2024-03-21 05:57:54 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 20f42c101f7876648705a4fb8a9e2a647dc936ce:

  t/run-fio-tests: restrict t0031 to Linux only (2024-03-21 08:36:14 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      t/run-fio-tests: restrict t0031 to Linux only

 t/run-fio-tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index 1b884d87..22580613 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -876,7 +876,7 @@ TEST_LIST = [
         'success':          SUCCESS_DEFAULT,
         'pre_job':          't0031-pre.fio',
         'pre_success':      SUCCESS_DEFAULT,
-        'requirements':     [],
+        'requirements':     [Requirements.linux, Requirements.libaio],
     },
     {
         'test_id':          1000,

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-21 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-21 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 7d6c99e917f7d68ffebbd1750802f7aed9c3d461:

  docs: fix documentation for rate_cycle (2024-03-18 14:51:10 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 140c58beeee44a10358a817c7699b66c5c7290f9:

  test: add the test for regrow logs with asynchronous I/O replay (2024-03-21 05:57:54 -0600)

----------------------------------------------------------------
Shin'ichiro Kawasaki (2):
      iolog: regrow logs in iolog_delay()
      test: add the test for regrow logs with asynchronous I/O replay

Vincent Fu (2):
      t/fiotestlib: pass command-line options to FioJobFileTest
      t/jobs/t0030: add test for --bandwidth-log option

 iolog.c              |  2 ++
 t/fiotestlib.py      |  8 +++++---
 t/jobs/t0030.fio     | 10 ++++++++++
 t/jobs/t0031-pre.fio |  8 ++++++++
 t/jobs/t0031.fio     |  7 +++++++
 t/run-fio-tests.py   | 19 +++++++++++++++++++
 6 files changed, 51 insertions(+), 3 deletions(-)
 create mode 100644 t/jobs/t0030.fio
 create mode 100644 t/jobs/t0031-pre.fio
 create mode 100644 t/jobs/t0031.fio

---

Diff of recent changes:

diff --git a/iolog.c b/iolog.c
index f52a9a80..251e9d7f 100644
--- a/iolog.c
+++ b/iolog.c
@@ -102,6 +102,8 @@ static void iolog_delay(struct thread_data *td, unsigned long delay)
 		ret = io_u_queued_complete(td, 0);
 		if (ret < 0)
 			td_verror(td, -ret, "io_u_queued_complete");
+		if (td->flags & TD_F_REGROW_LOGS)
+			regrow_logs(td);
 		if (utime_since_now(&ts) > delay)
 			break;
 	}
diff --git a/t/fiotestlib.py b/t/fiotestlib.py
index a96338a3..466e482d 100755
--- a/t/fiotestlib.py
+++ b/t/fiotestlib.py
@@ -175,7 +175,7 @@ class FioJobFileTest(FioExeTest):
 
         super().__init__(fio_path, success, testnum, artifact_root)
 
-    def setup(self, parameters=None):
+    def setup(self, parameters):
         """Setup instance variables for fio job test."""
 
         self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
@@ -185,6 +185,8 @@ class FioJobFileTest(FioExeTest):
             f"--output={self.filenames['fio_output']}",
             self.fio_job,
             ]
+        if parameters:
+            fio_args += parameters
 
         super().setup(fio_args)
 
@@ -206,7 +208,7 @@ class FioJobFileTest(FioExeTest):
                             self.testnum,
                             self.paths['artifacts'],
                             output_format=self.output_format)
-        precon.setup()
+        precon.setup(None)
         precon.run()
         precon.check_result()
         self.precon_failed = not precon.passed
@@ -412,7 +414,7 @@ def run_fio_tests(test_list, test_env, args):
                 fio_pre_success=fio_pre_success,
                 output_format=output_format)
             desc = config['job']
-            parameters = []
+            parameters = config['parameters'] if 'parameters' in config else None
         elif issubclass(config['test_class'], FioJobCmdTest):
             if not 'success' in config:
                 config['success'] = SUCCESS_DEFAULT
diff --git a/t/jobs/t0030.fio b/t/jobs/t0030.fio
new file mode 100644
index 00000000..8bbc810e
--- /dev/null
+++ b/t/jobs/t0030.fio
@@ -0,0 +1,10 @@
+# run with --bandwidth-log
+# broken behavior: seg fault
+# successful behavior: test runs to completion with 0 as the exit code
+
+[test]
+ioengine=null
+filesize=1T
+rw=read
+time_based
+runtime=2s
diff --git a/t/jobs/t0031-pre.fio b/t/jobs/t0031-pre.fio
new file mode 100644
index 00000000..ce4ee3b6
--- /dev/null
+++ b/t/jobs/t0031-pre.fio
@@ -0,0 +1,8 @@
+[job]
+rw=write
+ioengine=libaio
+size=1mb
+time_based=1
+runtime=1
+filename=t0030file
+write_iolog=iolog
diff --git a/t/jobs/t0031.fio b/t/jobs/t0031.fio
new file mode 100644
index 00000000..ae8f7442
--- /dev/null
+++ b/t/jobs/t0031.fio
@@ -0,0 +1,7 @@
+[job]
+rw=read
+ioengine=libaio
+iodepth=128
+filename=t0030file
+read_iolog=iolog
+write_lat_log=lat_log
diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index 08134e50..1b884d87 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -859,6 +859,25 @@ TEST_LIST = [
         'output_format':    'json',
         'requirements':     [],
     },
+    {
+        'test_id':          30,
+        'test_class':       FioJobFileTest,
+        'job':              't0030.fio',
+        'success':          SUCCESS_DEFAULT,
+        'pre_job':          None,
+        'pre_success':      None,
+        'parameters':       ['--bandwidth-log'],
+        'requirements':     [],
+    },
+    {
+        'test_id':          31,
+        'test_class':       FioJobFileTest,
+        'job':              't0031.fio',
+        'success':          SUCCESS_DEFAULT,
+        'pre_job':          't0031-pre.fio',
+        'pre_success':      SUCCESS_DEFAULT,
+        'requirements':     [],
+    },
     {
         'test_id':          1000,
         'test_class':       FioExeTest,

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-19 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-19 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit b140fc5e484638a467480e369485b91290288d58:

  t/nvmept_pi: add support for xNVMe ioengine (2024-03-07 19:36:30 +0000)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 7d6c99e917f7d68ffebbd1750802f7aed9c3d461:

  docs: fix documentation for rate_cycle (2024-03-18 14:51:10 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      docs: fix documentation for rate_cycle

 HOWTO.rst | 4 ++--
 fio.1     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 2386d806..4c8ac331 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -3323,8 +3323,8 @@ I/O rate
 
 .. option:: rate_cycle=int
 
-	Average bandwidth for :option:`rate` and :option:`rate_min` over this number
-	of milliseconds. Defaults to 1000.
+        Average bandwidth for :option:`rate_min` and :option:`rate_iops_min`
+        over this number of milliseconds. Defaults to 1000.
 
 
 I/O latency
diff --git a/fio.1 b/fio.1
index d955385d..09c6b621 100644
--- a/fio.1
+++ b/fio.1
@@ -3064,7 +3064,7 @@ ignore the thinktime and continue doing IO at the specified rate, instead of
 entering a catch-up mode after thinktime is done.
 .TP
 .BI rate_cycle \fR=\fPint
-Average bandwidth for \fBrate\fR and \fBrate_min\fR over this number
+Average bandwidth for \fBrate_min\fR and \fBrate_iops_min\fR over this number
 of milliseconds. Defaults to 1000.
 .SS "I/O latency"
 .TP

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-08 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-08 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 9b699fb150bbed56939d317ffc004b3bf19f098f:

  Doc: Make note of using bsrange with ':' (2024-03-05 10:54:36 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to b140fc5e484638a467480e369485b91290288d58:

  t/nvmept_pi: add support for xNVMe ioengine (2024-03-07 19:36:30 +0000)

----------------------------------------------------------------
Vincent Fu (6):
      examples: update plot for cmdprio-bssplit
      examples: add plots for xNVMe examples
      examples: add fiograph plot for uring-cmd-trim-multi-range
      examples: add fiograph plots for netio_vsock examples
      t/nvmept_pi: drop JSON output for error cases
      t/nvmept_pi: add support for xNVMe ioengine

 examples/cmdprio-bssplit.png            | Bin 45606 -> 134359 bytes
 examples/netio_vsock.png                | Bin 0 -> 46920 bytes
 examples/netio_vsock_receiver.png       | Bin 0 -> 34879 bytes
 examples/netio_vsock_sender.png         | Bin 0 -> 35539 bytes
 examples/uring-cmd-trim-multi-range.png | Bin 0 -> 67853 bytes
 examples/xnvme-fdp.png                  | Bin 0 -> 53284 bytes
 examples/xnvme-pi.png                   | Bin 0 -> 76492 bytes
 t/nvmept_pi.py                          |  18 +++++++++++-------
 8 files changed, 11 insertions(+), 7 deletions(-)
 create mode 100644 examples/netio_vsock.png
 create mode 100644 examples/netio_vsock_receiver.png
 create mode 100644 examples/netio_vsock_sender.png
 create mode 100644 examples/uring-cmd-trim-multi-range.png
 create mode 100644 examples/xnvme-fdp.png
 create mode 100644 examples/xnvme-pi.png

---

Diff of recent changes:

diff --git a/examples/cmdprio-bssplit.png b/examples/cmdprio-bssplit.png
index a0bb3ff4..83a5570b 100644
Binary files a/examples/cmdprio-bssplit.png and b/examples/cmdprio-bssplit.png differ
diff --git a/examples/netio_vsock.png b/examples/netio_vsock.png
new file mode 100644
index 00000000..01aadde5
Binary files /dev/null and b/examples/netio_vsock.png differ
diff --git a/examples/netio_vsock_receiver.png b/examples/netio_vsock_receiver.png
new file mode 100644
index 00000000..524a7a1c
Binary files /dev/null and b/examples/netio_vsock_receiver.png differ
diff --git a/examples/netio_vsock_sender.png b/examples/netio_vsock_sender.png
new file mode 100644
index 00000000..75802aaf
Binary files /dev/null and b/examples/netio_vsock_sender.png differ
diff --git a/examples/uring-cmd-trim-multi-range.png b/examples/uring-cmd-trim-multi-range.png
new file mode 100644
index 00000000..c3ffd546
Binary files /dev/null and b/examples/uring-cmd-trim-multi-range.png differ
diff --git a/examples/xnvme-fdp.png b/examples/xnvme-fdp.png
new file mode 100644
index 00000000..7f802741
Binary files /dev/null and b/examples/xnvme-fdp.png differ
diff --git a/examples/xnvme-pi.png b/examples/xnvme-pi.png
new file mode 100644
index 00000000..def7e680
Binary files /dev/null and b/examples/xnvme-pi.png differ
diff --git a/t/nvmept_pi.py b/t/nvmept_pi.py
index 5de77c9d..df7c0b9f 100755
--- a/t/nvmept_pi.py
+++ b/t/nvmept_pi.py
@@ -43,13 +43,11 @@ class DifDixTest(FioJobCmdTest):
 
         fio_args = [
             "--name=nvmept_pi",
-            "--ioengine=io_uring_cmd",
-            "--cmd_type=nvme",
+            f"--ioengine={self.fio_opts['ioengine']}",
             f"--filename={self.fio_opts['filename']}",
             f"--rw={self.fio_opts['rw']}",
             f"--bsrange={self.fio_opts['bsrange']}",
             f"--output={self.filenames['output']}",
-            f"--output-format={self.fio_opts['output-format']}",
             f"--md_per_io_size={self.fio_opts['md_per_io_size']}",
             f"--pi_act={self.fio_opts['pi_act']}",
             f"--pi_chk={self.fio_opts['pi_chk']}",
@@ -58,11 +56,18 @@ class DifDixTest(FioJobCmdTest):
         ]
         for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles',
                     'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait',
-                    'time_based', 'runtime', 'verify', 'io_size', 'offset', 'number_ios']:
+                    'time_based', 'runtime', 'verify', 'io_size', 'offset', 'number_ios',
+                    'output-format']:
             if opt in self.fio_opts:
                 option = f"--{opt}={self.fio_opts[opt]}"
                 fio_args.append(option)
 
+        if self.fio_opts['ioengine'] == 'io_uring_cmd':
+            fio_args.append('--cmd_type=nvme')
+        elif self.fio_opts['ioengine'] == 'xnvme':
+            fio_args.append('--thread=1')
+            fio_args.append('--xnvme_async=io_uring_cmd')
+
         super().setup(fio_args)
 
 
@@ -622,7 +627,6 @@ TEST_LIST = [
         "fio_opts": {
             "rw": 'read',
             "number_ios": NUMBER_IOS,
-            "output-format": "json",
             "pi_act": 0,
             "apptag": "0x8888",
             "apptag_mask": "0x0FFF",
@@ -639,7 +643,6 @@ TEST_LIST = [
         "fio_opts": {
             "rw": 'read',
             "number_ios": NUMBER_IOS,
-            "output-format": "json",
             "pi_act": 0,
             "apptag": "0x8888",
             "apptag_mask": "0x0FFF",
@@ -660,7 +663,6 @@ TEST_LIST = [
         "fio_opts": {
             "rw": 'read',
             "number_ios": NUMBER_IOS,
-            "output-format": "json",
             "pi_act": 0,
             "apptag": "0x8888",
             "apptag_mask": "0x0FFF",
@@ -689,6 +691,7 @@ def parse_args():
                         '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
     parser.add_argument('-l', '--lbaf', nargs='+', type=int,
                         help='list of lba formats to test')
+    parser.add_argument('-i', '--ioengine', default='io_uring_cmd')
     args = parser.parse_args()
 
     return args
@@ -909,6 +912,7 @@ def main():
 
     for test in TEST_LIST:
         test['fio_opts']['filename'] = args.dut
+        test['fio_opts']['ioengine'] = args.ioengine
 
     test_env = {
               'fio_path': fio_path,

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-06 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-06 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 2dddfd35396e2f7a1bb06cc7c92aa1e283be084e:

  Merge branch 'patch-ioengines' of https://github.com/kcoms555/fio (2024-03-04 07:31:47 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 9b699fb150bbed56939d317ffc004b3bf19f098f:

  Doc: Make note of using bsrange with ':' (2024-03-05 10:54:36 -0500)

----------------------------------------------------------------
Avri Altman (5):
      fio: Some minor code cleanups
      t/jobs: Further clarify regression test 7
      t/jobs: Rename test job 15
      t/jobs: Fix a typo in jobs 23 & 24
      Doc: Make note of using bsrange with ':'

 HOWTO.rst                                         | 4 ++--
 backend.c                                         | 4 ++--
 eta.c                                             | 8 +++++---
 fio.1                                             | 2 +-
 io_u.c                                            | 3 ++-
 parse.h                                           | 2 +-
 stat.h                                            | 1 -
 t/jobs/t0007-37cf9e3c.fio                         | 5 ++++-
 t/jobs/{t0015-e78980ff.fio => t0015-4e7e7898.fio} | 0
 t/jobs/t0023.fio                                  | 4 ++--
 t/jobs/t0024.fio                                  | 2 +-
 t/run-fio-tests.py                                | 2 +-
 12 files changed, 21 insertions(+), 16 deletions(-)
 rename t/jobs/{t0015-e78980ff.fio => t0015-4e7e7898.fio} (100%)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 169cdc2a..2386d806 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -1631,7 +1631,7 @@ Block size
 	Comma-separated ranges may be specified for reads, writes, and trims as
 	described in :option:`blocksize`.
 
-	Example: ``bsrange=1k-4k,2k-8k``.
+	Example: ``bsrange=1k-4k,2k-8k`` also the ':' delimiter ``bsrange=1k:4k,2k:8k``.
 
 .. option:: bssplit=str[,str][,str]
 
@@ -2786,7 +2786,7 @@ with the caveat that when used on the command line, they must come after the
 
 .. option:: sg_write_mode=str : [sg]
 
-	Specify the type of write commands to issue. This option can take three values:
+	Specify the type of write commands to issue. This option can take ten values:
 
 	**write**
 		This is the default where write opcodes are issued as usual.
diff --git a/backend.c b/backend.c
index 2f2221bf..fb7dc68a 100644
--- a/backend.c
+++ b/backend.c
@@ -2094,14 +2094,14 @@ static void reap_threads(unsigned int *nr_running, uint64_t *t_rate,
 			 uint64_t *m_rate)
 {
 	unsigned int cputhreads, realthreads, pending;
-	int status, ret;
+	int ret;
 
 	/*
 	 * reap exited threads (TD_EXITED -> TD_REAPED)
 	 */
 	realthreads = pending = cputhreads = 0;
 	for_each_td(td) {
-		int flags = 0;
+		int flags = 0, status;
 
 		if (!strcmp(td->o.ioengine, "cpuio"))
 			cputhreads++;
diff --git a/eta.c b/eta.c
index cc342461..7d07708f 100644
--- a/eta.c
+++ b/eta.c
@@ -215,8 +215,9 @@ static unsigned long thread_eta(struct thread_data *td)
 				perc = td->o.rwmix[DDIR_WRITE];
 
 			bytes_total += (bytes_total * perc) / 100;
-		} else
+		} else {
 			bytes_total <<= 1;
+		}
 	}
 
 	if (td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING) {
@@ -228,8 +229,9 @@ static unsigned long thread_eta(struct thread_data *td)
 			perc = (double) bytes_done / (double) bytes_total;
 			if (perc > 1.0)
 				perc = 1.0;
-		} else
+		} else {
 			perc = 0.0;
+		}
 
 		if (td->o.time_based) {
 			if (timeout) {
@@ -395,7 +397,7 @@ static bool skip_eta()
  * Print status of the jobs we know about. This includes rate estimates,
  * ETA, thread state, etc.
  */
-bool calc_thread_status(struct jobs_eta *je, int force)
+static bool calc_thread_status(struct jobs_eta *je, int force)
 {
 	int unified_rw_rep;
 	bool any_td_in_ramp;
diff --git a/fio.1 b/fio.1
index e6b291a7..d955385d 100644
--- a/fio.1
+++ b/fio.1
@@ -1434,7 +1434,7 @@ described in \fBblocksize\fR. Example:
 .RS
 .RS
 .P
-bsrange=1k\-4k,2k\-8k
+bsrange=1k\-4k,2k\-8k or bsrange=1k:4k,2k:8k
 .RE
 .RE
 .TP
diff --git a/io_u.c b/io_u.c
index 2b8e17f8..09e5f15a 100644
--- a/io_u.c
+++ b/io_u.c
@@ -1895,8 +1895,9 @@ struct io_u *get_io_u(struct thread_data *td)
 					io_u->buflen);
 			} else if ((td->flags & TD_F_SCRAMBLE_BUFFERS) &&
 				   !(td->flags & TD_F_COMPRESS) &&
-				   !(td->flags & TD_F_DO_VERIFY))
+				   !(td->flags & TD_F_DO_VERIFY)) {
 				do_scramble = 1;
+			}
 		} else if (io_u->ddir == DDIR_READ) {
 			/*
 			 * Reset the buf_filled parameters so next time if the
diff --git a/parse.h b/parse.h
index d68484ea..806a76ee 100644
--- a/parse.h
+++ b/parse.h
@@ -32,7 +32,7 @@ enum fio_opt_type {
  */
 struct value_pair {
 	const char *ival;		/* string option */
-	unsigned long long oval;/* output value */
+	unsigned long long oval;	/* output value */
 	const char *help;		/* help text for sub option */
 	int orval;			/* OR value */
 	void *cb;			/* sub-option callback */
diff --git a/stat.h b/stat.h
index bd986d4e..0d57cceb 100644
--- a/stat.h
+++ b/stat.h
@@ -345,7 +345,6 @@ extern void stat_exit(void);
 
 extern struct json_object * show_thread_status(struct thread_stat *ts, struct group_run_stats *rs, struct flist_head *, struct buf_output *);
 extern void show_group_stats(struct group_run_stats *rs, struct buf_output *);
-extern bool calc_thread_status(struct jobs_eta *je, int force);
 extern void display_thread_status(struct jobs_eta *je);
 extern void __show_run_stats(void);
 extern int __show_running_run_stats(void);
diff --git a/t/jobs/t0007-37cf9e3c.fio b/t/jobs/t0007-37cf9e3c.fio
index d3c98751..b2592694 100644
--- a/t/jobs/t0007-37cf9e3c.fio
+++ b/t/jobs/t0007-37cf9e3c.fio
@@ -1,4 +1,7 @@
-# Expected result: fio reads 87040KB of data
+# Expected result: fio reads 87040KB of data:
+# first read is at offset 0, then 2nd read is at offset 1.5m, then the 3rd
+# read is at offset 3m, and after the last read at offset 127m - we have only
+# read 87,040K data.
 # Buggy result: fio reads the full 128MB of data
 [foo]
 size=128mb
diff --git a/t/jobs/t0015-e78980ff.fio b/t/jobs/t0015-4e7e7898.fio
similarity index 100%
rename from t/jobs/t0015-e78980ff.fio
rename to t/jobs/t0015-4e7e7898.fio
diff --git a/t/jobs/t0023.fio b/t/jobs/t0023.fio
index 4f0bef89..8e14a110 100644
--- a/t/jobs/t0023.fio
+++ b/t/jobs/t0023.fio
@@ -33,7 +33,7 @@ bsrange=512-4k
 # 			block sizes match
 # Buggy result: 	something else
 [bssplit]
-bsrange=512/25:1k:25:2k:25:4k/25
+bssplit=512/25:1k/:2k/:4k/
 
 # Expected result: 	trim issued to random offset followed by write to same offset
 # 			block sizes match
@@ -59,5 +59,5 @@ norandommap=1
 # 			block sizes match
 # Buggy result: 	something else
 [bssplit_no_rm]
-bsrange=512/25:1k:25:2k:25:4k/25
+bssplit=512/25:1k/:2k/:4k/
 norandommap=1
diff --git a/t/jobs/t0024.fio b/t/jobs/t0024.fio
index 393a2b70..2b3dc94c 100644
--- a/t/jobs/t0024.fio
+++ b/t/jobs/t0024.fio
@@ -33,4 +33,4 @@ bsrange=512-4k
 # 			block sizes match
 # Buggy result: 	something else
 [bssplit]
-bsrange=512/25:1k:25:2k:25:4k/25
+bssplit=512/25:1k/:2k/:4k/
diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index d4742e96..08134e50 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -722,7 +722,7 @@ TEST_LIST = [
     {
         'test_id':          15,
         'test_class':       FioJobFileTest_t0015,
-        'job':              't0015-e78980ff.fio',
+        'job':              't0015-4e7e7898.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
         'pre_success':      None,

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-03-05 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-03-05 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 5ae4f4220a48dddddc84c8b839ef9d8a1ed4edb1:

  gettime: fix cpuclock-test on AMD platforms (2024-02-27 12:36:45 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 2dddfd35396e2f7a1bb06cc7c92aa1e283be084e:

  Merge branch 'patch-ioengines' of https://github.com/kcoms555/fio (2024-03-04 07:31:47 -0700)

----------------------------------------------------------------
Jaeho (1):
      ioengines: Make td_io_queue print log_err when got error

Jens Axboe (1):
      Merge branch 'patch-ioengines' of https://github.com/kcoms555/fio

 ioengines.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/ioengines.c b/ioengines.c
index 87cc2286..5dd4355d 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -413,7 +413,7 @@ enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u)
 	if (io_u->error == EINVAL && td->io_issues[io_u->ddir & 1] == 1 &&
 	    td->o.odirect) {
 
-		log_info("fio: first direct IO errored. File system may not "
+		log_err("fio: first direct IO errored. File system may not "
 			 "support direct IO, or iomem_align= is bad, or "
 			 "invalid block size. Try setting direct=0.\n");
 	}
@@ -421,7 +421,7 @@ enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u)
 	if (zbd_unaligned_write(io_u->error) &&
 	    td->io_issues[io_u->ddir & 1] == 1 &&
 	    td->o.zone_mode != ZONE_MODE_ZBD) {
-		log_info("fio: first I/O failed. If %s is a zoned block device, consider --zonemode=zbd\n",
+		log_err("fio: first I/O failed. If %s is a zoned block device, consider --zonemode=zbd\n",
 			 io_u->file->file_name);
 	}
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-28 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-28 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 9af4af7ab40c4e505033d0e077cc42ac84996b09:

  ci: fix macOS sphinx install issues (2024-02-22 20:01:27 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 5ae4f4220a48dddddc84c8b839ef9d8a1ed4edb1:

  gettime: fix cpuclock-test on AMD platforms (2024-02-27 12:36:45 -0500)

----------------------------------------------------------------
Vincent Fu (2):
      howto: fix job_start_clock_id formatting
      gettime: fix cpuclock-test on AMD platforms

 HOWTO.rst          | 5 +++--
 arch/arch-x86_64.h | 5 +++++
 arch/arch.h        | 7 +++++++
 gettime.c          | 2 +-
 4 files changed, 16 insertions(+), 3 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 4b02100c..169cdc2a 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -756,8 +756,9 @@ Time related parameters
 	CPU mask of other jobs.
 
 .. option:: job_start_clock_id=int
-   The clock_id passed to the call to `clock_gettime` used to record job_start
-   in the `json` output format. Default is 0, or CLOCK_REALTIME.
+
+        The clock_id passed to the call to `clock_gettime` used to record
+        job_start in the `json` output format. Default is 0, or CLOCK_REALTIME.
 
 
 Target file/device
diff --git a/arch/arch-x86_64.h b/arch/arch-x86_64.h
index 86ce1b7e..b402dc6d 100644
--- a/arch/arch-x86_64.h
+++ b/arch/arch-x86_64.h
@@ -26,6 +26,11 @@ static inline unsigned long arch_ffz(unsigned long bitmask)
 	return bitmask;
 }
 
+static inline void tsc_barrier(void)
+{
+	__asm__ __volatile__("mfence":::"memory");
+}
+
 static inline unsigned long long get_cpu_clock(void)
 {
 	unsigned int lo, hi;
diff --git a/arch/arch.h b/arch/arch.h
index 3ee9b053..7e294ddf 100644
--- a/arch/arch.h
+++ b/arch/arch.h
@@ -108,6 +108,13 @@ extern unsigned long arch_flags;
 #include "arch-generic.h"
 #endif
 
+#if !defined(__x86_64__) && defined(CONFIG_SYNC_SYNC)
+static inline void tsc_barrier(void)
+{
+	__sync_synchronize();
+}
+#endif
+
 #include "../lib/ffz.h"
 /* IWYU pragma: end_exports */
 
diff --git a/gettime.c b/gettime.c
index bc66a3ac..5ca31206 100644
--- a/gettime.c
+++ b/gettime.c
@@ -623,7 +623,7 @@ static void *clock_thread_fn(void *data)
 			seq = *t->seq;
 			if (seq == UINT_MAX)
 				break;
-			__sync_synchronize();
+			tsc_barrier();
 			tsc = get_cpu_clock();
 		} while (seq != atomic32_compare_and_swap(t->seq, seq, seq + 1));
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-23 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-23 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 38d01cc56366aa2fd3af42dbab522888b6359dec:

  verify: fix integer sizes in verify state file (2024-02-16 09:25:35 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 9af4af7ab40c4e505033d0e077cc42ac84996b09:

  ci: fix macOS sphinx install issues (2024-02-22 20:01:27 -0500)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'fix-tests-cfi' of https://github.com/mikoxyz/fio

Miko Larsson (3):
      t/io_uring: include libgen.h
      t/io_uring: use char * for name arg in detect_node
      options: declare *__val as long long

Vincent Fu (1):
      ci: fix macOS sphinx install issues

 ci/actions-install.sh |  5 ++---
 options.c             | 12 ++++++------
 t/io_uring.c          |  3 ++-
 3 files changed, 10 insertions(+), 10 deletions(-)

---

Diff of recent changes:

diff --git a/ci/actions-install.sh b/ci/actions-install.sh
index 76335fbc..6eb2d795 100755
--- a/ci/actions-install.sh
+++ b/ci/actions-install.sh
@@ -86,9 +86,8 @@ install_macos() {
     #echo "Updating homebrew..."
     #brew update >/dev/null 2>&1
     echo "Installing packages..."
-    HOMEBREW_NO_AUTO_UPDATE=1 brew install cunit libnfs sphinx-doc pygments python-certifi
-    brew link sphinx-doc --force
-    pip3 install scipy six statsmodels
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install cunit libnfs
+    pip3 install scipy six statsmodels sphinx
 }
 
 install_windows() {
diff --git a/options.c b/options.c
index 25e042d0..de935efc 100644
--- a/options.c
+++ b/options.c
@@ -647,7 +647,7 @@ static int fio_clock_source_cb(void *data, const char *str)
 	return 0;
 }
 
-static int str_rwmix_read_cb(void *data, unsigned long long *val)
+static int str_rwmix_read_cb(void *data, long long *val)
 {
 	struct thread_data *td = cb_data_to_td(data);
 
@@ -656,7 +656,7 @@ static int str_rwmix_read_cb(void *data, unsigned long long *val)
 	return 0;
 }
 
-static int str_rwmix_write_cb(void *data, unsigned long long *val)
+static int str_rwmix_write_cb(void *data, long long *val)
 {
 	struct thread_data *td = cb_data_to_td(data);
 
@@ -1625,7 +1625,7 @@ static int str_gtod_reduce_cb(void *data, int *il)
 	return 0;
 }
 
-static int str_offset_cb(void *data, unsigned long long *__val)
+static int str_offset_cb(void *data, long long *__val)
 {
 	struct thread_data *td = cb_data_to_td(data);
 	unsigned long long v = *__val;
@@ -1646,7 +1646,7 @@ static int str_offset_cb(void *data, unsigned long long *__val)
 	return 0;
 }
 
-static int str_offset_increment_cb(void *data, unsigned long long *__val)
+static int str_offset_increment_cb(void *data, long long *__val)
 {
 	struct thread_data *td = cb_data_to_td(data);
 	unsigned long long v = *__val;
@@ -1667,7 +1667,7 @@ static int str_offset_increment_cb(void *data, unsigned long long *__val)
 	return 0;
 }
 
-static int str_size_cb(void *data, unsigned long long *__val)
+static int str_size_cb(void *data, long long *__val)
 {
 	struct thread_data *td = cb_data_to_td(data);
 	unsigned long long v = *__val;
@@ -1711,7 +1711,7 @@ static int str_io_size_cb(void *data, unsigned long long *__val)
 	return 0;
 }
 
-static int str_zoneskip_cb(void *data, unsigned long long *__val)
+static int str_zoneskip_cb(void *data, long long *__val)
 {
 	struct thread_data *td = cb_data_to_td(data);
 	unsigned long long v = *__val;
diff --git a/t/io_uring.c b/t/io_uring.c
index 46b153dc..18e8b38e 100644
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -28,6 +28,7 @@
 #include <string.h>
 #include <pthread.h>
 #include <sched.h>
+#include <libgen.h>
 
 #include "../arch/arch.h"
 #include "../os/os.h"
@@ -819,7 +820,7 @@ static void set_affinity(struct submitter *s)
 #endif
 }
 
-static int detect_node(struct submitter *s, const char *name)
+static int detect_node(struct submitter *s, char *name)
 {
 #ifdef CONFIG_LIBNUMA
 	const char *base = basename(name);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-17 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-17 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 2d0debb3fca7ddc4374624acb8c70fc8292d860d:

  t/run-fio-tests: add t/nvmept_trim.py (2024-02-15 14:05:12 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 38d01cc56366aa2fd3af42dbab522888b6359dec:

  verify: fix integer sizes in verify state file (2024-02-16 09:25:35 -0500)

----------------------------------------------------------------
Vincent Fu (1):
      verify: fix integer sizes in verify state file

 verify.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/verify.c b/verify.c
index b438eed6..b2fede24 100644
--- a/verify.c
+++ b/verify.c
@@ -1619,8 +1619,8 @@ struct all_io_list *get_all_io_list(int save_mask, size_t *sz)
 		comps = fill_file_completions(td, s, &index);
 
 		s->no_comps = cpu_to_le64((uint64_t) comps);
-		s->depth = cpu_to_le64((uint64_t) td->o.iodepth);
-		s->nofiles = cpu_to_le64((uint64_t) td->o.nr_files);
+		s->depth = cpu_to_le32((uint32_t) td->o.iodepth);
+		s->nofiles = cpu_to_le32((uint32_t) td->o.nr_files);
 		s->numberio = cpu_to_le64((uint64_t) td->io_issues[DDIR_WRITE]);
 		s->index = cpu_to_le64((uint64_t) __td_index);
 		if (td->random_state.use64) {

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-16 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-16 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 7aec5ac0bdd1adcaeba707f26d5bc583de6ab6c9:

  test: add the test for loops option and read-verify workloads (2024-02-14 07:39:48 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 2d0debb3fca7ddc4374624acb8c70fc8292d860d:

  t/run-fio-tests: add t/nvmept_trim.py (2024-02-15 14:05:12 -0500)

----------------------------------------------------------------
Ankit Kumar (3):
      trim: add support for multiple ranges
      engines/nvme: pass offset and len instead of io_u
      engines/io_uring: add multi range dsm support

Jens Axboe (3):
      t/io_uring: account and ignore IO errors
      t/io_uring: pre-calculate per-file depth
      io_u: move number_trim to reclaim 8 bytes in struct io_u

Vincent Fu (2):
      t/nvmept_trim.py: test multi-range trim
      t/run-fio-tests: add t/nvmept_trim.py

 HOWTO.rst                               |   9 +
 backend.c                               |  20 +-
 cconv.c                                 |   2 +
 engines/io_uring.c                      |  34 +-
 engines/nvme.c                          |  72 ++--
 engines/nvme.h                          |   7 +-
 examples/uring-cmd-trim-multi-range.fio |  21 ++
 fio.1                                   |   7 +
 fio.h                                   |  18 +
 init.c                                  |  13 +
 io_u.c                                  |  97 +++++-
 io_u.h                                  |   5 +
 ioengines.h                             |   2 +
 options.c                               |  11 +
 server.h                                |   2 +-
 t/io_uring.c                            |  62 ++--
 t/nvmept_trim.py                        | 586 ++++++++++++++++++++++++++++++++
 t/run-fio-tests.py                      |   8 +
 thread_options.h                        |   3 +
 19 files changed, 899 insertions(+), 80 deletions(-)
 create mode 100644 examples/uring-cmd-trim-multi-range.fio
 create mode 100755 t/nvmept_trim.py

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 5bc1713c..4b02100c 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2534,6 +2534,15 @@ with the caveat that when used on the command line, they must come after the
 	Specifies logical block application tag mask value, if namespace is
 	formatted to use end to end protection information. Default: 0xffff.
 
+.. option:: num_range=int : [io_uring_cmd]
+
+	For trim command this will be the number of ranges to trim per I/O
+	request. The number of logical blocks per range is determined by the
+	:option:`bs` option which should be a multiple of logical block size.
+	This cannot be used with read or write. Note that setting this
+	option > 1, :option:`log_offset` will not be able to log all the
+	offsets. Default: 1.
+
 .. option:: cpuload=int : [cpuio]
 
 	Attempt to use the specified percentage of CPU cycles. This is a mandatory
diff --git a/backend.c b/backend.c
index 1fab467a..2f2221bf 100644
--- a/backend.c
+++ b/backend.c
@@ -1333,7 +1333,7 @@ static int init_io_u(struct thread_data *td)
 int init_io_u_buffers(struct thread_data *td)
 {
 	struct io_u *io_u;
-	unsigned long long max_bs, min_write;
+	unsigned long long max_bs, min_write, trim_bs = 0;
 	int i, max_units;
 	int data_xfer = 1;
 	char *p;
@@ -1344,7 +1344,18 @@ int init_io_u_buffers(struct thread_data *td)
 	td->orig_buffer_size = (unsigned long long) max_bs
 					* (unsigned long long) max_units;
 
-	if (td_ioengine_flagged(td, FIO_NOIO) || !(td_read(td) || td_write(td)))
+	if (td_trim(td) && td->o.num_range > 1) {
+		trim_bs = td->o.num_range * sizeof(struct trim_range);
+		td->orig_buffer_size = trim_bs
+					* (unsigned long long) max_units;
+	}
+
+	/*
+	 * For reads, writes, and multi-range trim operations we need a
+	 * data buffer
+	 */
+	if (td_ioengine_flagged(td, FIO_NOIO) ||
+	    !(td_read(td) || td_write(td) || (td_trim(td) && td->o.num_range > 1)))
 		data_xfer = 0;
 
 	/*
@@ -1396,7 +1407,10 @@ int init_io_u_buffers(struct thread_data *td)
 				fill_verify_pattern(td, io_u->buf, max_bs, io_u, 0, 0);
 			}
 		}
-		p += max_bs;
+		if (td_trim(td) && td->o.num_range > 1)
+			p += trim_bs;
+		else
+			p += max_bs;
 	}
 
 	return 0;
diff --git a/cconv.c b/cconv.c
index c9298408..ead47248 100644
--- a/cconv.c
+++ b/cconv.c
@@ -111,6 +111,7 @@ int convert_thread_options_to_cpu(struct thread_options *o,
 	o->serialize_overlap = le32_to_cpu(top->serialize_overlap);
 	o->size = le64_to_cpu(top->size);
 	o->io_size = le64_to_cpu(top->io_size);
+	o->num_range = le32_to_cpu(top->num_range);
 	o->size_percent = le32_to_cpu(top->size_percent);
 	o->io_size_percent = le32_to_cpu(top->io_size_percent);
 	o->fill_device = le32_to_cpu(top->fill_device);
@@ -609,6 +610,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 
 	top->size = __cpu_to_le64(o->size);
 	top->io_size = __cpu_to_le64(o->io_size);
+	top->num_range = __cpu_to_le32(o->num_range);
 	top->verify_backlog = __cpu_to_le64(o->verify_backlog);
 	top->start_delay = __cpu_to_le64(o->start_delay);
 	top->start_delay_high = __cpu_to_le64(o->start_delay_high);
diff --git a/engines/io_uring.c b/engines/io_uring.c
index c0cb5a78..9069fa3e 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -81,7 +81,7 @@ struct ioring_data {
 
 	struct cmdprio cmdprio;
 
-	struct nvme_dsm_range *dsm;
+	struct nvme_dsm *dsm;
 };
 
 struct ioring_options {
@@ -385,6 +385,9 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
 	struct fio_file *f = io_u->file;
 	struct nvme_uring_cmd *cmd;
 	struct io_uring_sqe *sqe;
+	struct nvme_dsm *dsm;
+	void *ptr = ld->dsm;
+	unsigned int dsm_size;
 
 	/* only supports nvme_uring_cmd */
 	if (o->cmd_type != FIO_URING_CMD_NVME)
@@ -423,9 +426,13 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
 	}
 
 	cmd = (struct nvme_uring_cmd *)sqe->cmd;
+	dsm_size = sizeof(*ld->dsm) + td->o.num_range * sizeof(struct nvme_dsm_range);
+	ptr += io_u->index * dsm_size;
+	dsm = (struct nvme_dsm *)ptr;
+
 	return fio_nvme_uring_cmd_prep(cmd, io_u,
 			o->nonvectored ? NULL : &ld->iovecs[io_u->index],
-			&ld->dsm[io_u->index]);
+			dsm);
 }
 
 static struct io_u *fio_ioring_event(struct thread_data *td, int event)
@@ -1133,8 +1140,11 @@ static int fio_ioring_init(struct thread_data *td)
 {
 	struct ioring_options *o = td->eo;
 	struct ioring_data *ld;
+	struct nvme_dsm *dsm;
+	void *ptr;
+	unsigned int dsm_size;
 	unsigned long long md_size;
-	int ret;
+	int ret, i;
 
 	/* sqthread submission requires registered files */
 	if (o->sqpoll_thread)
@@ -1195,10 +1205,19 @@ static int fio_ioring_init(struct thread_data *td)
 	 * in zbd mode where trim means zone reset.
 	 */
 	if (!strcmp(td->io_ops->name, "io_uring_cmd") && td_trim(td) &&
-	    td->o.zone_mode == ZONE_MODE_ZBD)
+	    td->o.zone_mode == ZONE_MODE_ZBD) {
 		td->io_ops->flags |= FIO_ASYNCIO_SYNC_TRIM;
-	else
-		ld->dsm = calloc(td->o.iodepth, sizeof(*ld->dsm));
+	} else {
+		dsm_size = sizeof(*ld->dsm) +
+			td->o.num_range * sizeof(struct nvme_dsm_range);
+		ld->dsm = calloc(td->o.iodepth, dsm_size);
+		ptr = ld->dsm;
+		for (i = 0; i < td->o.iodepth; i++) {
+			dsm = (struct nvme_dsm *)ptr;
+			dsm->nr_ranges = td->o.num_range;
+			ptr += dsm_size;
+		}
+	}
 
 	return 0;
 }
@@ -1466,7 +1485,8 @@ static struct ioengine_ops ioengine_uring_cmd = {
 	.name			= "io_uring_cmd",
 	.version		= FIO_IOOPS_VERSION,
 	.flags			= FIO_NO_OFFLOAD | FIO_MEMALIGN | FIO_RAWIO |
-					FIO_ASYNCIO_SETS_ISSUE_TIME,
+					FIO_ASYNCIO_SETS_ISSUE_TIME |
+					FIO_MULTI_RANGE_TRIM,
 	.init			= fio_ioring_init,
 	.post_init		= fio_ioring_cmd_post_init,
 	.io_u_init		= fio_ioring_io_u_init,
diff --git a/engines/nvme.c b/engines/nvme.c
index 75a5e0c1..c6629e86 100644
--- a/engines/nvme.c
+++ b/engines/nvme.c
@@ -8,20 +8,20 @@
 #include "../crc/crc-t10dif.h"
 #include "../crc/crc64.h"
 
-static inline __u64 get_slba(struct nvme_data *data, struct io_u *io_u)
+static inline __u64 get_slba(struct nvme_data *data, __u64 offset)
 {
 	if (data->lba_ext)
-		return io_u->offset / data->lba_ext;
-	else
-		return io_u->offset >> data->lba_shift;
+		return offset / data->lba_ext;
+
+	return offset >> data->lba_shift;
 }
 
-static inline __u32 get_nlb(struct nvme_data *data, struct io_u *io_u)
+static inline __u32 get_nlb(struct nvme_data *data, __u64 len)
 {
 	if (data->lba_ext)
-		return io_u->xfer_buflen / data->lba_ext - 1;
-	else
-		return (io_u->xfer_buflen >> data->lba_shift) - 1;
+		return len / data->lba_ext - 1;
+
+	return (len >> data->lba_shift) - 1;
 }
 
 static void fio_nvme_generate_pi_16b_guard(struct nvme_data *data,
@@ -32,8 +32,8 @@ static void fio_nvme_generate_pi_16b_guard(struct nvme_data *data,
 	struct nvme_16b_guard_pif *pi;
 	unsigned char *buf = io_u->xfer_buf;
 	unsigned char *md_buf = io_u->mmap_data;
-	__u64 slba = get_slba(data, io_u);
-	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u64 slba = get_slba(data, io_u->offset);
+	__u32 nlb = get_nlb(data, io_u->xfer_buflen) + 1;
 	__u32 lba_num = 0;
 	__u16 guard = 0;
 
@@ -99,8 +99,8 @@ static int fio_nvme_verify_pi_16b_guard(struct nvme_data *data,
 	struct fio_file *f = io_u->file;
 	unsigned char *buf = io_u->xfer_buf;
 	unsigned char *md_buf = io_u->mmap_data;
-	__u64 slba = get_slba(data, io_u);
-	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u64 slba = get_slba(data, io_u->offset);
+	__u32 nlb = get_nlb(data, io_u->xfer_buflen) + 1;
 	__u32 lba_num = 0;
 	__u16 unmask_app, unmask_app_exp, guard = 0;
 
@@ -185,8 +185,8 @@ static void fio_nvme_generate_pi_64b_guard(struct nvme_data *data,
 	unsigned char *buf = io_u->xfer_buf;
 	unsigned char *md_buf = io_u->mmap_data;
 	uint64_t guard = 0;
-	__u64 slba = get_slba(data, io_u);
-	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u64 slba = get_slba(data, io_u->offset);
+	__u32 nlb = get_nlb(data, io_u->xfer_buflen) + 1;
 	__u32 lba_num = 0;
 
 	if (data->pi_loc) {
@@ -251,9 +251,9 @@ static int fio_nvme_verify_pi_64b_guard(struct nvme_data *data,
 	struct fio_file *f = io_u->file;
 	unsigned char *buf = io_u->xfer_buf;
 	unsigned char *md_buf = io_u->mmap_data;
-	__u64 slba = get_slba(data, io_u);
+	__u64 slba = get_slba(data, io_u->offset);
 	__u64 ref, ref_exp, guard = 0;
-	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u32 nlb = get_nlb(data, io_u->xfer_buflen) + 1;
 	__u32 lba_num = 0;
 	__u16 unmask_app, unmask_app_exp;
 
@@ -329,24 +329,40 @@ next:
 	return 0;
 }
 void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
-				  struct nvme_dsm_range *dsm)
+				  struct nvme_dsm *dsm)
 {
 	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
+	struct trim_range *range;
+	uint8_t *buf_point;
+	int i;
 
 	cmd->opcode = nvme_cmd_dsm;
 	cmd->nsid = data->nsid;
-	cmd->cdw10 = 0;
 	cmd->cdw11 = NVME_ATTRIBUTE_DEALLOCATE;
-	cmd->addr = (__u64) (uintptr_t) dsm;
-	cmd->data_len = sizeof(*dsm);
-
-	dsm->slba = get_slba(data, io_u);
-	/* nlb is a 1-based value for deallocate */
-	dsm->nlb = get_nlb(data, io_u) + 1;
+	cmd->addr = (__u64) (uintptr_t) (&dsm->range[0]);
+
+	if (dsm->nr_ranges == 1) {
+		dsm->range[0].slba = get_slba(data, io_u->offset);
+		/* nlb is a 1-based value for deallocate */
+		dsm->range[0].nlb = get_nlb(data, io_u->xfer_buflen) + 1;
+		cmd->cdw10 = 0;
+		cmd->data_len = sizeof(struct nvme_dsm_range);
+	} else {
+		buf_point = io_u->xfer_buf;
+		for (i = 0; i < io_u->number_trim; i++) {
+			range = (struct trim_range *)buf_point;
+			dsm->range[i].slba = get_slba(data, range->start);
+			/* nlb is a 1-based value for deallocate */
+			dsm->range[i].nlb = get_nlb(data, range->len) + 1;
+			buf_point += sizeof(struct trim_range);
+		}
+		cmd->cdw10 = io_u->number_trim - 1;
+		cmd->data_len = io_u->number_trim * sizeof(struct nvme_dsm_range);
+	}
 }
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
-			    struct iovec *iov, struct nvme_dsm_range *dsm)
+			    struct iovec *iov, struct nvme_dsm *dsm)
 {
 	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
 	__u64 slba;
@@ -368,8 +384,8 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 		return -ENOTSUP;
 	}
 
-	slba = get_slba(data, io_u);
-	nlb = get_nlb(data, io_u);
+	slba = get_slba(data, io_u->offset);
+	nlb = get_nlb(data, io_u->xfer_buflen);
 
 	/* cdw10 and cdw11 represent starting lba */
 	cmd->cdw10 = slba & 0xffffffff;
@@ -400,7 +416,7 @@ void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
 	__u64 slba;
 
-	slba = get_slba(data, io_u);
+	slba = get_slba(data, io_u->offset);
 	cmd->cdw12 |= opts->io_flags;
 
 	if (data->pi_type && !(opts->io_flags & NVME_IO_PRINFO_PRACT)) {
diff --git a/engines/nvme.h b/engines/nvme.h
index 792b35d8..2d5204fc 100644
--- a/engines/nvme.h
+++ b/engines/nvme.h
@@ -408,6 +408,11 @@ struct nvme_dsm_range {
 	__le64	slba;
 };
 
+struct nvme_dsm {
+	__u32 nr_ranges;
+	struct nvme_dsm_range range[];
+};
+
 struct nvme_cmd_ext_io_opts {
 	__u32 io_flags;
 	__u16 apptag;
@@ -421,7 +426,7 @@ int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
 		      struct nvme_data *data);
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
-			    struct iovec *iov, struct nvme_dsm_range *dsm);
+			    struct iovec *iov, struct nvme_dsm *dsm);
 
 void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 		      struct nvme_cmd_ext_io_opts *opts);
diff --git a/examples/uring-cmd-trim-multi-range.fio b/examples/uring-cmd-trim-multi-range.fio
new file mode 100644
index 00000000..b376481b
--- /dev/null
+++ b/examples/uring-cmd-trim-multi-range.fio
@@ -0,0 +1,21 @@
+# Multi-range trim command test with io_uring_cmd I/O engine for nvme-ns
+# generic character device.
+#
+[global]
+filename=/dev/ng0n1
+ioengine=io_uring_cmd
+cmd_type=nvme
+size=10M
+iodepth=32
+thread=1
+stonewall=1
+
+[write_bs]
+bs=4096
+rw=randtrim
+num_range=8
+
+[write_bssplit]
+bssplit=4k/10:64k/50:32k/40
+rw=trim
+num_range=8
diff --git a/fio.1 b/fio.1
index 7ec5c745..e6b291a7 100644
--- a/fio.1
+++ b/fio.1
@@ -2293,6 +2293,13 @@ end to end protection information. Default: 0x1234.
 Specifies logical block application tag mask value, if namespace is formatted
 to use end to end protection information. Default: 0xffff.
 .TP
+.BI (io_uring_cmd)num_range \fR=\fPint
+For trim command this will be the number of ranges to trim per I/O request.
+The number of logical blocks per range is determined by the \fBbs\fR option
+which should be a multiple of logical block size. This cannot be used with
+read or write. Note that setting this option > 1, \fBlog_offset\fR will not be
+able to log all the offsets. Default: 1.
+.TP
 .BI (cpuio)cpuload \fR=\fPint
 Attempt to use the specified percentage of CPU cycles. This is a mandatory
 option when using cpuio I/O engine.
diff --git a/fio.h b/fio.h
index 1322656f..fc3e3ece 100644
--- a/fio.h
+++ b/fio.h
@@ -71,6 +71,16 @@
 
 struct fio_sem;
 
+#define MAX_TRIM_RANGE	256
+
+/*
+ * Range for trim command
+ */
+struct trim_range {
+	unsigned long long start;
+	unsigned long long len;
+};
+
 /*
  * offset generator types
  */
@@ -609,6 +619,14 @@ static inline void fio_ro_check(const struct thread_data *td, struct io_u *io_u)
 	       !(io_u->ddir == DDIR_TRIM && !td_trim(td)));
 }
 
+static inline bool multi_range_trim(struct thread_data *td, struct io_u *io_u)
+{
+	if (io_u->ddir == DDIR_TRIM && td->o.num_range > 1)
+		return true;
+
+	return false;
+}
+
 static inline bool should_fsync(struct thread_data *td)
 {
 	if (td->last_was_sync)
diff --git a/init.c b/init.c
index 105339fa..7a0b14a3 100644
--- a/init.c
+++ b/init.c
@@ -618,6 +618,19 @@ static int fixup_options(struct thread_data *td)
 		ret |= 1;
 	}
 
+	if (td_trimwrite(td) && o->num_range > 1) {
+		log_err("fio: trimwrite cannot be used with multiple"
+			" ranges.\n");
+		ret |= 1;
+	}
+
+	if (td_trim(td) && o->num_range > 1 &&
+	    !td_ioengine_flagged(td, FIO_MULTI_RANGE_TRIM)) {
+		log_err("fio: can't use multiple ranges with IO engine %s\n",
+			td->io_ops->name);
+		ret |= 1;
+	}
+
 #ifndef CONFIG_PSHARED
 	if (!o->use_thread) {
 		log_info("fio: this platform does not support process shared"
diff --git a/io_u.c b/io_u.c
index 4254675a..2b8e17f8 100644
--- a/io_u.c
+++ b/io_u.c
@@ -940,6 +940,65 @@ static void setup_strided_zone_mode(struct thread_data *td, struct io_u *io_u)
 		fio_file_reset(td, f);
 }
 
+static int fill_multi_range_io_u(struct thread_data *td, struct io_u *io_u)
+{
+	bool is_random;
+	uint64_t buflen, i = 0;
+	struct trim_range *range;
+	struct fio_file *f = io_u->file;
+	uint8_t *buf;
+
+	buf = io_u->buf;
+	buflen = 0;
+
+	while (i < td->o.num_range) {
+		range = (struct trim_range *)buf;
+		if (get_next_offset(td, io_u, &is_random)) {
+			dprint(FD_IO, "io_u %p, failed getting offset\n",
+			       io_u);
+			break;
+		}
+
+		io_u->buflen = get_next_buflen(td, io_u, is_random);
+		if (!io_u->buflen) {
+			dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
+			break;
+		}
+
+		if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
+			dprint(FD_IO, "io_u %p, off=0x%llx + len=0x%llx exceeds file size=0x%llx\n",
+			       io_u,
+			       (unsigned long long) io_u->offset, io_u->buflen,
+			       (unsigned long long) io_u->file->real_file_size);
+			break;
+		}
+
+		range->start = io_u->offset;
+		range->len = io_u->buflen;
+		buflen += io_u->buflen;
+		f->last_start[io_u->ddir] = io_u->offset;
+		f->last_pos[io_u->ddir] = io_u->offset + range->len;
+
+		buf += sizeof(struct trim_range);
+		i++;
+
+		if (td_random(td) && file_randommap(td, io_u->file))
+			mark_random_map(td, io_u, io_u->offset, io_u->buflen);
+		dprint_io_u(io_u, "fill");
+	}
+	if (buflen) {
+		/*
+		 * Set buffer length as overall trim length for this IO, and
+		 * tell the ioengine about the number of ranges to be trimmed.
+		 */
+		io_u->buflen = buflen;
+		io_u->number_trim = i;
+		return 0;
+	}
+
+	return 1;
+}
+
 static int fill_io_u(struct thread_data *td, struct io_u *io_u)
 {
 	bool is_random;
@@ -966,22 +1025,27 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u)
 	else if (td->o.zone_mode == ZONE_MODE_ZBD)
 		setup_zbd_zone_mode(td, io_u);
 
-	/*
-	 * No log, let the seq/rand engine retrieve the next buflen and
-	 * position.
-	 */
-	if (get_next_offset(td, io_u, &is_random)) {
-		dprint(FD_IO, "io_u %p, failed getting offset\n", io_u);
-		return 1;
-	}
+	if (multi_range_trim(td, io_u)) {
+		if (fill_multi_range_io_u(td, io_u))
+			return 1;
+	} else {
+		/*
+		 * No log, let the seq/rand engine retrieve the next buflen and
+		 * position.
+		 */
+		if (get_next_offset(td, io_u, &is_random)) {
+			dprint(FD_IO, "io_u %p, failed getting offset\n", io_u);
+			return 1;
+		}
 
-	io_u->buflen = get_next_buflen(td, io_u, is_random);
-	if (!io_u->buflen) {
-		dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
-		return 1;
+		io_u->buflen = get_next_buflen(td, io_u, is_random);
+		if (!io_u->buflen) {
+			dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
+			return 1;
+		}
 	}
-
 	offset = io_u->offset;
+
 	if (td->o.zone_mode == ZONE_MODE_ZBD) {
 		ret = zbd_adjust_block(td, io_u);
 		if (ret == io_u_eof) {
@@ -1004,11 +1068,12 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u)
 	/*
 	 * mark entry before potentially trimming io_u
 	 */
-	if (td_random(td) && file_randommap(td, io_u->file))
+	if (!multi_range_trim(td, io_u) && td_random(td) && file_randommap(td, io_u->file))
 		io_u->buflen = mark_random_map(td, io_u, offset, io_u->buflen);
 
 out:
-	dprint_io_u(io_u, "fill");
+	if (!multi_range_trim(td, io_u))
+		dprint_io_u(io_u, "fill");
 	io_u->verify_offset = io_u->offset;
 	td->zone_bytes += io_u->buflen;
 	return 0;
@@ -1814,7 +1879,7 @@ struct io_u *get_io_u(struct thread_data *td)
 
 	assert(fio_file_open(f));
 
-	if (ddir_rw(io_u->ddir)) {
+	if (ddir_rw(io_u->ddir) && !multi_range_trim(td, io_u)) {
 		if (!io_u->buflen && !td_ioengine_flagged(td, FIO_NOIO)) {
 			dprint(FD_IO, "get_io_u: zero buflen on %p\n", io_u);
 			goto err_put;
diff --git a/io_u.h b/io_u.h
index 786251d5..ab93d50f 100644
--- a/io_u.h
+++ b/io_u.h
@@ -52,6 +52,11 @@ struct io_u {
 	unsigned short ioprio;
 	unsigned short clat_prio_index;
 
+	/*
+	 * number of trim ranges for this IO.
+	 */
+	unsigned int number_trim;
+
 	/*
 	 * Allocated/set buffer and length
 	 */
diff --git a/ioengines.h b/ioengines.h
index 4391b31e..2fd7f52c 100644
--- a/ioengines.h
+++ b/ioengines.h
@@ -97,6 +97,8 @@ enum fio_ioengine_flags {
 	FIO_RO_NEEDS_RW_OPEN
 			= 1 << 18,	/* open files in rw mode even if we have a read job; only
 					   affects ioengines using generic_open_file */
+	FIO_MULTI_RANGE_TRIM
+			= 1 << 19,	/* ioengine supports trim with more than one range */
 };
 
 /*
diff --git a/options.c b/options.c
index 1da4de78..25e042d0 100644
--- a/options.c
+++ b/options.c
@@ -2395,6 +2395,17 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.category = FIO_OPT_C_IO,
 		.group	= FIO_OPT_G_INVALID,
 	},
+	{
+		.name	= "num_range",
+		.lname	= "Number of ranges",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct thread_options, num_range),
+		.maxval	= MAX_TRIM_RANGE,
+		.help	= "Number of ranges for trim command",
+		.def	= "1",
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_INVALID,
+	},
 	{
 		.name	= "bs",
 		.lname	= "Block size",
diff --git a/server.h b/server.h
index 0eb594ce..6d2659b0 100644
--- a/server.h
+++ b/server.h
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 102,
+	FIO_SERVER_VER			= 103,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,
diff --git a/t/io_uring.c b/t/io_uring.c
index efc50caa..46b153dc 100644
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -94,6 +94,7 @@ struct submitter {
 	unsigned long reaps;
 	unsigned long done;
 	unsigned long calls;
+	unsigned long io_errors;
 	volatile int finish;
 
 	__s32 *fds;
@@ -109,6 +110,7 @@ struct submitter {
 #endif
 
 	int numa_node;
+	int per_file_depth;
 	const char *filename;
 
 	struct file files[MAX_FDS];
@@ -490,11 +492,6 @@ static int io_uring_enter(struct submitter *s, unsigned int to_submit,
 #endif
 }
 
-static unsigned file_depth(struct submitter *s)
-{
-	return (depth + s->nr_files - 1) / s->nr_files;
-}
-
 static unsigned long long get_offset(struct submitter *s, struct file *f)
 {
 	unsigned long long offset;
@@ -516,7 +513,7 @@ static unsigned long long get_offset(struct submitter *s, struct file *f)
 	return offset;
 }
 
-static struct file *init_new_io(struct submitter *s)
+static struct file *get_next_file(struct submitter *s)
 {
 	struct file *f;
 
@@ -524,7 +521,7 @@ static struct file *init_new_io(struct submitter *s)
 		f = &s->files[0];
 	} else {
 		f = &s->files[s->cur_file];
-		if (f->pending_ios >= file_depth(s)) {
+		if (f->pending_ios >= s->per_file_depth) {
 			s->cur_file++;
 			if (s->cur_file == s->nr_files)
 				s->cur_file = 0;
@@ -546,7 +543,7 @@ static void init_io(struct submitter *s, unsigned index)
 		return;
 	}
 
-	f = init_new_io(s);
+	f = get_next_file(s);
 
 	if (register_files) {
 		sqe->flags = IOSQE_FIXED_FILE;
@@ -587,7 +584,7 @@ static void init_io_pt(struct submitter *s, unsigned index)
 	unsigned long long slba;
 	unsigned long long nlb;
 
-	f = init_new_io(s);
+	f = get_next_file(s);
 
 	offset = get_offset(s, f);
 
@@ -717,10 +714,14 @@ static int reap_events_uring(struct submitter *s)
 			f = &s->files[fileno];
 			f->pending_ios--;
 			if (cqe->res != bs) {
-				printf("io: unexpected ret=%d\n", cqe->res);
-				if (polled && cqe->res == -EOPNOTSUPP)
-					printf("Your filesystem/driver/kernel doesn't support polled IO\n");
-				return -1;
+				if (cqe->res == -ENODATA || cqe->res == -EIO) {
+					s->io_errors++;
+				} else {
+					printf("io: unexpected ret=%d\n", cqe->res);
+					if (polled && cqe->res == -EOPNOTSUPP)
+						printf("Your filesystem/driver/kernel doesn't support polled IO\n");
+					return -1;
+				}
 			}
 		}
 		if (stats) {
@@ -867,6 +868,7 @@ static int setup_aio(struct submitter *s)
 		fixedbufs = register_files = 0;
 	}
 
+	s->per_file_depth = (depth + s->nr_files - 1) / s->nr_files;
 	return io_queue_init(roundup_pow2(depth), &s->aio_ctx);
 #else
 	fprintf(stderr, "Legacy AIO not available on this system/build\n");
@@ -971,6 +973,7 @@ static int setup_ring(struct submitter *s)
 	for (i = 0; i < p.sq_entries; i++)
 		sring->array[i] = i;
 
+	s->per_file_depth = (depth + s->nr_files - 1) / s->nr_files;
 	return 0;
 }
 
@@ -997,8 +1000,8 @@ static int submitter_init(struct submitter *s)
 	static int init_printed;
 	char buf[80];
 	s->tid = gettid();
-	printf("submitter=%d, tid=%d, file=%s, node=%d\n", s->index, s->tid,
-							s->filename, s->numa_node);
+	printf("submitter=%d, tid=%d, file=%s, nfiles=%d, node=%d\n", s->index, s->tid,
+							s->filename, s->nr_files, s->numa_node);
 
 	set_affinity(s);
 
@@ -1077,7 +1080,7 @@ static int prep_more_ios_aio(struct submitter *s, int max_ios, struct iocb *iocb
 	while (index < max_ios) {
 		struct iocb *iocb = &iocbs[index];
 
-		f = init_new_io(s);
+		f = get_next_file(s);
 
 		io_prep_pread(iocb, f->real_fd, s->iovecs[index].iov_base,
 				s->iovecs[index].iov_len, get_offset(s, f));
@@ -1102,10 +1105,14 @@ static int reap_events_aio(struct submitter *s, struct io_event *events, int evs
 
 		f->pending_ios--;
 		if (events[reaped].res != bs) {
-			printf("io: unexpected ret=%ld\n", events[reaped].res);
-			return -1;
-		}
-		if (stats) {
+			if (events[reaped].res == -ENODATA ||
+			    events[reaped].res == -EIO) {
+				s->io_errors++;
+			} else {
+				printf("io: unexpected ret=%ld\n", events[reaped].res);
+				return -1;
+			}
+		} else if (stats) {
 			int clock_index = data >> 32;
 
 			if (last_idx != clock_index) {
@@ -1379,7 +1386,7 @@ static void *submitter_sync_fn(void *data)
 		uint64_t offset;
 		struct file *f;
 
-		f = init_new_io(s);
+		f = get_next_file(s);
 
 #ifdef ARCH_HAVE_CPU_CLOCK
 		if (stats)
@@ -1550,7 +1557,7 @@ static void write_tsc_rate(void)
 int main(int argc, char *argv[])
 {
 	struct submitter *s;
-	unsigned long done, calls, reap;
+	unsigned long done, calls, reap, io_errors;
 	int i, j, flags, fd, opt, threads_per_f, threads_rem = 0, nfiles;
 	struct file f;
 	void *ret;
@@ -1661,7 +1668,7 @@ int main(int argc, char *argv[])
 		s = get_submitter(j);
 		s->numa_node = -1;
 		s->index = j;
-		s->done = s->calls = s->reaps = 0;
+		s->done = s->calls = s->reaps = s->io_errors = 0;
 	}
 
 	flags = O_RDONLY | O_NOATIME;
@@ -1746,11 +1753,12 @@ int main(int argc, char *argv[])
 #endif
 	}
 
-	reap = calls = done = 0;
+	reap = calls = done = io_errors = 0;
 	do {
 		unsigned long this_done = 0;
 		unsigned long this_reap = 0;
 		unsigned long this_call = 0;
+		unsigned long this_io_errors = 0;
 		unsigned long rpc = 0, ipc = 0;
 		unsigned long iops, bw;
 
@@ -1771,6 +1779,7 @@ int main(int argc, char *argv[])
 			this_done += s->done;
 			this_call += s->calls;
 			this_reap += s->reaps;
+			this_io_errors += s->io_errors;
 		}
 		if (this_call - calls) {
 			rpc = (this_done - done) / (this_call - calls);
@@ -1778,6 +1787,7 @@ int main(int argc, char *argv[])
 		} else
 			rpc = ipc = -1;
 		iops = this_done - done;
+		iops -= this_io_errors - io_errors;
 		if (bs > 1048576)
 			bw = iops * (bs / 1048576);
 		else
@@ -1805,6 +1815,7 @@ int main(int argc, char *argv[])
 		done = this_done;
 		calls = this_call;
 		reap = this_reap;
+		io_errors = this_io_errors;
 	} while (!finish);
 
 	for (j = 0; j < nthreads; j++) {
@@ -1812,6 +1823,9 @@ int main(int argc, char *argv[])
 		pthread_join(s->thread, &ret);
 		close(s->ring_fd);
 
+		if (s->io_errors)
+			printf("%d: %lu IO errors\n", s->tid, s->io_errors);
+
 		if (stats) {
 			unsigned long nr;
 
diff --git a/t/nvmept_trim.py b/t/nvmept_trim.py
new file mode 100755
index 00000000..57568384
--- /dev/null
+++ b/t/nvmept_trim.py
@@ -0,0 +1,586 @@
+#!/usr/bin/env python3
+#
+# Copyright 2024 Samsung Electronics Co., Ltd All Rights Reserved
+#
+# For conditions of distribution and use, see the accompanying COPYING file.
+#
+"""
+# nvmept_trim.py
+#
+# Test fio's io_uring_cmd ioengine with NVMe pass-through dataset management
+# commands that trim multiple ranges.
+#
+# USAGE
+# see python3 nvmept_trim.py --help
+#
+# EXAMPLES
+# python3 t/nvmept_trim.py --dut /dev/ng0n1
+# python3 t/nvmept_trim.py --dut /dev/ng1n1 -f ./fio
+#
+# REQUIREMENTS
+# Python 3.6
+#
+"""
+import os
+import sys
+import time
+import logging
+import argparse
+from pathlib import Path
+from fiotestlib import FioJobCmdTest, run_fio_tests
+from fiotestcommon import SUCCESS_NONZERO
+
+
+class TrimTest(FioJobCmdTest):
+    """
+    NVMe pass-through test class. Check to make sure output for selected data
+    direction(s) is non-zero and that zero data appears for other directions.
+    """
+
+    def setup(self, parameters):
+        """Setup a test."""
+
+        fio_args = [
+            "--name=nvmept-trim",
+            "--ioengine=io_uring_cmd",
+            "--cmd_type=nvme",
+            f"--filename={self.fio_opts['filename']}",
+            f"--rw={self.fio_opts['rw']}",
+            f"--output={self.filenames['output']}",
+            f"--output-format={self.fio_opts['output-format']}",
+        ]
+        for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles',
+                    'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait',
+                    'time_based', 'runtime', 'verify', 'io_size', 'num_range',
+                    'iodepth', 'iodepth_batch', 'iodepth_batch_complete',
+                    'size', 'rate', 'bs', 'bssplit', 'bsrange', 'randrepeat',
+                    'buffer_pattern', 'verify_pattern', 'verify', 'offset']:
+            if opt in self.fio_opts:
+                option = f"--{opt}={self.fio_opts[opt]}"
+                fio_args.append(option)
+
+        super().setup(fio_args)
+
+
+    def check_result(self):
+
+        super().check_result()
+
+        if 'rw' not in self.fio_opts or \
+                not self.passed or \
+                'json' not in self.fio_opts['output-format']:
+            return
+
+        job = self.json_data['jobs'][0]
+
+        if self.fio_opts['rw'] in ['read', 'randread']:
+            self.passed = self.check_all_ddirs(['read'], job)
+        elif self.fio_opts['rw'] in ['write', 'randwrite']:
+            if 'verify' not in self.fio_opts:
+                self.passed = self.check_all_ddirs(['write'], job)
+            else:
+                self.passed = self.check_all_ddirs(['read', 'write'], job)
+        elif self.fio_opts['rw'] in ['trim', 'randtrim']:
+            self.passed = self.check_all_ddirs(['trim'], job)
+        elif self.fio_opts['rw'] in ['readwrite', 'randrw']:
+            self.passed = self.check_all_ddirs(['read', 'write'], job)
+        elif self.fio_opts['rw'] in ['trimwrite', 'randtrimwrite']:
+            self.passed = self.check_all_ddirs(['trim', 'write'], job)
+        else:
+            logging.error("Unhandled rw value %s", self.fio_opts['rw'])
+            self.passed = False
+
+        if 'iodepth' in self.fio_opts:
+            # We will need to figure something out if any test uses an iodepth
+            # different from 8
+            if job['iodepth_level']['8'] < 95:
+                logging.error("Did not achieve requested iodepth")
+                self.passed = False
+            else:
+                logging.debug("iodepth 8 target met %s", job['iodepth_level']['8'])
+
+
+class RangeTrimTest(TrimTest):
+    """
+    Multi-range trim test class.
+    """
+
+    def get_bs(self):
+        """Calculate block size and determine whether bs will be an average or exact."""
+
+        if 'bs' in self.fio_opts:
+            exact_size = True
+            bs = self.fio_opts['bs']
+        elif 'bssplit' in self.fio_opts:
+            exact_size = False
+            bs = 0
+            total = 0
+            for split in self.fio_opts['bssplit'].split(':'):
+                [blocksize, share] = split.split('/')
+                total += int(share)
+                bs += int(blocksize) * int(share) / 100
+            if total != 100:
+                logging.error("bssplit '%s' total percentage is not 100", self.fio_opts['bssplit'])
+                self.passed = False
+            else:
+                logging.debug("bssplit: average block size is %d", int(bs))
+            # The only check we do here for bssplit is to calculate an average
+            # blocksize and see if the IOPS and bw are consistent
+        elif 'bsrange' in self.fio_opts:
+            exact_size = False
+            [minbs, maxbs] = self.fio_opts['bsrange'].split('-')
+            minbs = int(minbs)
+            maxbs = int(maxbs)
+            bs = int((minbs + maxbs) / 2)
+            logging.debug("bsrange: average block size is %d", int(bs))
+            # The only check we do here for bsrange is to calculate an average
+            # blocksize and see if the IOPS and bw are consistent
+        else:
+            exact_size = True
+            bs = 4096
+
+        return bs, exact_size
+
+
+    def check_result(self):
+        """
+        Make sure that the number of IO requests is consistent with the
+        blocksize and num_range values. In other words, if the blocksize is
+        4KiB and num_range is 2, we should have 128 IO requests to trim 1MiB.
+        """
+        # TODO Enable debug output to check the actual offsets
+
+        super().check_result()
+
+        if not self.passed or 'json' not in self.fio_opts['output-format']:
+            return
+
+        job = self.json_data['jobs'][0]['trim']
+        bs, exact_size = self.get_bs()
+
+        # make sure bw and IOPS are consistent
+        bw = job['bw_bytes']
+        iops = job['iops']
+        runtime = job['runtime']
+
+        calculated = int(bw*runtime/1000)
+        expected = job['io_bytes']
+        if abs(calculated - expected) / expected > 0.05:
+            logging.error("Total bytes %d from bw does not match reported total bytes %d",
+                          calculated, expected)
+            self.passed = False
+        else:
+            logging.debug("Total bytes %d from bw matches reported total bytes %d", calculated,
+                          expected)
+
+        calculated = int(iops*runtime/1000*bs*self.fio_opts['num_range'])
+        if abs(calculated - expected) / expected > 0.05:
+            logging.error("Total bytes %d from IOPS does not match reported total bytes %d",
+                          calculated, expected)
+            self.passed = False
+        else:
+            logging.debug("Total bytes %d from IOPS matches reported total bytes %d", calculated,
+                          expected)
+
+        if 'size' in self.fio_opts:
+            io_count = self.fio_opts['size'] / self.fio_opts['num_range'] / bs
+            if exact_size:
+                delta = 0.1
+            else:
+                delta = 0.05*job['total_ios']
+
+            if abs(job['total_ios'] - io_count) > delta:
+                logging.error("Expected numbers of IOs %d does not match actual value %d",
+                              io_count, job['total_ios'])
+                self.passed = False
+            else:
+                logging.debug("Expected numbers of IOs %d matches actual value %d", io_count,
+                              job['total_ios'])
+
+        if 'rate' in self.fio_opts:
+            if abs(bw - self.fio_opts['rate']) / self.fio_opts['rate'] > 0.05:
+                logging.error("Actual rate %f does not match expected rate %f", bw,
+                              self.fio_opts['rate'])
+                self.passed = False
+            else:
+                logging.debug("Actual rate %f matches expeected rate %f", bw, self.fio_opts['rate'])
+
+
+
+TEST_LIST = [
+    # The group of tests below checks existing use cases to make sure there are
+    # no regressions.
+    {
+        "test_id": 1,
+        "fio_opts": {
+            "rw": 'trim',
+            "time_based": 1,
+            "runtime": 3,
+            "output-format": "json",
+            },
+        "test_class": TrimTest,
+    },
+    {
+        "test_id": 2,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "time_based": 1,
+            "runtime": 3,
+            "output-format": "json",
+            },
+        "test_class": TrimTest,
+    },
+    {
+        "test_id": 3,
+        "fio_opts": {
+            "rw": 'trim',
+            "time_based": 1,
+            "runtime": 3,
+            "iodepth": 8,
+            "iodepth_batch": 4,
+            "iodepth_batch_complete": 4,
+            "output-format": "json",
+            },
+        "test_class": TrimTest,
+    },
+    {
+        "test_id": 4,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "time_based": 1,
+            "runtime": 3,
+            "iodepth": 8,
+            "iodepth_batch": 4,
+            "iodepth_batch_complete": 4,
+            "output-format": "json",
+            },
+        "test_class": TrimTest,
+    },
+    {
+        "test_id": 5,
+        "fio_opts": {
+            "rw": 'trimwrite',
+            "time_based": 1,
+            "runtime": 3,
+            "output-format": "json",
+            },
+        "test_class": TrimTest,
+    },
+    {
+        "test_id": 6,
+        "fio_opts": {
+            "rw": 'randtrimwrite',
+            "time_based": 1,
+            "runtime": 3,
+            "output-format": "json",
+            },
+        "test_class": TrimTest,
+    },
+    {
+        "test_id": 7,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "time_based": 1,
+            "runtime": 3,
+            "fixedbufs": 0,
+            "nonvectored": 1,
+            "force_async": 1,
+            "registerfiles": 1,
+            "sqthread_poll": 1,
+            "fixedbuffs": 1,
+            "output-format": "json",
+            },
+        "test_class": TrimTest,
+    },
+    # The group of tests below try out the new functionality
+    {
+        "test_id": 100,
+        "fio_opts": {
+            "rw": 'trim',
+            "num_range": 2,
+            "size": 16*1024*1024,
+            "output-format": "json",
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 101,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 2,
+            "size": 16*1024*1024,
+            "output-format": "json",
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 102,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 256,
+            "size": 64*1024*1024,
+            "output-format": "json",
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 103,
+        "fio_opts": {
+            "rw": 'trim',
+            "num_range": 2,
+            "bs": 16*1024,
+            "size": 32*1024*1024,
+            "output-format": "json",
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 104,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 2,
+            "bs": 16*1024,
+            "size": 32*1024*1024,
+            "output-format": "json",
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 105,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 2,
+            "bssplit": "4096/50:16384/50",
+            "size": 80*1024*1024,
+            "output-format": "json",
+            "randrepeat": 0,
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 106,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 4,
+            "bssplit": "4096/25:8192/25:12288/25:16384/25",
+            "size": 80*1024*1024,
+            "output-format": "json",
+            "randrepeat": 0,
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 107,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 4,
+            "bssplit": "4096/20:8192/20:12288/20:16384/20:20480/20",
+            "size": 72*1024*1024,
+            "output-format": "json",
+            "randrepeat": 0,
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 108,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 2,
+            "bsrange": "4096-16384",
+            "size": 80*1024*1024,
+            "output-format": "json",
+            "randrepeat": 0,
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 109,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 4,
+            "bsrange": "4096-20480",
+            "size": 72*1024*1024,
+            "output-format": "json",
+            "randrepeat": 0,
+            },
+        "test_class": RangeTrimTest,
+    },
+    {
+        "test_id": 110,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "time_based": 1,
+            "runtime": 10,
+            "rate": 1024*1024,
+            "num_range": 2,
+            "output-format": "json",
+            },
+        "test_class": RangeTrimTest,
+    },
+    # All of the tests below should fail
+    # TODO check the error messages resulting from the jobs below
+    {
+        "test_id": 200,
+        "fio_opts": {
+            "rw": 'randtrimwrite',
+            "time_based": 1,
+            "runtime": 10,
+            "rate": 1024*1024,
+            "num_range": 2,
+            "output-format": "normal",
+            },
+        "test_class": RangeTrimTest,
+        "success": SUCCESS_NONZERO,
+    },
+    {
+        "test_id": 201,
+        "fio_opts": {
+            "rw": 'trimwrite',
+            "time_based": 1,
+            "runtime": 10,
+            "rate": 1024*1024,
+            "num_range": 2,
+            "output-format": "normal",
+            },
+        "test_class": RangeTrimTest,
+        "success": SUCCESS_NONZERO,
+    },
+    {
+        "test_id": 202,
+        "fio_opts": {
+            "rw": 'trim',
+            "time_based": 1,
+            "runtime": 10,
+            "num_range": 257,
+            "output-format": "normal",
+            },
+        "test_class": RangeTrimTest,
+        "success": SUCCESS_NONZERO,
+    },
+    # The sequence of jobs below constitute a single test with multiple steps
+    # - write a data pattern
+    # - verify the data pattern
+    # - trim the first half of the LBA space
+    # - verify that the trim'd LBA space no longer returns the original data pattern
+    # - verify that the remaining LBA space has the expected pattern
+    {
+        "test_id": 300,
+        "fio_opts": {
+            "rw": 'write',
+            "output-format": 'json',
+            "buffer_pattern": 0x0f,
+            "size": 256*1024*1024,
+            },
+        "test_class": TrimTest,
+    },
+    {
+        "test_id": 301,
+        "fio_opts": {
+            "rw": 'read',
+            "output-format": 'json',
+            "verify_pattern": 0x0f,
+            "verify": "pattern",
+            "size": 256*1024*1024,
+            },
+        "test_class": TrimTest,
+    },
+    {
+        "test_id": 302,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "num_range": 8,
+            "output-format": 'json',
+            "size": 128*1024*1024,
+            },
+        "test_class": TrimTest,
+    },
+    # The identify namespace data structure has a DLFEAT field which specifies
+    # what happens when reading data from deallocated blocks. There are three
+    # options:
+    # - read behavior not reported
+    # - deallocated logical block returns all bytes 0x0
+    # - deallocated logical block returns all bytes 0xff
+    # The test below merely checks that the original data pattern is not returned.
+    # Source: Figure 97 from
+    # https://nvmexpress.org/wp-content/uploads/NVM-Express-NVM-Command-Set-Specification-1.0c-2022.10.03-Ratified.pdf
+    {
+        "test_id": 303,
+        "fio_opts": {
+            "rw": 'read',
+            "output-format": 'json',
+            "verify_pattern": 0x0f,
+            "verify": "pattern",
+            "size": 128*1024*1024,
+            },
+        "test_class": TrimTest,
+        "success": SUCCESS_NONZERO,
+    },
+    {
+        "test_id": 304,
+        "fio_opts": {
+            "rw": 'read',
+            "output-format": 'json',
+            "verify_pattern": 0x0f,
+            "verify": "pattern",
+            "offset": 128*1024*1024,
+            "size": 128*1024*1024,
+            },
+        "test_class": TrimTest,
+    },
+]
+
+def parse_args():
+    """Parse command-line arguments."""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-d', '--debug', help='Enable debug messages', action='store_true')
+    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
+    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
+    parser.add_argument('-s', '--skip', nargs='+', type=int,
+                        help='list of test(s) to skip')
+    parser.add_argument('-o', '--run-only', nargs='+', type=int,
+                        help='list of test(s) to run, skipping all others')
+    parser.add_argument('--dut', help='target NVMe character device to test '
+                        '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
+    args = parser.parse_args()
+
+    return args
+
+
+def main():
+    """Run tests using fio's io_uring_cmd ioengine to send NVMe pass through commands."""
+
+    args = parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    artifact_root = args.artifact_root if args.artifact_root else \
+        f"nvmept-trim-test-{time.strftime('%Y%m%d-%H%M%S')}"
+    os.mkdir(artifact_root)
+    print(f"Artifact directory is {artifact_root}")
+
+    if args.fio:
+        fio_path = str(Path(args.fio).absolute())
+    else:
+        fio_path = 'fio'
+    print(f"fio path is {fio_path}")
+
+    for test in TEST_LIST:
+        test['fio_opts']['filename'] = args.dut
+
+    test_env = {
+              'fio_path': fio_path,
+              'fio_root': str(Path(__file__).absolute().parent.parent),
+              'artifact_root': artifact_root,
+              'basename': 'nvmept-trim',
+              }
+
+    _, failed, _ = run_fio_tests(TEST_LIST, test_env, args)
+    sys.exit(failed)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index 2f76d3fc..d4742e96 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -981,6 +981,14 @@ TEST_LIST = [
         'success':          SUCCESS_DEFAULT,
         'requirements':     [Requirements.linux, Requirements.nvmecdev],
     },
+    {
+        'test_id':          1015,
+        'test_class':       FioExeTest,
+        'exe':              't/nvmept_trim.py',
+        'parameters':       ['-f', '{fio_path}', '--dut', '{nvmecdev}'],
+        'success':          SUCCESS_DEFAULT,
+        'requirements':     [Requirements.linux, Requirements.nvmecdev],
+    },
 ]
 
 
diff --git a/thread_options.h b/thread_options.h
index 24f695fe..c2e71518 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -353,6 +353,8 @@ struct thread_options {
 	unsigned long long offset_increment;
 	unsigned long long number_ios;
 
+	unsigned int num_range;
+
 	unsigned int sync_file_range;
 
 	unsigned long long latency_target;
@@ -711,6 +713,7 @@ struct thread_options_pack {
 	uint32_t fdp_plis[FIO_MAX_PLIS];
 	uint32_t fdp_nrpli;
 
+	uint32_t num_range;
 	/*
 	 * verify_pattern followed by buffer_pattern from the unpacked struct
 	 */

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-15 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-15 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit a99bd37f690ab246caa9b9d65adfa65a25967190:

  examples: add PI example with xnvme ioengine (2024-02-13 14:24:59 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 7aec5ac0bdd1adcaeba707f26d5bc583de6ab6c9:

  test: add the test for loops option and read-verify workloads (2024-02-14 07:39:48 -0700)

----------------------------------------------------------------
Shin'ichiro Kawasaki (2):
      verify: fix loops option behavior of read-verify workloads
      test: add the test for loops option and read-verify workloads

 io_u.c             |  3 ++-
 t/jobs/t0029.fio   | 14 ++++++++++++++
 t/run-fio-tests.py | 21 +++++++++++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100644 t/jobs/t0029.fio

---

Diff of recent changes:

diff --git a/io_u.c b/io_u.c
index 13187882..4254675a 100644
--- a/io_u.c
+++ b/io_u.c
@@ -2151,7 +2151,8 @@ static void io_u_update_bytes_done(struct thread_data *td,
 
 	if (td->runstate == TD_VERIFYING) {
 		td->bytes_verified += icd->bytes_done[DDIR_READ];
-		return;
+		if (td_write(td))
+			return;
 	}
 
 	for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++)
diff --git a/t/jobs/t0029.fio b/t/jobs/t0029.fio
new file mode 100644
index 00000000..481de6f3
--- /dev/null
+++ b/t/jobs/t0029.fio
@@ -0,0 +1,14 @@
+[global]
+filename=t0029file
+size=4k
+verify=md5
+
+[write]
+rw=write
+do_verify=0
+
+[read]
+stonewall=1
+rw=read
+loops=2
+do_verify=1
diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index 1448f7cb..2f76d3fc 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -542,6 +542,17 @@ class FioJobFileTest_t0027(FioJobFileTest):
         if data != self.pattern:
             self.passed = False
 
+class FioJobFileTest_t0029(FioJobFileTest):
+    """Test loops option works with read-verify workload."""
+    def check_result(self):
+        super().check_result()
+
+        if not self.passed:
+            return
+
+        if self.json_data['jobs'][1]['read']['io_kbytes'] != 8:
+            self.passed = False
+
 class FioJobFileTest_iops_rate(FioJobFileTest):
     """Test consists of fio test job t0011
     Confirm that job0 iops == 1000
@@ -838,6 +849,16 @@ TEST_LIST = [
         'pre_success':      None,
         'requirements':     [],
     },
+    {
+        'test_id':          29,
+        'test_class':       FioJobFileTest_t0029,
+        'job':              't0029.fio',
+        'success':          SUCCESS_DEFAULT,
+        'pre_job':          None,
+        'pre_success':      None,
+        'output_format':    'json',
+        'requirements':     [],
+    },
     {
         'test_id':          1000,
         'test_class':       FioExeTest,

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-14 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-14 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 097af663e93f86106f32580bfa59e68fae007035:

  Merge branch 'vsock' of https://github.com/MPinna/fio (2024-02-12 11:56:33 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to a99bd37f690ab246caa9b9d65adfa65a25967190:

  examples: add PI example with xnvme ioengine (2024-02-13 14:24:59 -0500)

----------------------------------------------------------------
Ankit Kumar (5):
      engines/xnvme: allocate iovecs only if vectored I/O is enabled
      engines/xnvme: add support for metadata
      engines:xnvme: add support for end to end data protection
      engines/xnvme: add checks for verify, block size and metadata size
      examples: add PI example with xnvme ioengine

Vincent Fu (4):
      logging: record timestamp for each thread
      helper_thread: do not send A_EXIT message when exit is called
      logging: expand runstates eligible for logging
      docs: explain duplicate logging timestamps

 HOWTO.rst             |  25 ++--
 configure             |   2 +-
 engines/xnvme.c       | 331 +++++++++++++++++++++++++++++++++++++++++++++++---
 examples/xnvme-pi.fio |  53 ++++++++
 fio.1                 |  24 ++--
 helper_thread.c       |   1 -
 stat.c                |  24 +++-
 7 files changed, 414 insertions(+), 46 deletions(-)
 create mode 100644 examples/xnvme-pi.fio

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 53b03021..5bc1713c 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2491,11 +2491,11 @@ with the caveat that when used on the command line, they must come after the
         want fio to use placement identifier only at indices 0, 2 and 5 specify
         ``fdp_pli=0,2,5``.
 
-.. option:: md_per_io_size=int : [io_uring_cmd]
+.. option:: md_per_io_size=int : [io_uring_cmd] [xnvme]
 
 	Size in bytes for separate metadata buffer per IO. Default: 0.
 
-.. option:: pi_act=int : [io_uring_cmd]
+.. option:: pi_act=int : [io_uring_cmd] [xnvme]
 
 	Action to take when nvme namespace is formatted with protection
 	information. If this is set to 1 and namespace is formatted with
@@ -2511,7 +2511,7 @@ with the caveat that when used on the command line, they must come after the
 	it will use the default slower generator.
 	(see: https://github.com/intel/isa-l)
 
-.. option:: pi_chk=str[,str][,str] : [io_uring_cmd]
+.. option:: pi_chk=str[,str][,str] : [io_uring_cmd] [xnvme]
 
 	Controls the protection information check. This can take one or more
 	of these values. Default: none.
@@ -2524,12 +2524,12 @@ with the caveat that when used on the command line, they must come after the
 	**APPTAG**
 		Enables protection information checking of application tag field.
 
-.. option:: apptag=int : [io_uring_cmd]
+.. option:: apptag=int : [io_uring_cmd] [xnvme]
 
 	Specifies logical block application tag value, if namespace is
 	formatted to use end to end protection information. Default: 0x1234.
 
-.. option:: apptag_mask=int : [io_uring_cmd]
+.. option:: apptag_mask=int : [io_uring_cmd] [xnvme]
 
 	Specifies logical block application tag mask value, if namespace is
 	formatted to use end to end protection information. Default: 0xffff.
@@ -4066,12 +4066,15 @@ Measurements and reporting
 
 .. option:: log_avg_msec=int
 
-	By default, fio will log an entry in the iops, latency, or bw log for every
-	I/O that completes. When writing to the disk log, that can quickly grow to a
-	very large size. Setting this option makes fio average the each log entry
-	over the specified period of time, reducing the resolution of the log.  See
-	:option:`log_window_value` as well. Defaults to 0, logging all entries.
-	Also see `Log File Formats`_.
+        By default, fio will log an entry in the iops, latency, or bw log for
+        every I/O that completes. When writing to the disk log, that can
+        quickly grow to a very large size. Setting this option directs fio to
+        instead record an average over the specified duration for each log
+        entry, reducing the resolution of the log. When the job completes, fio
+        will flush any accumulated latency log data, so the final log interval
+        may not match the value specified by this option and there may even be
+        duplicate timestamps. See :option:`log_window_value` as well. Defaults
+        to 0, logging entries for each I/O. Also see `Log File Formats`_.
 
 .. option:: log_hist_msec=int
 
diff --git a/configure b/configure
index becb193e..3eef022b 100755
--- a/configure
+++ b/configure
@@ -2697,7 +2697,7 @@ fi
 ##########################################
 # Check if we have xnvme
 if test "$xnvme" != "no" ; then
-  if check_min_lib_version xnvme 0.7.0; then
+  if check_min_lib_version xnvme 0.7.4; then
     xnvme="yes"
     xnvme_cflags=$(pkg-config --cflags xnvme)
     xnvme_libs=$(pkg-config --libs xnvme)
diff --git a/engines/xnvme.c b/engines/xnvme.c
index 2a0b3520..a8137286 100644
--- a/engines/xnvme.c
+++ b/engines/xnvme.c
@@ -11,6 +11,7 @@
 #include <assert.h>
 #include <libxnvme.h>
 #include "fio.h"
+#include "verify.h"
 #include "zbd_types.h"
 #include "fdp.h"
 #include "optgroup.h"
@@ -30,8 +31,10 @@ struct xnvme_fioe_fwrap {
 
 	uint32_t ssw;
 	uint32_t lba_nbytes;
+	uint32_t md_nbytes;
+	uint32_t lba_pow2;
 
-	uint8_t _pad[24];
+	uint8_t _pad[16];
 };
 XNVME_STATIC_ASSERT(sizeof(struct xnvme_fioe_fwrap) == 64, "Incorrect size")
 
@@ -58,19 +61,31 @@ struct xnvme_fioe_data {
 	uint64_t nallocated;
 
 	struct iovec *iovec;
-
-	uint8_t _pad[8];
+	struct iovec *md_iovec;
 
 	struct xnvme_fioe_fwrap files[];
 };
 XNVME_STATIC_ASSERT(sizeof(struct xnvme_fioe_data) == 64, "Incorrect size")
 
+struct xnvme_fioe_request {
+	/* Context for NVMe PI */
+	struct xnvme_pi_ctx pi_ctx;
+
+	/* Separate metadata buffer pointer */
+	void *md_buf;
+};
+
 struct xnvme_fioe_options {
 	void *padding;
 	unsigned int hipri;
 	unsigned int sqpoll_thread;
 	unsigned int xnvme_dev_nsid;
 	unsigned int xnvme_iovec;
+	unsigned int md_per_io_size;
+	unsigned int pi_act;
+	unsigned int apptag;
+	unsigned int apptag_mask;
+	unsigned int prchk;
 	char *xnvme_be;
 	char *xnvme_mem;
 	char *xnvme_async;
@@ -79,6 +94,20 @@ struct xnvme_fioe_options {
 	char *xnvme_dev_subnqn;
 };
 
+static int str_pi_chk_cb(void *data, const char *str)
+{
+	struct xnvme_fioe_options *o = data;
+
+	if (strstr(str, "GUARD") != NULL)
+		o->prchk = XNVME_PI_FLAGS_GUARD_CHECK;
+	if (strstr(str, "REFTAG") != NULL)
+		o->prchk |= XNVME_PI_FLAGS_REFTAG_CHECK;
+	if (strstr(str, "APPTAG") != NULL)
+		o->prchk |= XNVME_PI_FLAGS_APPTAG_CHECK;
+
+	return 0;
+}
+
 static struct fio_option options[] = {
 	{
 		.name = "hipri",
@@ -171,6 +200,56 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group = FIO_OPT_G_XNVME,
 	},
+	{
+		.name	= "md_per_io_size",
+		.lname	= "Separate Metadata Buffer Size per I/O",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct xnvme_fioe_options, md_per_io_size),
+		.def	= "0",
+		.help	= "Size of separate metadata buffer per I/O (Default: 0)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_XNVME,
+	},
+	{
+		.name	= "pi_act",
+		.lname	= "Protection Information Action",
+		.type	= FIO_OPT_BOOL,
+		.off1	= offsetof(struct xnvme_fioe_options, pi_act),
+		.def	= "1",
+		.help	= "Protection Information Action bit (pi_act=1 or pi_act=0)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_XNVME,
+	},
+	{
+		.name	= "pi_chk",
+		.lname	= "Protection Information Check",
+		.type	= FIO_OPT_STR_STORE,
+		.def	= NULL,
+		.help	= "Control of Protection Information Checking (pi_chk=GUARD,REFTAG,APPTAG)",
+		.cb	= str_pi_chk_cb,
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_XNVME,
+	},
+	{
+		.name	= "apptag",
+		.lname	= "Application Tag used in Protection Information",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct xnvme_fioe_options, apptag),
+		.def	= "0x1234",
+		.help	= "Application Tag used in Protection Information field (Default: 0x1234)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_XNVME,
+	},
+	{
+		.name	= "apptag_mask",
+		.lname	= "Application Tag Mask",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct xnvme_fioe_options, apptag_mask),
+		.def	= "0xffff",
+		.help	= "Application Tag Mask used with Application Tag (Default: 0xffff)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_XNVME,
+	},
 
 	{
 		.name = NULL,
@@ -181,6 +260,10 @@ static void cb_pool(struct xnvme_cmd_ctx *ctx, void *cb_arg)
 {
 	struct io_u *io_u = cb_arg;
 	struct xnvme_fioe_data *xd = io_u->mmap_data;
+	struct xnvme_fioe_request *fio_req = io_u->engine_data;
+	struct xnvme_fioe_fwrap *fwrap = &xd->files[io_u->file->fileno];
+	bool pi_act = (fio_req->pi_ctx.pi_flags >> 3);
+	int err;
 
 	if (xnvme_cmd_ctx_cpl_status(ctx)) {
 		xnvme_cmd_ctx_pr(ctx, XNVME_PR_DEF);
@@ -188,6 +271,15 @@ static void cb_pool(struct xnvme_cmd_ctx *ctx, void *cb_arg)
 		io_u->error = EIO;
 	}
 
+	if (!io_u->error && fwrap->geo->pi_type && (io_u->ddir == DDIR_READ) && !pi_act) {
+		err = xnvme_pi_verify(&fio_req->pi_ctx, io_u->xfer_buf,
+				      fio_req->md_buf, io_u->xfer_buflen / fwrap->lba_nbytes);
+		if (err) {
+			xd->ecount += 1;
+			io_u->error = EIO;
+		}
+	}
+
 	xd->iocq[xd->completed++] = io_u;
 	xnvme_queue_put_cmd_ctx(ctx->async.queue, ctx);
 }
@@ -249,10 +341,54 @@ static void xnvme_fioe_cleanup(struct thread_data *td)
 
 	free(xd->iocq);
 	free(xd->iovec);
+	free(xd->md_iovec);
 	free(xd);
 	td->io_ops_data = NULL;
 }
 
+static int _verify_options(struct thread_data *td, struct fio_file *f,
+			   struct xnvme_fioe_fwrap *fwrap)
+{
+	struct xnvme_fioe_options *o = td->eo;
+	unsigned int correct_md_size;
+
+	for_each_rw_ddir(ddir) {
+		if (td->o.min_bs[ddir] % fwrap->lba_nbytes || td->o.max_bs[ddir] % fwrap->lba_nbytes) {
+			if (!fwrap->lba_pow2) {
+				log_err("ioeng->_verify_options(%s): block size must be a multiple of %u "
+					"(LBA data size + Metadata size)\n", f->file_name, fwrap->lba_nbytes);
+			} else {
+				log_err("ioeng->_verify_options(%s): block size must be a multiple of LBA data size\n",
+					f->file_name);
+			}
+			return 1;
+		}
+		if (ddir == DDIR_TRIM)
+			continue;
+
+		correct_md_size = (td->o.max_bs[ddir] / fwrap->lba_nbytes) * fwrap->md_nbytes;
+		if (fwrap->md_nbytes && fwrap->lba_pow2 && (o->md_per_io_size < correct_md_size)) {
+			log_err("ioeng->_verify_options(%s): md_per_io_size should be at least %u bytes\n",
+				f->file_name, correct_md_size);
+			return 1;
+		}
+	}
+
+	/*
+	 * For extended logical block sizes we cannot use verify when
+	 * end to end data protection checks are enabled, as the PI
+	 * section of data buffer conflicts with verify.
+	 */
+	if (fwrap->md_nbytes && fwrap->geo->pi_type && !fwrap->lba_pow2 &&
+	    td->o.verify != VERIFY_NONE) {
+		log_err("ioeng->_verify_options(%s): for extended LBA, verify cannot be used when E2E data protection is enabled\n",
+			f->file_name);
+		return 1;
+	}
+
+	return 0;
+}
+
 /**
  * Helper function setting up device handles as addressed by the naming
  * convention of the given `fio_file` filename.
@@ -263,6 +399,7 @@ static void xnvme_fioe_cleanup(struct thread_data *td)
 static int _dev_open(struct thread_data *td, struct fio_file *f)
 {
 	struct xnvme_opts opts = xnvme_opts_from_fioe(td);
+	struct xnvme_fioe_options *o = td->eo;
 	struct xnvme_fioe_data *xd = td->io_ops_data;
 	struct xnvme_fioe_fwrap *fwrap;
 	int flags = 0;
@@ -297,6 +434,31 @@ static int _dev_open(struct thread_data *td, struct fio_file *f)
 
 	fwrap->ssw = xnvme_dev_get_ssw(fwrap->dev);
 	fwrap->lba_nbytes = fwrap->geo->lba_nbytes;
+	fwrap->md_nbytes = fwrap->geo->nbytes_oob;
+
+	if (fwrap->geo->lba_extended)
+		fwrap->lba_pow2 = 0;
+	else
+		fwrap->lba_pow2 = 1;
+
+	/*
+	 * When PI action is set and PI size is equal to metadata size, the
+	 * controller inserts/removes PI. So update the LBA data and metadata
+	 * sizes accordingly.
+	 */
+	if (o->pi_act && fwrap->geo->pi_type &&
+	    fwrap->geo->nbytes_oob == xnvme_pi_size(fwrap->geo->pi_format)) {
+		if (fwrap->geo->lba_extended) {
+			fwrap->lba_nbytes -= fwrap->geo->nbytes_oob;
+			fwrap->lba_pow2 = 1;
+		}
+		fwrap->md_nbytes = 0;
+	}
+
+	if (_verify_options(td, f, fwrap)) {
+		td_verror(td, EINVAL, "_dev_open");
+		goto failure;
+	}
 
 	fwrap->fio_file = f;
 	fwrap->fio_file->filetype = FIO_TYPE_BLOCK;
@@ -325,6 +487,7 @@ failure:
 static int xnvme_fioe_init(struct thread_data *td)
 {
 	struct xnvme_fioe_data *xd = NULL;
+	struct xnvme_fioe_options *o = td->eo;
 	struct fio_file *f;
 	unsigned int i;
 
@@ -347,12 +510,25 @@ static int xnvme_fioe_init(struct thread_data *td)
 		return 1;
 	}
 
-	xd->iovec = calloc(td->o.iodepth, sizeof(*xd->iovec));
-	if (!xd->iovec) {
-		free(xd->iocq);
-		free(xd);
-		log_err("ioeng->init(): !calloc(xd->iovec), err(%d)\n", errno);
-		return 1;
+	if (o->xnvme_iovec) {
+		xd->iovec = calloc(td->o.iodepth, sizeof(*xd->iovec));
+		if (!xd->iovec) {
+			free(xd->iocq);
+			free(xd);
+			log_err("ioeng->init(): !calloc(xd->iovec), err(%d)\n", errno);
+			return 1;
+		}
+	}
+
+	if (o->xnvme_iovec && o->md_per_io_size) {
+		xd->md_iovec = calloc(td->o.iodepth, sizeof(*xd->md_iovec));
+		if (!xd->md_iovec) {
+			free(xd->iocq);
+			free(xd->iovec);
+			free(xd);
+			log_err("ioeng->init(): !calloc(xd->md_iovec), err(%d)\n", errno);
+			return 1;
+		}
 	}
 
 	xd->prev = -1;
@@ -362,8 +538,8 @@ static int xnvme_fioe_init(struct thread_data *td)
 	{
 		if (_dev_open(td, f)) {
 			/*
-			 * Note: We are not freeing xd, iocq and iovec. This
-			 * will be done as part of cleanup routine.
+			 * Note: We are not freeing xd, iocq, iovec and md_iovec.
+			 * This will be done as part of cleanup routine.
 			 */
 			log_err("ioeng->init(): failed; _dev_open(%s)\n", f->file_name);
 			return 1;
@@ -418,13 +594,61 @@ static void xnvme_fioe_iomem_free(struct thread_data *td)
 
 static int xnvme_fioe_io_u_init(struct thread_data *td, struct io_u *io_u)
 {
+	struct xnvme_fioe_request *fio_req;
+	struct xnvme_fioe_options *o = td->eo;
+	struct xnvme_fioe_data *xd = td->io_ops_data;
+	struct xnvme_fioe_fwrap *fwrap = &xd->files[0];
+
+	if (!fwrap->dev) {
+		log_err("ioeng->io_u_init(): failed; no dev-handle\n");
+		return 1;
+	}
+
 	io_u->mmap_data = td->io_ops_data;
+	io_u->engine_data = NULL;
+
+	fio_req = calloc(1, sizeof(*fio_req));
+	if (!fio_req) {
+		log_err("ioeng->io_u_init(): !calloc(fio_req), err(%d)\n", errno);
+		return 1;
+	}
+
+	if (o->md_per_io_size) {
+		fio_req->md_buf = xnvme_buf_alloc(fwrap->dev, o->md_per_io_size);
+		if (!fio_req->md_buf) {
+			free(fio_req);
+			return 1;
+		}
+	}
+
+	io_u->engine_data = fio_req;
 
 	return 0;
 }
 
 static void xnvme_fioe_io_u_free(struct thread_data *td, struct io_u *io_u)
 {
+	struct xnvme_fioe_data *xd = NULL;
+	struct xnvme_fioe_fwrap *fwrap = NULL;
+	struct xnvme_fioe_request *fio_req = NULL;
+
+	if (!td->io_ops_data)
+		return;
+
+	xd = td->io_ops_data;
+	fwrap = &xd->files[0];
+
+	if (!fwrap->dev) {
+		log_err("ioeng->io_u_free(): failed no dev-handle\n");
+		return;
+	}
+
+	fio_req = io_u->engine_data;
+	if (fio_req->md_buf)
+		xnvme_buf_free(fwrap->dev, fio_req->md_buf);
+
+	free(fio_req);
+
 	io_u->mmap_data = NULL;
 }
 
@@ -499,8 +723,10 @@ static int xnvme_fioe_getevents(struct thread_data *td, unsigned int min, unsign
 static enum fio_q_status xnvme_fioe_queue(struct thread_data *td, struct io_u *io_u)
 {
 	struct xnvme_fioe_data *xd = td->io_ops_data;
+	struct xnvme_fioe_options *o = td->eo;
 	struct xnvme_fioe_fwrap *fwrap;
 	struct xnvme_cmd_ctx *ctx;
+	struct xnvme_fioe_request *fio_req = io_u->engine_data;
 	uint32_t nsid;
 	uint64_t slba;
 	uint16_t nlb;
@@ -513,8 +739,13 @@ static enum fio_q_status xnvme_fioe_queue(struct thread_data *td, struct io_u *i
 	fwrap = &xd->files[io_u->file->fileno];
 	nsid = xnvme_dev_get_nsid(fwrap->dev);
 
-	slba = io_u->offset >> fwrap->ssw;
-	nlb = (io_u->xfer_buflen >> fwrap->ssw) - 1;
+	if (fwrap->lba_pow2) {
+		slba = io_u->offset >> fwrap->ssw;
+		nlb = (io_u->xfer_buflen >> fwrap->ssw) - 1;
+	} else {
+		slba = io_u->offset / fwrap->lba_nbytes;
+		nlb = (io_u->xfer_buflen / fwrap->lba_nbytes) - 1;
+	}
 
 	ctx = xnvme_queue_get_cmd_ctx(fwrap->queue);
 	ctx->async.cb_arg = io_u;
@@ -545,14 +776,80 @@ static enum fio_q_status xnvme_fioe_queue(struct thread_data *td, struct io_u *i
 		return FIO_Q_COMPLETED;
 	}
 
+	if (fwrap->geo->pi_type && !o->pi_act) {
+		err = xnvme_pi_ctx_init(&fio_req->pi_ctx, fwrap->lba_nbytes,
+					fwrap->geo->nbytes_oob, fwrap->geo->lba_extended,
+					fwrap->geo->pi_loc, fwrap->geo->pi_type,
+					(o->pi_act << 3 | o->prchk), slba, o->apptag_mask,
+					o->apptag, fwrap->geo->pi_format);
+		if (err) {
+			log_err("ioeng->queue(): err: '%d'\n", err);
+
+			xnvme_queue_put_cmd_ctx(ctx->async.queue, ctx);
+
+			io_u->error = abs(err);
+			return FIO_Q_COMPLETED;
+		}
+
+		if (io_u->ddir == DDIR_WRITE)
+			xnvme_pi_generate(&fio_req->pi_ctx, io_u->xfer_buf, fio_req->md_buf,
+					  nlb + 1);
+	}
+
+	if (fwrap->geo->pi_type)
+		ctx->cmd.nvm.prinfo = (o->pi_act << 3 | o->prchk);
+
+	switch (fwrap->geo->pi_type) {
+	case XNVME_PI_TYPE1:
+	case XNVME_PI_TYPE2:
+		switch (fwrap->geo->pi_format) {
+		case XNVME_SPEC_NVM_NS_16B_GUARD:
+			if (o->prchk & XNVME_PI_FLAGS_REFTAG_CHECK)
+				ctx->cmd.nvm.ilbrt = (uint32_t)slba;
+			break;
+		case XNVME_SPEC_NVM_NS_64B_GUARD:
+			if (o->prchk & XNVME_PI_FLAGS_REFTAG_CHECK) {
+				ctx->cmd.nvm.ilbrt = (uint32_t)slba;
+				ctx->cmd.common.cdw03 = ((slba >> 32) & 0xffff);
+			}
+			break;
+		default:
+			break;
+		}
+		if (o->prchk & XNVME_PI_FLAGS_APPTAG_CHECK) {
+			ctx->cmd.nvm.lbat = o->apptag;
+			ctx->cmd.nvm.lbatm = o->apptag_mask;
+		}
+		break;
+	case XNVME_PI_TYPE3:
+		if (o->prchk & XNVME_PI_FLAGS_APPTAG_CHECK) {
+			ctx->cmd.nvm.lbat = o->apptag;
+			ctx->cmd.nvm.lbatm = o->apptag_mask;
+		}
+		break;
+	case XNVME_PI_DISABLE:
+		break;
+	}
+
 	if (vectored_io) {
 		xd->iovec[io_u->index].iov_base = io_u->xfer_buf;
 		xd->iovec[io_u->index].iov_len = io_u->xfer_buflen;
-
-		err = xnvme_cmd_passv(ctx, &xd->iovec[io_u->index], 1, io_u->xfer_buflen, NULL, 0,
-				      0);
+		if (fwrap->md_nbytes && fwrap->lba_pow2) {
+			xd->md_iovec[io_u->index].iov_base = fio_req->md_buf;
+			xd->md_iovec[io_u->index].iov_len = fwrap->md_nbytes * (nlb + 1);
+			err = xnvme_cmd_passv(ctx, &xd->iovec[io_u->index], 1, io_u->xfer_buflen,
+					      &xd->md_iovec[io_u->index], 1,
+					      fwrap->md_nbytes * (nlb + 1));
+		} else {
+			err = xnvme_cmd_passv(ctx, &xd->iovec[io_u->index], 1, io_u->xfer_buflen,
+					      NULL, 0, 0);
+		}
 	} else {
-		err = xnvme_cmd_pass(ctx, io_u->xfer_buf, io_u->xfer_buflen, NULL, 0);
+		if (fwrap->md_nbytes && fwrap->lba_pow2)
+			err = xnvme_cmd_pass(ctx, io_u->xfer_buf, io_u->xfer_buflen,
+					     fio_req->md_buf, fwrap->md_nbytes * (nlb + 1));
+		else
+			err = xnvme_cmd_pass(ctx, io_u->xfer_buf, io_u->xfer_buflen, NULL, 0);
 	}
 	switch (err) {
 	case 0:
diff --git a/examples/xnvme-pi.fio b/examples/xnvme-pi.fio
new file mode 100644
index 00000000..ca8c0101
--- /dev/null
+++ b/examples/xnvme-pi.fio
@@ -0,0 +1,53 @@
+; README
+;
+; This job-file is intended to be used either as:
+;
+; # Use the xNVMe io-engine engine io_uring_cmd async. impl.
+; fio examples/xnvme-pi.fio \
+;   --ioengine=xnvme \
+;   --xnvme_async=io_uring_cmd \
+;   --filename=/dev/ng0n1
+;
+; # Use the xNVMe io-engine engine with nvme sync. impl.
+; fio examples/xnvme-pi.fio \
+;   --ioengine=xnvme \
+;   --xnvme_sync=nvme \
+;   --filename=/dev/ng0n1
+;
+; # Use the xNVMe io-engine engine with SPDK backend, note that you have to set the Namespace-id
+; fio examples/xnvme-pi.fio \
+;   --ioengine=xnvme \
+;   --xnvme_dev_nsid=1 \
+;   --filename=0000\\:01\\:00.0
+;
+; NOTE: The URI encoded in the filename above, the ":" must be escaped.
+;
+; On the command-line using two "\\":
+;
+; --filename=0000\\:01\\:00.0
+;
+; Within a fio-script using a single "\":
+;
+; filename=0000\:01\:00.0
+;
+; NOTE: This example configuration assumes that the NVMe device is formatted
+; with a separate metadata buffer. If you want to run on an extended LBA format
+; update the "bs" accordingly.
+;
+[global]
+size=100M
+iodepth=16
+bs=4K
+md_per_io_size=64
+pi_act=0
+pi_chk=GUARD,APPTAG,REFTAG
+apptag=0x0234
+apptag_mask=0xFFFF
+thread=1
+stonewall=1
+
+[write]
+rw=write
+
+[read]
+rw=read
diff --git a/fio.1 b/fio.1
index 227fcb47..7ec5c745 100644
--- a/fio.1
+++ b/fio.1
@@ -2251,10 +2251,10 @@ By default, the job will cycle through all available Placement IDs, so use this
 to isolate these identifiers to specific jobs. If you want fio to use placement
 identifier only at indices 0, 2 and 5 specify, you would set `fdp_pli=0,2,5`.
 .TP
-.BI (io_uring_cmd)md_per_io_size \fR=\fPint
+.BI (io_uring_cmd,xnvme)md_per_io_size \fR=\fPint
 Size in bytes for separate metadata buffer per IO. Default: 0.
 .TP
-.BI (io_uring_cmd)pi_act \fR=\fPint
+.BI (io_uring_cmd,xnvme)pi_act \fR=\fPint
 Action to take when nvme namespace is formatted with protection information.
 If this is set to 1 and namespace is formatted with metadata size equal to
 protection information size, fio won't use separate metadata buffer or extended
@@ -2268,7 +2268,7 @@ For 16 bit CRC generation fio will use isa-l if available otherwise it will
 use the default slower generator.
 (see: https://github.com/intel/isa-l)
 .TP
-.BI (io_uring_cmd)pi_chk \fR=\fPstr[,str][,str]
+.BI (io_uring_cmd,xnvme)pi_chk \fR=\fPstr[,str][,str]
 Controls the protection information check. This can take one or more of these
 values. Default: none.
 .RS
@@ -2285,11 +2285,11 @@ Enables protection information checking of application tag field.
 .RE
 .RE
 .TP
-.BI (io_uring_cmd)apptag \fR=\fPint
+.BI (io_uring_cmd,xnvme)apptag \fR=\fPint
 Specifies logical block application tag value, if namespace is formatted to use
 end to end protection information. Default: 0x1234.
 .TP
-.BI (io_uring_cmd)apptag_mask \fR=\fPint
+.BI (io_uring_cmd,xnvme)apptag_mask \fR=\fPint
 Specifies logical block application tag mask value, if namespace is formatted
 to use end to end protection information. Default: 0xffff.
 .TP
@@ -3765,12 +3765,14 @@ resulting in more precise time-related I/O statistics.
 Also see \fBlog_avg_msec\fR as well. Defaults to 1024.
 .TP
 .BI log_avg_msec \fR=\fPint
-By default, fio will log an entry in the iops, latency, or bw log for every
-I/O that completes. When writing to the disk log, that can quickly grow to a
-very large size. Setting this option makes fio average the each log entry
-over the specified period of time, reducing the resolution of the log. See
-\fBlog_window_value\fR as well. Defaults to 0, logging all entries.
-Also see \fBLOG FILE FORMATS\fR section.
+By default, fio will log an entry in the iops, latency, or bw log for every I/O
+that completes. When writing to the disk log, that can quickly grow to a very
+large size. Setting this option directs fio to instead record an average over
+the specified duration for each log entry, reducing the resolution of the log.
+When the job completes, fio will flush any accumulated latency log data, so the
+final log interval may not match the value specified by this option and there
+may even be duplicate timestamps. See \fBlog_window_value\fR as well. Defaults
+to 0, logging entries for each I/O. Also see \fBLOG FILE FORMATS\fR section.
 .TP
 .BI log_hist_msec \fR=\fPint
 Same as \fBlog_avg_msec\fR, but logs entries for completion latency
diff --git a/helper_thread.c b/helper_thread.c
index 2a9dabf5..332ccb53 100644
--- a/helper_thread.c
+++ b/helper_thread.c
@@ -161,7 +161,6 @@ void helper_thread_exit(void)
 		return;
 
 	helper_data->exit = 1;
-	submit_action(A_EXIT);
 	pthread_join(helper_data->thread, NULL);
 }
 
diff --git a/stat.c b/stat.c
index 11b58626..b98e8b27 100644
--- a/stat.c
+++ b/stat.c
@@ -3576,6 +3576,22 @@ static int add_iops_samples(struct thread_data *td, struct timespec *t)
 				td->ts.iops_stat, td->iops_log, false);
 }
 
+static bool td_in_logging_state(struct thread_data *td)
+{
+	if (in_ramp_time(td))
+		return false;
+
+	switch(td->runstate) {
+	case TD_RUNNING:
+	case TD_VERIFYING:
+	case TD_FINISHING:
+	case TD_EXITED:
+		return true;
+	default:
+		return false;
+	}
+}
+
 /*
  * Returns msecs to next event
  */
@@ -3585,15 +3601,13 @@ int calc_log_samples(void)
 	struct timespec now;
 	long elapsed_time = 0;
 
-	fio_gettime(&now, NULL);
-
 	for_each_td(td) {
-		elapsed_time = mtime_since_now(&td->epoch);
+		fio_gettime(&now, NULL);
+		elapsed_time = mtime_since(&td->epoch, &now);
 
 		if (!td->o.stats)
 			continue;
-		if (in_ramp_time(td) ||
-		    !(td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING)) {
+		if (!td_in_logging_state(td)) {
 			next = min(td->o.iops_avg_time, td->o.bw_avg_time);
 			continue;
 		}

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-13 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-13 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 9cfa60d874e9a1da057677619a370409428ea3cf:

  verify: fix potential overflow before widen (2024-02-08 17:45:41 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 097af663e93f86106f32580bfa59e68fae007035:

  Merge branch 'vsock' of https://github.com/MPinna/fio (2024-02-12 11:56:33 -0700)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'vsock' of https://github.com/MPinna/fio

Marco Pinna (1):
      Add support for VSOCK to engine/net.c

 HOWTO.rst                         |   7 +-
 configure                         |  22 +++++++
 engines/net.c                     | 132 +++++++++++++++++++++++++++++++++++++-
 examples/netio_vsock.fio          |  22 +++++++
 examples/netio_vsock_receiver.fio |  14 ++++
 examples/netio_vsock_sender.fio   |  17 +++++
 fio.1                             |   9 ++-
 7 files changed, 217 insertions(+), 6 deletions(-)
 create mode 100644 examples/netio_vsock.fio
 create mode 100644 examples/netio_vsock_receiver.fio
 create mode 100644 examples/netio_vsock_sender.fio

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index ba160551..53b03021 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2626,10 +2626,13 @@ with the caveat that when used on the command line, they must come after the
 		User datagram protocol V6.
 	**unix**
 		UNIX domain socket.
+	**vsock**
+		VSOCK protocol.
 
-	When the protocol is TCP or UDP, the port must also be given, as well as the
-	hostname if the job is a TCP listener or UDP reader. For unix sockets, the
+	When the protocol is TCP, UDP or VSOCK, the port must also be given, as well as the
+	hostname if the job is a TCP or VSOCK listener or UDP reader. For unix sockets, the
 	normal :option:`filename` option should be used and the port is invalid.
+	When the protocol is VSOCK, the :option:`hostname` is the CID of the remote VM.
 
 .. option:: listen : [netsplice] [net]
 
diff --git a/configure b/configure
index dea8d07d..becb193e 100755
--- a/configure
+++ b/configure
@@ -1728,6 +1728,25 @@ elif compile_prog "" "-lws2_32" "TCP_NODELAY"; then
 fi
 print_config "TCP_NODELAY" "$tcp_nodelay"
 
+##########################################
+# Check whether we have vsock
+if test "$vsock" != "yes" ; then
+  vsock="no"
+fi
+cat > $TMPC << EOF
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <linux/vm_sockets.h>
+int main(int argc, char **argv)
+{
+  return socket(AF_VSOCK, SOCK_STREAM, 0);
+}
+EOF
+if compile_prog "" "" "vsock"; then
+  vsock="yes"
+fi
+print_config "vsock" "$vsock"
+
 ##########################################
 # Check whether we have SO_SNDBUF
 if test "$window_size" != "yes" ; then
@@ -3192,6 +3211,9 @@ fi
 if test "$ipv6" = "yes" ; then
   output_sym "CONFIG_IPV6"
 fi
+if test "$vsock" = "yes"; then
+  output_sym "CONFIG_VSOCK"
+fi
 if test "$http" = "yes" ; then
   output_sym "CONFIG_HTTP"
 fi
diff --git a/engines/net.c b/engines/net.c
index fec53d74..29150bb3 100644
--- a/engines/net.c
+++ b/engines/net.c
@@ -18,6 +18,16 @@
 #include <sys/socket.h>
 #include <sys/un.h>
 
+#ifdef CONFIG_VSOCK
+#include <linux/vm_sockets.h>
+#else
+struct sockaddr_vm {
+};
+#ifndef AF_VSOCK
+#define AF_VSOCK	-1
+#endif
+#endif
+
 #include "../fio.h"
 #include "../verify.h"
 #include "../optgroup.h"
@@ -30,6 +40,7 @@ struct netio_data {
 	struct sockaddr_in addr;
 	struct sockaddr_in6 addr6;
 	struct sockaddr_un addr_un;
+	struct sockaddr_vm addr_vm;
 	uint64_t udp_send_seq;
 	uint64_t udp_recv_seq;
 };
@@ -69,6 +80,7 @@ enum {
 	FIO_TYPE_UNIX	= 3,
 	FIO_TYPE_TCP_V6	= 4,
 	FIO_TYPE_UDP_V6	= 5,
+	FIO_TYPE_VSOCK_STREAM   = 6,
 };
 
 static int str_hostname_cb(void *data, const char *input);
@@ -126,6 +138,10 @@ static struct fio_option options[] = {
 			    .oval = FIO_TYPE_UNIX,
 			    .help = "UNIX domain socket",
 			  },
+			  { .ival = "vsock",
+			    .oval = FIO_TYPE_VSOCK_STREAM,
+			    .help = "Virtual socket",
+			  },
 		},
 		.category = FIO_OPT_C_ENGINE,
 		.group	= FIO_OPT_G_NETIO,
@@ -223,6 +239,11 @@ static inline int is_ipv6(struct netio_options *o)
 	return o->proto == FIO_TYPE_UDP_V6 || o->proto == FIO_TYPE_TCP_V6;
 }
 
+static inline int is_vsock(struct netio_options *o)
+{
+	return o->proto == FIO_TYPE_VSOCK_STREAM;
+}
+
 static int set_window_size(struct thread_data *td, int fd)
 {
 #ifdef CONFIG_NET_WINDOWSIZE
@@ -732,6 +753,9 @@ static int fio_netio_connect(struct thread_data *td, struct fio_file *f)
 	} else if (o->proto == FIO_TYPE_UNIX) {
 		domain = AF_UNIX;
 		type = SOCK_STREAM;
+	} else if (is_vsock(o)) {
+		domain = AF_VSOCK;
+		type = SOCK_STREAM;
 	} else {
 		log_err("fio: bad network type %d\n", o->proto);
 		f->fd = -1;
@@ -809,7 +833,14 @@ static int fio_netio_connect(struct thread_data *td, struct fio_file *f)
 			close(f->fd);
 			return 1;
 		}
+	} else if (is_vsock(o)) {
+		socklen_t len = sizeof(nd->addr_vm);
 
+		if (connect(f->fd, (struct sockaddr *) &nd->addr_vm, len) < 0) {
+			td_verror(td, errno, "connect");
+			close(f->fd);
+			return 1;
+		}
 	} else {
 		struct sockaddr_un *addr = &nd->addr_un;
 		socklen_t len;
@@ -849,6 +880,9 @@ static int fio_netio_accept(struct thread_data *td, struct fio_file *f)
 	if (o->proto == FIO_TYPE_TCP) {
 		socklen = sizeof(nd->addr);
 		f->fd = accept(nd->listenfd, (struct sockaddr *) &nd->addr, &socklen);
+	} else if (is_vsock(o)) {
+		socklen = sizeof(nd->addr_vm);
+		f->fd = accept(nd->listenfd, (struct sockaddr *) &nd->addr_vm, &socklen);
 	} else {
 		socklen = sizeof(nd->addr6);
 		f->fd = accept(nd->listenfd, (struct sockaddr *) &nd->addr6, &socklen);
@@ -890,6 +924,9 @@ static void fio_netio_send_close(struct thread_data *td, struct fio_file *f)
 	if (is_ipv6(o)) {
 		to = (struct sockaddr *) &nd->addr6;
 		len = sizeof(nd->addr6);
+	} else if (is_vsock(o)) {
+		to = NULL;
+		len = 0;
 	} else {
 		to = (struct sockaddr *) &nd->addr;
 		len = sizeof(nd->addr);
@@ -960,6 +997,9 @@ static int fio_netio_send_open(struct thread_data *td, struct fio_file *f)
 	if (is_ipv6(o)) {
 		len = sizeof(nd->addr6);
 		to = (struct sockaddr *) &nd->addr6;
+	} else if (is_vsock(o)) {
+		len = sizeof(nd->addr_vm);
+		to = (struct sockaddr *) &nd->addr_vm;
 	} else {
 		len = sizeof(nd->addr);
 		to = (struct sockaddr *) &nd->addr;
@@ -1023,13 +1063,17 @@ static int fio_fill_addr(struct thread_data *td, const char *host, int af,
 
 	memset(&hints, 0, sizeof(hints));
 
-	if (is_tcp(o))
+	if (is_tcp(o) || is_vsock(o))
 		hints.ai_socktype = SOCK_STREAM;
 	else
 		hints.ai_socktype = SOCK_DGRAM;
 
 	if (is_ipv6(o))
 		hints.ai_family = AF_INET6;
+#ifdef CONFIG_VSOCK
+	else if (is_vsock(o))
+		hints.ai_family = AF_VSOCK;
+#endif
 	else
 		hints.ai_family = AF_INET;
 
@@ -1110,12 +1154,50 @@ static int fio_netio_setup_connect_unix(struct thread_data *td,
 	return 0;
 }
 
+static int fio_netio_setup_connect_vsock(struct thread_data *td,
+					const char *host, unsigned short port)
+{
+#ifdef CONFIG_VSOCK
+	struct netio_data *nd = td->io_ops_data;
+	struct sockaddr_vm *addr = &nd->addr_vm;
+	int cid;
+
+	if (!host) {
+		log_err("fio: connect with no host to connect to.\n");
+		if (td_read(td))
+			log_err("fio: did you forget to set 'listen'?\n");
+
+		td_verror(td, EINVAL, "no hostname= set");
+		return 1;
+	}
+
+	addr->svm_family = AF_VSOCK;
+	addr->svm_port = port;
+
+	if (host) {
+		cid = atoi(host);
+		if (cid < 0 || cid > UINT32_MAX) {
+			log_err("fio: invalid CID %d\n", cid);
+			return 1;
+		}
+		addr->svm_cid = cid;
+	}
+
+	return 0;
+#else
+	td_verror(td, -EINVAL, "vsock not supported");
+	return 1;
+#endif
+}
+
 static int fio_netio_setup_connect(struct thread_data *td)
 {
 	struct netio_options *o = td->eo;
 
 	if (is_udp(o) || is_tcp(o))
 		return fio_netio_setup_connect_inet(td, td->o.filename,o->port);
+	else if (is_vsock(o))
+		return fio_netio_setup_connect_vsock(td, td->o.filename, o->port);
 	else
 		return fio_netio_setup_connect_unix(td, td->o.filename);
 }
@@ -1268,6 +1350,47 @@ static int fio_netio_setup_listen_inet(struct thread_data *td, short port)
 	return 0;
 }
 
+static int fio_netio_setup_listen_vsock(struct thread_data *td, short port, int type)
+{
+#ifdef CONFIG_VSOCK
+	struct netio_data *nd = td->io_ops_data;
+	struct sockaddr_vm *addr = &nd->addr_vm;
+	int fd, opt;
+	socklen_t len;
+
+	fd = socket(AF_VSOCK, type, 0);
+	if (fd < 0) {
+		td_verror(td, errno, "socket");
+		return 1;
+	}
+
+	opt = 1;
+	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (void *) &opt, sizeof(opt)) < 0) {
+		td_verror(td, errno, "setsockopt");
+		close(fd);
+		return 1;
+	}
+
+	len = sizeof(*addr);
+
+	nd->addr_vm.svm_family = AF_VSOCK;
+	nd->addr_vm.svm_cid = VMADDR_CID_ANY;
+	nd->addr_vm.svm_port = port;
+
+	if (bind(fd, (struct sockaddr *) addr, len) < 0) {
+		td_verror(td, errno, "bind");
+		close(fd);
+		return 1;
+	}
+
+	nd->listenfd = fd;
+	return 0;
+#else
+	td_verror(td, -EINVAL, "vsock not supported");
+	return -1;
+#endif
+}
+
 static int fio_netio_setup_listen(struct thread_data *td)
 {
 	struct netio_data *nd = td->io_ops_data;
@@ -1276,6 +1399,8 @@ static int fio_netio_setup_listen(struct thread_data *td)
 
 	if (is_udp(o) || is_tcp(o))
 		ret = fio_netio_setup_listen_inet(td, o->port);
+	else if (is_vsock(o))
+		ret = fio_netio_setup_listen_vsock(td, o->port, SOCK_STREAM);
 	else
 		ret = fio_netio_setup_listen_unix(td, td->o.filename);
 
@@ -1311,6 +1436,9 @@ static int fio_netio_init(struct thread_data *td)
 	if (o->proto == FIO_TYPE_UNIX && o->port) {
 		log_err("fio: network IO port not valid with unix socket\n");
 		return 1;
+	} else if (is_vsock(o) && !o->port) {
+		log_err("fio: network IO requires port for vsock\n");
+		return 1;
 	} else if (o->proto != FIO_TYPE_UNIX && !o->port) {
 		log_err("fio: network IO requires port for tcp or udp\n");
 		return 1;
@@ -1318,7 +1446,7 @@ static int fio_netio_init(struct thread_data *td)
 
 	o->port += td->subjob_number;
 
-	if (!is_tcp(o)) {
+	if (!is_tcp(o) && !is_vsock(o)) {
 		if (o->listen) {
 			log_err("fio: listen only valid for TCP proto IO\n");
 			return 1;
diff --git a/examples/netio_vsock.fio b/examples/netio_vsock.fio
new file mode 100644
index 00000000..8c328f7d
--- /dev/null
+++ b/examples/netio_vsock.fio
@@ -0,0 +1,22 @@
+# Example network vsock job, just defines two clients that send/recv data
+[global]
+ioengine=net
+
+port=8888
+protocol=vsock
+bs=4k
+size=100g
+
+#set the below option to enable end-to-end data integrity tests
+#verify=md5
+
+[receiver]
+listen
+rw=read
+
+[sender]
+# 1 (VMADDR_CID_LOCAL) is the well-known address
+# for local communication (loopback)
+hostname=1
+startdelay=1
+rw=write
diff --git a/examples/netio_vsock_receiver.fio b/examples/netio_vsock_receiver.fio
new file mode 100644
index 00000000..e2a00c4d
--- /dev/null
+++ b/examples/netio_vsock_receiver.fio
@@ -0,0 +1,14 @@
+# Example network vsock job, just defines a receiver
+[global]
+ioengine=net
+port=8888
+protocol=vsock
+bs=4k
+size=100g
+
+#set the below option to enable end-to-end data integrity tests
+#verify=md5
+
+[receiver]
+listen
+rw=read
diff --git a/examples/netio_vsock_sender.fio b/examples/netio_vsock_sender.fio
new file mode 100644
index 00000000..2451d990
--- /dev/null
+++ b/examples/netio_vsock_sender.fio
@@ -0,0 +1,17 @@
+# Example network vsock job, just defines a sender
+[global]
+ioengine=net
+port=8888
+protocol=vsock
+bs=4k
+size=100g
+
+#set the below option to enable end-to-end data integrity tests
+#verify=md5
+
+[sender]
+# set the 'hostname' option to the CID of the listening domain
+hostname=3
+startdelay=1
+rw=write
+
diff --git a/fio.1 b/fio.1
index aef1dc85..227fcb47 100644
--- a/fio.1
+++ b/fio.1
@@ -2376,11 +2376,16 @@ User datagram protocol V6.
 .TP
 .B unix
 UNIX domain socket.
+.TP
+.B vsock
+VSOCK protocol.
 .RE
 .P
-When the protocol is TCP or UDP, the port must also be given, as well as the
-hostname if the job is a TCP listener or UDP reader. For unix sockets, the
+When the protocol is TCP, UDP or VSOCK, the port must also be given, as well as the
+hostname if the job is a TCP or VSOCK listener or UDP reader. For unix sockets, the
 normal \fBfilename\fR option should be used and the port is invalid.
+When the protocol is VSOCK, the \fBhostname\fR is the CID of the remote VM.
+
 .RE
 .TP
 .BI (netsplice,net)listen

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-09 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-09 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 12067650d11d4777dee0cd64a136923c2fd2d073:

  t/zbd: add -s option to test-zbd-support script (2024-02-07 08:43:13 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 9cfa60d874e9a1da057677619a370409428ea3cf:

  verify: fix potential overflow before widen (2024-02-08 17:45:41 -0500)

----------------------------------------------------------------
Oleg Krasnov (1):
      fix wrong offset for VERIFY_PATTERN_NO_HDR

Vincent Fu (2):
      Merge branch 'fix-offset' of https://github.com/onkrasnov/fio
      verify: fix potential overflow before widen

 verify.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/verify.c b/verify.c
index 78f333e6..b438eed6 100644
--- a/verify.c
+++ b/verify.c
@@ -338,12 +338,20 @@ static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc)
 static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
 {
 	unsigned long long offset;
+	uint32_t len;
+	struct thread_data *td = vc->td;
 
 	offset = vc->io_u->verify_offset;
-	offset += vc->hdr_num * hdr->len;
+	if (td->o.verify != VERIFY_PATTERN_NO_HDR) {
+		len = hdr->len;
+		offset += (unsigned long long) vc->hdr_num * len;
+	} else {
+		len = vc->io_u->buflen;
+	}
+
 	log_err("%.8s: verify failed at file %s offset %llu, length %u"
 			" (requested block: offset=%llu, length=%llu, flags=%x)\n",
-			vc->name, vc->io_u->file->file_name, offset, hdr->len,
+			vc->name, vc->io_u->file->file_name, offset, len,
 			vc->io_u->verify_offset, vc->io_u->buflen, vc->io_u->flags);
 
 	if (vc->good_crc && vc->bad_crc) {

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-02-08 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-02-08 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 625b155dcd3d56595ced60806e091126446c1e08:

  examples: cmdprio_bssplit: add CDL example (2024-01-27 11:18:57 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 12067650d11d4777dee0cd64a136923c2fd2d073:

  t/zbd: add -s option to test-zbd-support script (2024-02-07 08:43:13 -0500)

----------------------------------------------------------------
Dmitry Fomichev (5):
      zbd: avoid assertions during sequential read I/O
      oslib: log BLKREPORTZONE error code
      zbd: use a helper to calculate zone index
      t/zbd: check device for unrestricted read support
      t/zbd: add -s option to test-zbd-support script

 oslib/linux-blkzoned.c |  2 ++
 t/zbd/functions        | 22 ++++++++++++++++++++++
 t/zbd/test-zbd-support | 20 ++++++++++++++++++--
 zbd.c                  | 21 +++++++++++++++------
 4 files changed, 57 insertions(+), 8 deletions(-)

---

Diff of recent changes:

diff --git a/oslib/linux-blkzoned.c b/oslib/linux-blkzoned.c
index 2c3ecf33..1cc8d288 100644
--- a/oslib/linux-blkzoned.c
+++ b/oslib/linux-blkzoned.c
@@ -242,6 +242,8 @@ int blkzoned_report_zones(struct thread_data *td, struct fio_file *f,
 	hdr->sector = offset >> 9;
 	ret = ioctl(fd, BLKREPORTZONE, hdr);
 	if (ret) {
+		log_err("%s: BLKREPORTZONE ioctl failed, ret=%d, err=%d.\n",
+			f->file_name, ret, -errno);
 		ret = -errno;
 		goto out;
 	}
diff --git a/t/zbd/functions b/t/zbd/functions
index 028df404..7734371e 100644
--- a/t/zbd/functions
+++ b/t/zbd/functions
@@ -290,6 +290,28 @@ min_seq_write_size() {
 	fi
 }
 
+urswrz() {
+    local dev=$1
+
+    if [ -n "${sg_inq}" ] && [ ! -n "${use_libzbc}" ]; then
+	if ! ${sg_inq} -e --page=0xB6 --len=10 --hex "$dev" \
+		 > /dev/null 2>&1; then
+	    # Couldn't get URSWRZ bit. Assume the reads are unrestricted
+	    # because this configuration is more common.
+	    echo 1
+	else
+	    ${sg_inq} -e --page=0xB6 --len=10 --hex "$dev" | tail -1 |
+		{
+		    read -r offset b0 b1 b2 b3 b4 trailer && \
+			echo $(( $b4 & 0x01 )) || echo 0
+		}
+	fi
+    else
+	${zbc_info} "$dev" |
+	    sed -n 's/^[[:blank:]].*Read commands are \(un\)restricted*/\1/p' | grep -q ^ && echo 1 || echo 0
+    fi
+}
+
 is_zbc() {
 	local dev=$1
 
diff --git a/t/zbd/test-zbd-support b/t/zbd/test-zbd-support
index 532860eb..c27d2ad6 100755
--- a/t/zbd/test-zbd-support
+++ b/t/zbd/test-zbd-support
@@ -15,6 +15,7 @@ usage() {
 	echo -e "\t-w Reset all zones before executing each write test case"
 	echo -e "\t-o <max_open_zones> Run fio with max_open_zones limit"
 	echo -e "\t-t <test #> Run only a single test case with specified number"
+	echo -e "\t-s <test #> Start testing from the case with the specified number"
 	echo -e "\t-q Quit the test run after any failed test"
 	echo -e "\t-z Run fio with debug=zbd option"
 	echo -e "\t-u Use io_uring ioengine in place of libaio"
@@ -412,8 +413,16 @@ test4() {
     opts+=("--size=$size" "--thread=1" "--read_beyond_wp=1")
     opts+=("$(ioengine "psync")" "--rw=read" "--direct=1" "--disable_lat=1")
     opts+=("--zonemode=zbd" "--zonesize=${zone_size}")
-    run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $?
-    check_read $size || return $?
+    run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1
+    fio_rc=$?
+    if [[ $unrestricted_reads != 0 ]]; then
+	if [[ $fio_rc != 0 ]]; then
+		return "$fio_rc"
+	fi
+	check_read $size || return $?
+    else
+        [ $fio_rc == 0 ] && return 1 || return 0
+    fi
 }
 
 # Sequential write to sequential zones.
@@ -1594,6 +1603,7 @@ zbd_debug=
 max_open_zones_opt=
 quit_on_err=
 force_io_uring=
+start_test=1
 
 while [ "${1#-}" != "$1" ]; do
   case "$1" in
@@ -1607,6 +1617,7 @@ while [ "${1#-}" != "$1" ]; do
     -w) reset_before_write=1; shift;;
     -t) tests+=("$2"); shift; shift;;
     -o) max_open_zones_opt="${2}"; shift; shift;;
+    -s) start_test=$2; shift; shift;;
     -v) dynamic_analyzer=(valgrind "--read-var-info=yes");
 	shift;;
     -q) quit_on_err=1; shift;;
@@ -1664,6 +1675,7 @@ if [[ -b "$realdev" ]]; then
 		first_sequential_zone_sector=${result[0]}
 		sectors_per_zone=${result[1]}
 		zone_size=$((sectors_per_zone * 512))
+		unrestricted_reads=$(urswrz "$dev")
 		if ! max_open_zones=$(max_open_zones "$dev"); then
 			echo "Failed to determine maximum number of open zones"
 			exit 1
@@ -1681,9 +1693,11 @@ if [[ -b "$realdev" ]]; then
 		sectors_per_zone=$((zone_size / 512))
 		max_open_zones=128
 		max_active_zones=0
+		unrestricted_reads=1
 		set_io_scheduler "$basename" none || exit $?
 		;;
 	esac
+
 elif [[ -c "$realdev" ]]; then
 	# For an SG node, we must have libzbc option specified
 	if [[ ! -n "$use_libzbc" ]]; then
@@ -1712,6 +1726,7 @@ elif [[ -c "$realdev" ]]; then
 	first_sequential_zone_sector=${result[0]}
 	sectors_per_zone=${result[1]}
 	zone_size=$((sectors_per_zone * 512))
+	unrestricted_reads=$(urswrz "$dev")
 	if ! max_open_zones=$(max_open_zones "$dev"); then
 		echo "Failed to determine maximum number of open zones"
 		exit 1
@@ -1761,6 +1776,7 @@ trap 'intr=1' SIGINT
 ret=0
 
 for test_number in "${tests[@]}"; do
+    [ "${test_number}" -lt "${start_test}" ] && continue
     rm -f "${logfile}.${test_number}"
     unset SKIP_REASON
     echo -n "Running test $(printf "%02d" $test_number) ... "
diff --git a/zbd.c b/zbd.c
index 61b5b688..37417660 100644
--- a/zbd.c
+++ b/zbd.c
@@ -104,8 +104,7 @@ static void zone_lock(struct thread_data *td, const struct fio_file *f,
 		      struct fio_zone_info *z)
 {
 #ifndef NDEBUG
-	struct zoned_block_device_info *zbd = f->zbd_info;
-	uint32_t const nz = z - zbd->zone_info;
+	unsigned int const nz = zbd_zone_idx(f, z);
 	/* A thread should never lock zones outside its working area. */
 	assert(f->min_zone <= nz && nz < f->max_zone);
 	assert(z->has_wp);
@@ -674,9 +673,20 @@ static bool zbd_zone_align_file_sizes(struct thread_data *td,
 		return false;
 	}
 
+	if (td->o.td_ddir == TD_DDIR_READ) {
+		z = zbd_offset_to_zone(f, f->file_offset + f->io_size);
+		new_end = z->start;
+		if (f->file_offset + f->io_size > new_end) {
+			log_info("%s: rounded io_size from %"PRIu64" to %"PRIu64"\n",
+				 f->file_name, f->io_size,
+				 new_end - f->file_offset);
+			f->io_size = new_end - f->file_offset;
+		}
+		return true;
+	}
+
 	z = zbd_offset_to_zone(f, f->file_offset);
-	if ((f->file_offset != z->start) &&
-	    (td->o.td_ddir != TD_DDIR_READ)) {
+	if (f->file_offset != z->start) {
 		new_offset = zbd_zone_end(z);
 		if (new_offset >= f->file_offset + f->io_size) {
 			log_info("%s: io_size must be at least one zone\n",
@@ -692,8 +702,7 @@ static bool zbd_zone_align_file_sizes(struct thread_data *td,
 
 	z = zbd_offset_to_zone(f, f->file_offset + f->io_size);
 	new_end = z->start;
-	if ((td->o.td_ddir != TD_DDIR_READ) &&
-	    (f->file_offset + f->io_size != new_end)) {
+	if (f->file_offset + f->io_size != new_end) {
 		if (new_end <= f->file_offset) {
 			log_info("%s: io_size must be at least one zone\n",
 				 f->file_name);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-28 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-28 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 90a0fbc83c59ddfacc3e90dcb7721ff322bfe26f:

  Merge branch 'coverity-fix' of https://github.com/ankit-sam/fio (2024-01-25 10:20:10 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 625b155dcd3d56595ced60806e091126446c1e08:

  examples: cmdprio_bssplit: add CDL example (2024-01-27 11:18:57 -0500)

----------------------------------------------------------------
Niklas Cassel (2):
      examples: cmdprio_bssplit: s,IO,I/O,
      examples: cmdprio_bssplit: add CDL example

 examples/cmdprio-bssplit.fio | 47 ++++++++++++++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 6 deletions(-)

---

Diff of recent changes:

diff --git a/examples/cmdprio-bssplit.fio b/examples/cmdprio-bssplit.fio
index f3b2fac0..ee202d74 100644
--- a/examples/cmdprio-bssplit.fio
+++ b/examples/cmdprio-bssplit.fio
@@ -12,9 +12,9 @@ iodepth=16
 ; use the same prio class and prio level defined by the cmdprio_class
 ; and cmdprio options.
 [cmdprio]
-; 40% of read IOs are 64kB and 60% are 1MB. 100% of writes are 1MB.
+; 40% of read I/Os are 64kB and 60% are 1MB. 100% of writes are 1MB.
 ; 100% of the 64kB reads are executed with prio class 1 and prio level 0.
-; All other IOs are executed without a priority set.
+; All other I/Os are executed without a priority set.
 bssplit=64k/40:1024k/60,1024k/100
 cmdprio_bssplit=64k/100:1024k/0,1024k/0
 cmdprio_class=1
@@ -23,22 +23,57 @@ cmdprio=0
 ; Advanced cmdprio_bssplit format. Each non-zero percentage entry can
 ; use a different prio class and prio level (appended to each entry).
 [cmdprio-adv]
-; 40% of read IOs are 64kB and 60% are 1MB. 100% of writes are 1MB.
+; 40% of read I/Os are 64kB and 60% are 1MB. 100% of writes are 1MB.
 ; 25% of the 64kB reads are executed with prio class 1 and prio level 1,
 ; 75% of the 64kB reads are executed with prio class 3 and prio level 2.
-; All other IOs are executed without a priority set.
+; All other I/Os are executed without a priority set.
 stonewall
 bssplit=64k/40:1024k/60,1024k/100
 cmdprio_bssplit=64k/25/1/1:64k/75/3/2:1024k/0,1024k/0
 
 ; Identical to the previous example, but with a default priority defined.
 [cmdprio-adv-def]
-; 40% of read IOs are 64kB and 60% are 1MB. 100% of writes are 1MB.
+; 40% of read I/Os are 64kB and 60% are 1MB. 100% of writes are 1MB.
 ; 25% of the 64kB reads are executed with prio class 1 and prio level 1,
 ; 75% of the 64kB reads are executed with prio class 3 and prio level 2.
-; All other IOs are executed with prio class 2 and prio level 7.
+; All other I/Os are executed with prio class 2 and prio level 7.
 stonewall
 prioclass=2
 prio=7
 bssplit=64k/40:1024k/60,1024k/100
 cmdprio_bssplit=64k/25/1/1:64k/75/3/2:1024k/0,1024k/0
+
+; Example of how to use cmdprio_bssplit with Command Duration Limits (CDL)
+; using I/O priority hints. The drive has to support CDL, and CDL has to be
+; enabled in sysfs, otherwise the hints will not be sent down to the drive.
+[cmdprio-hints]
+; 40% of the I/Os are 1MB reads and 60% of the I/Os are 2MB reads.
+;
+; 10% of the 1MB reads are executed with prio class 2 (Best Effort),
+; prio level 0, and prio hint 1. Prio hint 1 means CDL descriptor 1.
+; Since 40% of read I/Os are 1MB, and 10% of the 1MB I/Os use CDL desc 1,
+; this means that 4% of all the issued I/O will use this configuration.
+;
+; 30% of the 1MB reads are executed with prio class 2 (Best Effort),
+; prio level 0, and prio hint 2. Prio hint 2 means CDL descriptor 2.
+; Since 40% of read I/Os are 1MB, and 30% of the 1MB I/Os use CDL desc 2,
+; this means that 12% of all the issued I/O will use this configuration.
+;
+; 60% of the 1MB reads are executed with prio class 2 (Best Effort),
+; prio level 0, and prio hint 0. Prio hint 0 means no hint.
+; Since 40% of read I/Os are 1MB, and 60% of the 1MB I/Os use no hint,
+; this means that 24% of all the issued I/O will use this configuration.
+;
+; 10% of the 2MB reads are executed with prio class 2 (Best Effort),
+; prio level 0, and prio hint 3. Prio hint 3 means CDL descriptor 3.
+; Since 60% of read I/Os are 2MB, and 10% of the 2MB I/Os use CDL desc 3,
+; this means that 6% of all the issued I/O will use this configuration.
+;
+; 90% of the 2MB reads are executed with prio class 2 (Best Effort),
+; prio level 0, and prio hint 0. Prio hint 0 means no hint.
+; Since 60% of read I/Os are 2MB, and 90% of the 2MB I/Os use no hint,
+; this means that 54% of all the issued I/O will use this configuration.
+stonewall
+rw=randread
+bssplit=1M/40:2M/60
+cmdprio_bssplit=1M/10/2/0/1:1M/30/2/0/2:1M/60/2/0/0:2M/10/2/0/3:2M/90/2/0/0

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-26 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-26 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 7f250f7514bacef1a3cea24a22ecce8bd30378bd:

  ci: resolve GitHub Actions Node.js warnings (2024-01-24 19:45:33 +0000)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 90a0fbc83c59ddfacc3e90dcb7721ff322bfe26f:

  Merge branch 'coverity-fix' of https://github.com/ankit-sam/fio (2024-01-25 10:20:10 -0700)

----------------------------------------------------------------
Ankit Kumar (3):
      stat: log out both average and max over the window
      docs: update fio man page for log_window_value
      iolog: fix reported defect from coverity scan

Jens Axboe (2):
      t/io_uring: remove dma map option
      Merge branch 'coverity-fix' of https://github.com/ankit-sam/fio

Vincent Fu (1):
      docs: change listed type for log_window_value to str

 HOWTO.rst    | 45 ++++++++++++++++++++++++++--------
 client.c     |  6 +++--
 fio.1        | 47 ++++++++++++++++++++++++++++--------
 iolog.c      | 79 +++++++++++++++++++++++++++++++++++++++++++++---------------
 iolog.h      | 21 +++++++++++++---
 options.c    | 34 ++++++++++++++++++++++----
 server.c     |  6 +++--
 stat.c       | 32 ++++++++++++++----------
 t/io_uring.c | 46 ++---------------------------------
 9 files changed, 207 insertions(+), 109 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index d0ba8021..ba160551 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -4067,7 +4067,7 @@ Measurements and reporting
 	I/O that completes. When writing to the disk log, that can quickly grow to a
 	very large size. Setting this option makes fio average the each log entry
 	over the specified period of time, reducing the resolution of the log.  See
-	:option:`log_max_value` as well. Defaults to 0, logging all entries.
+	:option:`log_window_value` as well. Defaults to 0, logging all entries.
 	Also see `Log File Formats`_.
 
 .. option:: log_hist_msec=int
@@ -4088,11 +4088,28 @@ Measurements and reporting
 	histogram logs contain 1216 latency bins. See :option:`write_hist_log`
 	and `Log File Formats`_.
 
-.. option:: log_max_value=bool
+.. option:: log_window_value=str, log_max_value=str
 
-	If :option:`log_avg_msec` is set, fio logs the average over that window. If
-	you instead want to log the maximum value, set this option to 1. Defaults to
-	0, meaning that averaged values are logged.
+	If :option:`log_avg_msec` is set, fio by default logs the average over that
+	window. This option determines whether fio logs the average, maximum or
+	both the values over the window. This only affects the latency logging,
+	as both average and maximum values for iops or bw log will be same.
+	Accepted values are:
+
+		**avg**
+			Log average value over the window. The default.
+
+		**max**
+			Log maximum value in the window.
+
+		**both**
+			Log both average and maximum value over the window.
+
+		**0**
+			Backward-compatible alias for **avg**.
+
+		**1**
+			Backward-compatible alias for **max**.
 
 .. option:: log_offset=bool
 
@@ -5061,11 +5078,19 @@ toggled with :option:`log_offset`.
 by the ioengine specific :option:`cmdprio_percentage`.
 
 Fio defaults to logging every individual I/O but when windowed logging is set
-through :option:`log_avg_msec`, either the average (by default) or the maximum
-(:option:`log_max_value` is set) *value* seen over the specified period of time
-is recorded. Each *data direction* seen within the window period will aggregate
-its values in a separate row. Further, when using windowed logging the *block
-size* and *offset* entries will always contain 0.
+through :option:`log_avg_msec`, either the average (by default), the maximum
+(:option:`log_window_value` is set to max) *value* seen over the specified period
+of time, or both the average *value* and maximum *value1* (:option:`log_window_value`
+is set to both) is recorded. The log file format when both the values are reported
+takes this form:
+
+    *time* (`msec`), *value*, *value1*, *data direction*, *block size* (`bytes`),
+    *offset* (`bytes`), *command priority*
+
+
+Each *data direction* seen within the window period will aggregate its values in a
+separate row. Further, when using windowed logging the *block size* and *offset*
+entries will always contain 0.
 
 
 Client/Server
diff --git a/client.c b/client.c
index 699a2e5b..4cb7dffe 100644
--- a/client.c
+++ b/client.c
@@ -1718,8 +1718,10 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd,
 			s = (struct io_sample *)((char *)s + sizeof(struct io_u_plat_entry) * i);
 
 		s->time		= le64_to_cpu(s->time);
-		if (ret->log_type != IO_LOG_TYPE_HIST)
-			s->data.val	= le64_to_cpu(s->data.val);
+		if (ret->log_type != IO_LOG_TYPE_HIST) {
+			s->data.val.val0	= le64_to_cpu(s->data.val.val0);
+			s->data.val.val1	= le64_to_cpu(s->data.val.val1);
+		}
 		s->__ddir	= __le32_to_cpu(s->__ddir);
 		s->bs		= le64_to_cpu(s->bs);
 		s->priority	= le16_to_cpu(s->priority);
diff --git a/fio.1 b/fio.1
index 8f659f1d..aef1dc85 100644
--- a/fio.1
+++ b/fio.1
@@ -3764,7 +3764,7 @@ By default, fio will log an entry in the iops, latency, or bw log for every
 I/O that completes. When writing to the disk log, that can quickly grow to a
 very large size. Setting this option makes fio average the each log entry
 over the specified period of time, reducing the resolution of the log. See
-\fBlog_max_value\fR as well. Defaults to 0, logging all entries.
+\fBlog_window_value\fR as well. Defaults to 0, logging all entries.
 Also see \fBLOG FILE FORMATS\fR section.
 .TP
 .BI log_hist_msec \fR=\fPint
@@ -3782,10 +3782,28 @@ the histogram logs enabled with \fBlog_hist_msec\fR. For each increment
 in coarseness, fio outputs half as many bins. Defaults to 0, for which
 histogram logs contain 1216 latency bins. See \fBLOG FILE FORMATS\fR section.
 .TP
-.BI log_max_value \fR=\fPbool
-If \fBlog_avg_msec\fR is set, fio logs the average over that window. If
-you instead want to log the maximum value, set this option to 1. Defaults to
-0, meaning that averaged values are logged.
+.BI log_window_value \fR=\fPstr "\fR,\fP log_max_value" \fR=\fPstr
+If \fBlog_avg_msec\fR is set, fio by default logs the average over that window.
+This option determines whether fio logs the average, maximum or both the
+values over the window. This only affects the latency logging, as both average
+and maximum values for iops or bw log will be same. Accepted values are:
+.RS
+.TP
+.B avg
+Log average value over the window. The default.
+.TP
+.B max
+Log maximum value in the window.
+.TP
+.B both
+Log both average and maximum value over the window.
+.TP
+.B 0
+Backward-compatible alias for \fBavg\fR.
+.TP
+.B 1
+Backward-compatible alias for \fBmax\fR.
+.RE
 .TP
 .BI log_offset \fR=\fPbool
 If this is set, the iolog options will include the byte offset for the I/O
@@ -4797,11 +4815,20 @@ number with the lowest 13 bits indicating the priority value (\fBprio\fR and
 (\fBprioclass\fR and \fBcmdprio_class\fR options).
 .P
 Fio defaults to logging every individual I/O but when windowed logging is set
-through \fBlog_avg_msec\fR, either the average (by default) or the maximum
-(\fBlog_max_value\fR is set) `value' seen over the specified period of time
-is recorded. Each `data direction' seen within the window period will aggregate
-its values in a separate row. Further, when using windowed logging the `block
-size' and `offset' entries will always contain 0.
+through \fBlog_avg_msec\fR, either the average (by default), the maximum
+(\fBlog_window_value\fR is set to max) `value' seen over the specified period of
+time, or both the average `value' and maximum `value1' (\fBlog_window_value\fR is
+set to both) is recorded. The log file format when both the values are reported
+takes this form:
+.RS
+.P
+time (msec), value, value1, data direction, block size (bytes), offset (bytes),
+command priority
+.RE
+.P
+Each `data direction' seen within the window period will aggregate its values
+in a separate row. Further, when using windowed logging the `block size' and
+`offset' entries will always contain 0.
 .SH CLIENT / SERVER
 Normally fio is invoked as a stand-alone application on the machine where the
 I/O workload should be generated. However, the backend and frontend of fio can
diff --git a/iolog.c b/iolog.c
index 5213c60f..f52a9a80 100644
--- a/iolog.c
+++ b/iolog.c
@@ -862,6 +862,13 @@ void setup_log(struct io_log **log, struct log_params *p,
 		l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT;
 	if (l->log_prio)
 		l->log_ddir_mask |= LOG_PRIO_SAMPLE_BIT;
+	/*
+	 * The bandwidth-log option generates agg-read_bw.log,
+	 * agg-write_bw.log and agg-trim_bw.log for which l->td is NULL.
+	 * Check if l->td is valid before dereferencing it.
+	 */
+	if (l->td && l->td->o.log_max == IO_LOG_SAMPLE_BOTH)
+		l->log_ddir_mask |= LOG_AVG_MAX_SAMPLE_BIT;
 
 	INIT_FLIST_HEAD(&l->chunk_list);
 
@@ -988,7 +995,7 @@ static void flush_hist_samples(FILE *f, int hist_coarseness, void *samples,
 void flush_samples(FILE *f, void *samples, uint64_t sample_size)
 {
 	struct io_sample *s;
-	int log_offset, log_prio;
+	int log_offset, log_prio, log_avg_max;
 	uint64_t i, nr_samples;
 	unsigned int prio_val;
 	const char *fmt;
@@ -999,17 +1006,32 @@ void flush_samples(FILE *f, void *samples, uint64_t sample_size)
 	s = __get_sample(samples, 0, 0);
 	log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
 	log_prio = (s->__ddir & LOG_PRIO_SAMPLE_BIT) != 0;
+	log_avg_max = (s->__ddir & LOG_AVG_MAX_SAMPLE_BIT) != 0;
 
 	if (log_offset) {
-		if (log_prio)
-			fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %llu, 0x%04x\n";
-		else
-			fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %llu, %u\n";
+		if (log_prio) {
+			if (log_avg_max)
+				fmt = "%" PRIu64 ", %" PRId64 ", %" PRId64 ", %u, %llu, %llu, 0x%04x\n";
+			else
+				fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %llu, 0x%04x\n";
+		} else {
+			if (log_avg_max)
+				fmt = "%" PRIu64 ", %" PRId64 ", %" PRId64 ", %u, %llu, %llu, %u\n";
+			else
+				fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %llu, %u\n";
+		}
 	} else {
-		if (log_prio)
-			fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, 0x%04x\n";
-		else
-			fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %u\n";
+		if (log_prio) {
+			if (log_avg_max)
+				fmt = "%" PRIu64 ", %" PRId64 ", %" PRId64 ", %u, %llu, 0x%04x\n";
+			else
+				fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, 0x%04x\n";
+		} else {
+			if (log_avg_max)
+				fmt = "%" PRIu64 ", %" PRId64 ", %" PRId64 ", %u, %llu, %u\n";
+			else
+				fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %u\n";
+		}
 	}
 
 	nr_samples = sample_size / __log_entry_sz(log_offset);
@@ -1023,20 +1045,37 @@ void flush_samples(FILE *f, void *samples, uint64_t sample_size)
 			prio_val = ioprio_value_is_class_rt(s->priority);
 
 		if (!log_offset) {
-			fprintf(f, fmt,
-				s->time,
-				s->data.val,
-				io_sample_ddir(s), (unsigned long long) s->bs,
-				prio_val);
+			if (log_avg_max)
+				fprintf(f, fmt,
+					s->time,
+					s->data.val.val0,
+					s->data.val.val1,
+					io_sample_ddir(s), (unsigned long long) s->bs,
+					prio_val);
+			else
+				fprintf(f, fmt,
+					s->time,
+					s->data.val.val0,
+					io_sample_ddir(s), (unsigned long long) s->bs,
+					prio_val);
 		} else {
 			struct io_sample_offset *so = (void *) s;
 
-			fprintf(f, fmt,
-				s->time,
-				s->data.val,
-				io_sample_ddir(s), (unsigned long long) s->bs,
-				(unsigned long long) so->offset,
-				prio_val);
+			if (log_avg_max)
+				fprintf(f, fmt,
+					s->time,
+					s->data.val.val0,
+					s->data.val.val1,
+					io_sample_ddir(s), (unsigned long long) s->bs,
+					(unsigned long long) so->offset,
+					prio_val);
+			else
+				fprintf(f, fmt,
+					s->time,
+					s->data.val.val0,
+					io_sample_ddir(s), (unsigned long long) s->bs,
+					(unsigned long long) so->offset,
+					prio_val);
 		}
 	}
 }
diff --git a/iolog.h b/iolog.h
index 62cbd1b0..26dd5cca 100644
--- a/iolog.h
+++ b/iolog.h
@@ -26,13 +26,23 @@ struct io_hist {
 	struct flist_head list;
 };
 
+enum {
+	IO_LOG_SAMPLE_AVG = 0,
+	IO_LOG_SAMPLE_MAX,
+	IO_LOG_SAMPLE_BOTH,
+};
+
+struct io_sample_value {
+	uint64_t val0;
+	uint64_t val1;
+};
 
 union io_sample_data {
-	uint64_t val;
+	struct io_sample_value val;
 	struct io_u_plat_entry *plat_entry;
 };
 
-#define sample_val(value) ((union io_sample_data) { .val = value })
+#define sample_val(value) ((union io_sample_data) { .val.val0 = value })
 #define sample_plat(plat) ((union io_sample_data) { .plat_entry = plat })
 
 /*
@@ -154,8 +164,13 @@ struct io_log {
  * If the bit following the upper bit is set, then we have the priority
  */
 #define LOG_PRIO_SAMPLE_BIT	0x40000000U
+/*
+ * If the bit following prioity sample vit is set, we report both avg and max
+ */
+#define LOG_AVG_MAX_SAMPLE_BIT	0x20000000U
 
-#define LOG_SAMPLE_BITS		(LOG_OFFSET_SAMPLE_BIT | LOG_PRIO_SAMPLE_BIT)
+#define LOG_SAMPLE_BITS		(LOG_OFFSET_SAMPLE_BIT | LOG_PRIO_SAMPLE_BIT |\
+					LOG_AVG_MAX_SAMPLE_BIT)
 #define io_sample_ddir(io)	((io)->__ddir & ~LOG_SAMPLE_BITS)
 
 static inline void io_sample_set_ddir(struct io_log *log,
diff --git a/options.c b/options.c
index 53df03de..1da4de78 100644
--- a/options.c
+++ b/options.c
@@ -4540,14 +4540,38 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.group	= FIO_OPT_G_INVALID,
 	},
 	{
-		.name	= "log_max_value",
-		.lname	= "Log maximum instead of average",
-		.type	= FIO_OPT_BOOL,
+		.name	= "log_window_value",
+		.alias  = "log_max_value",
+		.lname	= "Log maximum, average or both values",
+		.type	= FIO_OPT_STR,
 		.off1	= offsetof(struct thread_options, log_max),
-		.help	= "Log max sample in a window instead of average",
-		.def	= "0",
+		.help	= "Log max, average or both sample in a window",
+		.def	= "avg",
 		.category = FIO_OPT_C_LOG,
 		.group	= FIO_OPT_G_INVALID,
+		.posval	= {
+			  { .ival = "avg",
+			    .oval = IO_LOG_SAMPLE_AVG,
+			    .help = "Log average value over the window",
+			  },
+			  { .ival = "max",
+			    .oval = IO_LOG_SAMPLE_MAX,
+			    .help = "Log maximum value in the window",
+			  },
+			  { .ival = "both",
+			    .oval = IO_LOG_SAMPLE_BOTH,
+			    .help = "Log both average and maximum values over the window"
+			  },
+			  /* Compatibility with former boolean values */
+			  { .ival = "0",
+			    .oval = IO_LOG_SAMPLE_AVG,
+			    .help = "Alias for 'avg'",
+			  },
+			  { .ival = "1",
+			    .oval = IO_LOG_SAMPLE_MAX,
+			    .help = "Alias for 'max'",
+			  },
+		},
 	},
 	{
 		.name	= "log_offset",
diff --git a/server.c b/server.c
index b9f0e2ac..afaeb348 100644
--- a/server.c
+++ b/server.c
@@ -2288,8 +2288,10 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
 			struct io_sample *s = get_sample(log, cur_log, i);
 
 			s->time		= cpu_to_le64(s->time);
-			if (log->log_type != IO_LOG_TYPE_HIST)
-				s->data.val	= cpu_to_le64(s->data.val);
+			if (log->log_type != IO_LOG_TYPE_HIST) {
+				s->data.val.val0	= cpu_to_le64(s->data.val.val0);
+				s->data.val.val1	= cpu_to_le64(s->data.val.val1);
+			}
 			s->__ddir	= __cpu_to_le32(s->__ddir);
 			s->bs		= cpu_to_le64(s->bs);
 
diff --git a/stat.c b/stat.c
index 7cf6bee1..11b58626 100644
--- a/stat.c
+++ b/stat.c
@@ -3149,7 +3149,7 @@ void reset_io_stats(struct thread_data *td)
 }
 
 static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir,
-			      unsigned long elapsed, bool log_max)
+			      unsigned long elapsed, int log_max)
 {
 	/*
 	 * Note an entry in the log. Use the mean from the logged samples,
@@ -3159,10 +3159,16 @@ static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir,
 	if (iolog->avg_window[ddir].samples) {
 		union io_sample_data data;
 
-		if (log_max)
-			data.val = iolog->avg_window[ddir].max_val;
-		else
-			data.val = iolog->avg_window[ddir].mean.u.f + 0.50;
+		if (log_max == IO_LOG_SAMPLE_AVG) {
+			data.val.val0 = iolog->avg_window[ddir].mean.u.f + 0.50;
+			data.val.val1 = 0;
+		} else if (log_max == IO_LOG_SAMPLE_MAX) {
+			data.val.val0 = iolog->avg_window[ddir].max_val;
+			data.val.val1 = 0;
+		} else {
+			data.val.val0 = iolog->avg_window[ddir].mean.u.f + 0.50;
+			data.val.val1 = iolog->avg_window[ddir].max_val;
+		}
 
 		__add_log_sample(iolog, data, ddir, 0, elapsed, 0, 0);
 	}
@@ -3171,7 +3177,7 @@ static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir,
 }
 
 static void _add_stat_to_log(struct io_log *iolog, unsigned long elapsed,
-			     bool log_max)
+			     int log_max)
 {
 	enum fio_ddir ddir;
 
@@ -3205,7 +3211,7 @@ static unsigned long add_log_sample(struct thread_data *td,
 	 * Add the sample. If the time period has passed, then
 	 * add that entry to the log and clear.
 	 */
-	add_stat_sample(&iolog->avg_window[ddir], data.val);
+	add_stat_sample(&iolog->avg_window[ddir], data.val.val0);
 
 	/*
 	 * If period hasn't passed, adding the above sample is all we
@@ -3221,7 +3227,7 @@ static unsigned long add_log_sample(struct thread_data *td,
 			return diff;
 	}
 
-	__add_stat_to_log(iolog, ddir, elapsed, td->o.log_max != 0);
+	__add_stat_to_log(iolog, ddir, elapsed, td->o.log_max);
 
 	iolog->avg_last[ddir] = elapsed - (elapsed % iolog->avg_msec);
 
@@ -3235,15 +3241,15 @@ void finalize_logs(struct thread_data *td, bool unit_logs)
 	elapsed = mtime_since_now(&td->epoch);
 
 	if (td->clat_log && unit_logs)
-		_add_stat_to_log(td->clat_log, elapsed, td->o.log_max != 0);
+		_add_stat_to_log(td->clat_log, elapsed, td->o.log_max);
 	if (td->slat_log && unit_logs)
-		_add_stat_to_log(td->slat_log, elapsed, td->o.log_max != 0);
+		_add_stat_to_log(td->slat_log, elapsed, td->o.log_max);
 	if (td->lat_log && unit_logs)
-		_add_stat_to_log(td->lat_log, elapsed, td->o.log_max != 0);
+		_add_stat_to_log(td->lat_log, elapsed, td->o.log_max);
 	if (td->bw_log && (unit_logs == per_unit_log(td->bw_log)))
-		_add_stat_to_log(td->bw_log, elapsed, td->o.log_max != 0);
+		_add_stat_to_log(td->bw_log, elapsed, td->o.log_max);
 	if (td->iops_log && (unit_logs == per_unit_log(td->iops_log)))
-		_add_stat_to_log(td->iops_log, elapsed, td->o.log_max != 0);
+		_add_stat_to_log(td->iops_log, elapsed, td->o.log_max);
 }
 
 void add_agg_sample(union io_sample_data data, enum fio_ddir ddir,
diff --git a/t/io_uring.c b/t/io_uring.c
index bf0aa26e..efc50caa 100644
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -129,7 +129,6 @@ static int batch_complete = BATCH_COMPLETE;
 static int bs = BS;
 static int polled = 1;		/* use IO polling */
 static int fixedbufs = 1;	/* use fixed user buffers */
-static int dma_map;		/* pre-map DMA buffers */
 static int register_files = 1;	/* use fixed files */
 static int buffered = 0;	/* use buffered IO, not O_DIRECT */
 static int sq_thread_poll = 0;	/* use kernel submission/poller thread */
@@ -155,17 +154,6 @@ static float plist[] = { 1.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0,
 			80.0, 90.0, 95.0, 99.0, 99.5, 99.9, 99.95, 99.99 };
 static int plist_len = 17;
 
-#ifndef IORING_REGISTER_MAP_BUFFERS
-#define IORING_REGISTER_MAP_BUFFERS	26
-struct io_uring_map_buffers {
-	__s32	fd;
-	__u32	buf_start;
-	__u32	buf_end;
-	__u32	flags;
-	__u64	rsvd[2];
-};
-#endif
-
 static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
 			 enum nvme_csi csi, void *data)
 {
@@ -405,22 +393,6 @@ static void add_stat(struct submitter *s, int clock_index, int nr)
 #endif
 }
 
-static int io_uring_map_buffers(struct submitter *s)
-{
-	struct io_uring_map_buffers map = {
-		.fd		= s->files[0].real_fd,
-		.buf_end	= depth,
-	};
-
-	if (do_nop)
-		return 0;
-	if (s->nr_files > 1)
-		fprintf(stdout, "Mapping buffers may not work with multiple files\n");
-
-	return syscall(__NR_io_uring_register, s->ring_fd,
-			IORING_REGISTER_MAP_BUFFERS, &map, 1);
-}
-
 static int io_uring_register_buffers(struct submitter *s)
 {
 	if (do_nop)
@@ -950,14 +922,6 @@ static int setup_ring(struct submitter *s)
 			perror("io_uring_register_buffers");
 			return 1;
 		}
-
-		if (dma_map) {
-			ret = io_uring_map_buffers(s);
-			if (ret < 0) {
-				perror("io_uring_map_buffers");
-				return 1;
-			}
-		}
 	}
 
 	if (register_files) {
@@ -1071,7 +1035,7 @@ static int submitter_init(struct submitter *s)
 	}
 
 	if (!init_printed) {
-		printf("polled=%d, fixedbufs=%d/%d, register_files=%d, buffered=%d, QD=%d\n", polled, fixedbufs, dma_map, register_files, buffered, depth);
+		printf("polled=%d, fixedbufs=%d, register_files=%d, buffered=%d, QD=%d\n", polled, fixedbufs, register_files, buffered, depth);
 		printf("%s", buf);
 		init_printed = 1;
 	}
@@ -1519,7 +1483,6 @@ static void usage(char *argv, int status)
 		" -b <int>  : Block size, default %d\n"
 		" -p <bool> : Polled IO, default %d\n"
 		" -B <bool> : Fixed buffers, default %d\n"
-		" -D <bool> : DMA map fixed buffers, default %d\n"
 		" -F <bool> : Register files, default %d\n"
 		" -n <int>  : Number of threads, default %d\n"
 		" -O <bool> : Use O_DIRECT, default %d\n"
@@ -1534,7 +1497,7 @@ static void usage(char *argv, int status)
 		" -P <bool> : Automatically place on device home node %d\n"
 		" -u <bool> : Use nvme-passthrough I/O, default %d\n",
 		argv, DEPTH, BATCH_SUBMIT, BATCH_COMPLETE, BS, polled,
-		fixedbufs, dma_map, register_files, nthreads, !buffered, do_nop,
+		fixedbufs, register_files, nthreads, !buffered, do_nop,
 		stats, runtime == 0 ? "unlimited" : runtime_str, random_io, aio,
 		use_sync, register_ring, numa_placement, pt);
 	exit(status);
@@ -1656,9 +1619,6 @@ int main(int argc, char *argv[])
 		case 'r':
 			runtime = atoi(optarg);
 			break;
-		case 'D':
-			dma_map = !!atoi(optarg);
-			break;
 		case 'R':
 			random_io = !!atoi(optarg);
 			break;
@@ -1694,8 +1654,6 @@ int main(int argc, char *argv[])
 		batch_complete = depth;
 	if (batch_submit > depth)
 		batch_submit = depth;
-	if (!fixedbufs && dma_map)
-		dma_map = 0;
 
 	submitter = calloc(nthreads, sizeof(*submitter) +
 				roundup_pow2(depth) * sizeof(struct iovec));

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-25 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-25 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 1ee0469f6d180a98d31196bea787f37269ff9cdd:

  configure: Don't use cross_prefix when invoking pkg-config (2024-01-23 16:28:31 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 7f250f7514bacef1a3cea24a22ecce8bd30378bd:

  ci: resolve GitHub Actions Node.js warnings (2024-01-24 19:45:33 +0000)

----------------------------------------------------------------
Vincent Fu (1):
      ci: resolve GitHub Actions Node.js warnings

 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b8000024..e53082c3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -64,7 +64,7 @@ jobs:
       if: ${{ contains( matrix.build, 'windows' ) }}
       run: git config --global core.autocrlf input
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Install Cygwin toolchain (Windows)
       if: ${{ startsWith(matrix.build, 'windows-cygwin') }}
       uses: cygwin/cygwin-install-action@master
@@ -110,7 +110,7 @@ jobs:
 
     - name: Upload installer as artifact (Windows)
       if: ${{ contains( matrix.build, 'windows' ) }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{ matrix.build }}-installer
         path: os\windows\*.msi

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-24 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-24 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 8b3190c3ea38af87778a68c576947f8797215d33:

  filesetup: clear O_RDWR flag for verify_only write workloads (2024-01-22 11:51:05 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 1ee0469f6d180a98d31196bea787f37269ff9cdd:

  configure: Don't use cross_prefix when invoking pkg-config (2024-01-23 16:28:31 -0700)

----------------------------------------------------------------
Chris Packham (1):
      configure: Don't use cross_prefix when invoking pkg-config

 configure | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

---

Diff of recent changes:

diff --git a/configure b/configure
index f86fcf77..dea8d07d 100755
--- a/configure
+++ b/configure
@@ -155,11 +155,11 @@ output_sym() {
 check_min_lib_version() {
   _feature=$3
 
-  if "${cross_prefix}"pkg-config --atleast-version="$2" "$1" > /dev/null 2>&1; then
+  if pkg-config --atleast-version="$2" "$1" > /dev/null 2>&1; then
     return 0
   fi
   : "${_feature:=${1}}"
-  if "${cross_prefix}"pkg-config --version > /dev/null 2>&1; then
+  if pkg-config --version > /dev/null 2>&1; then
     if test "$(eval echo \"\$$_feature\")" = "yes" ; then
       feature_not_found "$_feature" "$1 >= $2"
     fi
@@ -1631,14 +1631,14 @@ int main(void)
   return GTK_CHECK_VERSION(2, 18, 0) ? 0 : 1; /* 0 on success */
 }
 EOF
-GTK_CFLAGS=$(${cross_prefix}pkg-config --cflags gtk+-2.0 gthread-2.0)
+GTK_CFLAGS=$(pkg-config --cflags gtk+-2.0 gthread-2.0)
 ORG_LDFLAGS=$LDFLAGS
 LDFLAGS=$(echo $LDFLAGS | sed s/"-static"//g)
 if test "$?" != "0" ; then
   echo "configure: gtk and gthread not found"
   exit 1
 fi
-GTK_LIBS=$(${cross_prefix}pkg-config --libs gtk+-2.0 gthread-2.0)
+GTK_LIBS=$(pkg-config --libs gtk+-2.0 gthread-2.0)
 if test "$?" != "0" ; then
   echo "configure: gtk and gthread not found"
   exit 1

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-23 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-23 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit aa84b5ba581add84ce6e73b20ca0fbd04f6058c8:

  ci: stop hard coding number of jobs for make (2024-01-18 13:00:31 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 8b3190c3ea38af87778a68c576947f8797215d33:

  filesetup: clear O_RDWR flag for verify_only write workloads (2024-01-22 11:51:05 -0500)

----------------------------------------------------------------
Vincent Fu (1):
      filesetup: clear O_RDWR flag for verify_only write workloads

 filesetup.c | 5 +++++
 1 file changed, 5 insertions(+)

---

Diff of recent changes:

diff --git a/filesetup.c b/filesetup.c
index 816d1081..2d277a64 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -749,6 +749,11 @@ open_again:
 		if (!read_only)
 			flags |= O_RDWR;
 
+		if (td->o.verify_only) {
+			flags &= ~O_RDWR;
+			flags |= O_RDONLY;
+		}
+
 		if (f->filetype == FIO_TYPE_FILE && td->o.allow_create)
 			flags |= O_CREAT;
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-19 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-19 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 9eefdcc1dd820a936684168468fa9c81960ea461:

  configure: enable NVME_URING_CMD checking for Android (2024-01-17 09:11:15 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to aa84b5ba581add84ce6e73b20ca0fbd04f6058c8:

  ci: stop hard coding number of jobs for make (2024-01-18 13:00:31 -0500)

----------------------------------------------------------------
Vincent Fu (1):
      ci: stop hard coding number of jobs for make

 ci/actions-build.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/ci/actions-build.sh b/ci/actions-build.sh
index 31d3446c..47d4f044 100755
--- a/ci/actions-build.sh
+++ b/ci/actions-build.sh
@@ -61,7 +61,9 @@ main() {
     configure_flags+=(--extra-cflags="${extra_cflags}")
 
     ./configure "${configure_flags[@]}"
-    make -j 2
+    make -j "$(nproc 2>/dev/null || sysctl -n hw.logicalcpu)"
+# macOS does not have nproc, so we have to use sysctl to obtain the number of
+# logical CPUs.
 }
 
 main

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-18 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-18 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 9f9340cc3a15bca2aa6e883bd5be3d0c9471f573:

  Merge branch 'group_reporting_indentation' of https://github.com/0mp/fio (2024-01-16 09:03:35 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 9eefdcc1dd820a936684168468fa9c81960ea461:

  configure: enable NVME_URING_CMD checking for Android (2024-01-17 09:11:15 -0700)

----------------------------------------------------------------
Jens Axboe (1):
      configure: enable NVME_URING_CMD checking for Android

 configure | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/configure b/configure
index 420d97db..f86fcf77 100755
--- a/configure
+++ b/configure
@@ -2656,7 +2656,7 @@ if test "$libzbc" != "no" ; then
 fi
 print_config "libzbc engine" "$libzbc"
 
-if test "$targetos" = "Linux" ; then
+if test "$targetos" = "Linux" || test "$targetos" = "Android"; then
 ##########################################
 # Check NVME_URING_CMD support
 cat > $TMPC << EOF

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-18 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-18 13:00 UTC (permalink / raw)
  To: linux-btrace

The following changes since commit 7f5d2c5173d72018aa29c583c9291ef10abaf8df:

  blkparse: fix incorrectly sized memset in check_cpu_map (2021-10-21 08:45:17 -0600)

are available in the Git repository at:

  git://git.kernel.dk/blktrace.git master

for you to fetch changes up to b9ea6e507e8849f01d06aa48c0c59c5cee4820be:

  doc: btrace: fix wrong format on doc (2024-01-17 16:02:23 -0700)

----------------------------------------------------------------
Fukui Daichi (1):
      doc: btrace: fix wrong format on doc

 doc/btrace.8 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/doc/btrace.8 b/doc/btrace.8
index a1f34c2..2080e6e 100644
--- a/doc/btrace.8
+++ b/doc/btrace.8
@@ -6,7 +6,7 @@ btrace \- perform live tracing for block devices
 
 
 .SH SYNOPSIS
-.B btrace [\-s] [\-t] [\-w \fIN\fN] [\-n \fIN\fR] [\-b \fIN\fR] [\-r \fI<dbg mnt>\fR] [\-a <\fItrace\fR>...] <\fIdev\fR>...
+.B btrace [\-s] [\-t] [\-w \fIN\fR] [\-n \fIN\fR] [\-b \fIN\fR] [\-r \fI<dbg mnt>\fR] [\-a <\fItrace\fR>...] <\fIdev\fR>...
 .br
 
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2024-01-17 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2024-01-17 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 06c40418f97811092c0aece1760487400bcdd506:

  t/strided: check_result() has no return value (2023-12-28 22:19:44 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 9f9340cc3a15bca2aa6e883bd5be3d0c9471f573:

  Merge branch 'group_reporting_indentation' of https://github.com/0mp/fio (2024-01-16 09:03:35 -0500)

----------------------------------------------------------------
Mateusz Piotrowski (1):
      doc: group_reporting: Fix indentation and syntax

Vincent Fu (1):
      Merge branch 'group_reporting_indentation' of https://github.com/0mp/fio

 HOWTO.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 847c0356..d0ba8021 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -3984,12 +3984,12 @@ Measurements and reporting
 	same reporting group, unless if separated by a :option:`stonewall`, or by
 	using :option:`new_group`.
 
-    NOTE: When :option: `group_reporting` is used along with `json` output,
-    there are certain per-job properties which can be different between jobs
-    but do not have a natural group-level equivalent. Examples include
-    `kb_base`, `unit_base`, `sig_figs`, `thread_number`, `pid`, and
-    `job_start`. For these properties, the values for the first job are
-    recorded for the group.
+	NOTE: When :option:`group_reporting` is used along with `json` output,
+	there are certain per-job properties which can be different between jobs
+	but do not have a natural group-level equivalent. Examples include
+	`kb_base`, `unit_base`, `sig_figs`, `thread_number`, `pid`, and
+	`job_start`. For these properties, the values for the first job are
+	recorded for the group.
 
 .. option:: new_group
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-12-30 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-12-30 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit be943a3ef5d94d8a9fefa11dc004789f66beb8e6:

  t/zbd: add test case to confirm no write with rwmixwrite=0 option (2023-12-19 19:52:35 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 06c40418f97811092c0aece1760487400bcdd506:

  t/strided: check_result() has no return value (2023-12-28 22:19:44 -0500)

----------------------------------------------------------------
Vincent Fu (4):
      t/nvmept: call parent class check_result()
      t/random_seed: call parent class check_result()
      t/strided: call parent class check_result()
      t/strided: check_result() has no return value

 t/nvmept.py      |  2 ++
 t/random_seed.py |  8 ++++++++
 t/strided.py     | 12 +++++++-----
 3 files changed, 17 insertions(+), 5 deletions(-)

---

Diff of recent changes:

diff --git a/t/nvmept.py b/t/nvmept.py
index c08fb350..1ade64dc 100755
--- a/t/nvmept.py
+++ b/t/nvmept.py
@@ -55,6 +55,8 @@ class PassThruTest(FioJobCmdTest):
 
 
     def check_result(self):
+        super().check_result()
+
         if 'rw' not in self.fio_opts:
             return
 
diff --git a/t/random_seed.py b/t/random_seed.py
index 02187046..82beca65 100755
--- a/t/random_seed.py
+++ b/t/random_seed.py
@@ -91,6 +91,10 @@ class TestRR(FioRandTest):
     def check_result(self):
         """Check output for allrandrepeat=1."""
 
+        super().check_result()
+        if not self.passed:
+            return
+
         opt = 'randrepeat' if 'randrepeat' in self.fio_opts else 'allrandrepeat'
         rr = self.fio_opts[opt]
         rand_seeds = self.get_rand_seeds()
@@ -131,6 +135,10 @@ class TestRS(FioRandTest):
     def check_result(self):
         """Check output for randseed=something."""
 
+        super().check_result()
+        if not self.passed:
+            return
+
         rand_seeds = self.get_rand_seeds()
         randseed = self.fio_opts['randseed']
 
diff --git a/t/strided.py b/t/strided.py
index b7655e1e..75c429e4 100755
--- a/t/strided.py
+++ b/t/strided.py
@@ -71,6 +71,10 @@ class StridedTest(FioJobCmdTest):
         super().setup(fio_args)
 
     def check_result(self):
+        super().check_result()
+        if not self.passed:
+            return
+
         zonestart = 0 if 'offset' not in self.fio_opts else self.fio_opts['offset']
         iospersize = self.fio_opts['zonesize'] / self.fio_opts['bs']
         iosperrange = self.fio_opts['zonerange'] / self.fio_opts['bs']
@@ -95,7 +99,7 @@ class StridedTest(FioJobCmdTest):
             offset = int(tokens[4])
             if offset < zonestart or offset >= zonestart + self.fio_opts['zonerange']:
                 print(f"Offset {offset} outside of zone starting at {zonestart}")
-                return False
+                return
 
             # skip next section if norandommap is enabled with no
             # random_generator or with a random_generator != lfsr
@@ -113,17 +117,15 @@ class StridedTest(FioJobCmdTest):
             block = (offset - zonestart) / self.fio_opts['bs']
             if block in zoneset:
                 print(f"Offset {offset} in zone already touched")
-                return False
+                return
 
             zoneset.add(block)
             if iosperzone % iosperrange == 0:
                 if len(zoneset) != iosperrange:
                     print(f"Expected {iosperrange} blocks in zone but only saw {len(zoneset)}")
-                    return False
+                    return
                 zoneset = set()
 
-        return True
-
 
 TEST_LIST = [   # randommap enabled
     {

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-12-20 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-12-20 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit c77fe6859b6ef937a6ca900c1fab009175d721f8:

  engines/http: use proper error value (2023-12-15 13:17:13 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to be943a3ef5d94d8a9fefa11dc004789f66beb8e6:

  t/zbd: add test case to confirm no write with rwmixwrite=0 option (2023-12-19 19:52:35 -0700)

----------------------------------------------------------------
Shin'ichiro Kawasaki (2):
      zbd: avoid write with rwmixwrite=0 option
      t/zbd: add test case to confirm no write with rwmixwrite=0 option

 t/zbd/test-zbd-support | 23 +++++++++++++++++++++++
 zbd.c                  |  3 ++-
 2 files changed, 25 insertions(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/t/zbd/test-zbd-support b/t/zbd/test-zbd-support
index 2f15a191..532860eb 100755
--- a/t/zbd/test-zbd-support
+++ b/t/zbd/test-zbd-support
@@ -1561,6 +1561,29 @@ test67() {
 	grep -q 'Exceeded max_active_zones limit' "${logfile}.${test_number}"
 }
 
+# Test rw=randrw and rwmixwrite=0 options do not issue write I/O unit
+test68() {
+	local off size
+
+	require_zbd || return "$SKIP_TESTCASE"
+
+	reset_zone "${dev}" -1
+
+	# Write some data as preparation
+	off=$((first_sequential_zone_sector * 512))
+	size=$min_seq_write_size
+	run_one_fio_job "$(ioengine "psync")" --rw=write --offset="$off" \
+			--io_size="$size" --zonemode=strided \
+			--zonesize="$zone_size" --zonerange="$zone_size" \
+		       >> "${logfile}.${test_number}" 2>&1 || return $?
+	# Run random mixed read and write specifying zero write ratio
+	run_fio_on_seq "$(ioengine "psync")" --rw=randrw --rwmixwrite=0 \
+		       --time_based --runtime=1s \
+		       >> "${logfile}.${test_number}" 2>&1 || return $?
+	# "WRITE:" shall be recoreded only once for the preparation
+	[[ $(grep -c "WRITE:" "${logfile}.${test_number}") == 1 ]]
+}
+
 SECONDS=0
 tests=()
 dynamic_analyzer=()
diff --git a/zbd.c b/zbd.c
index c4f7b12f..61b5b688 100644
--- a/zbd.c
+++ b/zbd.c
@@ -1876,7 +1876,8 @@ enum fio_ddir zbd_adjust_ddir(struct thread_data *td, struct io_u *io_u,
 	if (ddir != DDIR_READ || !td_rw(td))
 		return ddir;
 
-	if (io_u->file->last_start[DDIR_WRITE] != -1ULL || td->o.read_beyond_wp)
+	if (io_u->file->last_start[DDIR_WRITE] != -1ULL ||
+	    td->o.read_beyond_wp || td->o.rwmix[DDIR_WRITE] == 0)
 		return DDIR_READ;
 
 	return DDIR_WRITE;

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-12-16 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-12-16 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 4e472f8806571ea5799bc898e44609697ba0e140:

  Merge branch 'master' of https://github.com/preichl/fio (2023-12-14 14:15:34 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to c77fe6859b6ef937a6ca900c1fab009175d721f8:

  engines/http: use proper error value (2023-12-15 13:17:13 -0700)

----------------------------------------------------------------
Jens Axboe (2):
      Merge branch 'master' of https://github.com/preichl/fio
      engines/http: use proper error value

Pavel Reichl (3):
      engines/rdma: remove dead code
      client/server: remove dead code
      engines/http: Drop unused varible

 engines/http.c | 3 +--
 engines/rdma.c | 1 -
 server.c       | 1 -
 3 files changed, 1 insertion(+), 4 deletions(-)

---

Diff of recent changes:

diff --git a/engines/http.c b/engines/http.c
index 83cfe8bb..99f4e119 100644
--- a/engines/http.c
+++ b/engines/http.c
@@ -640,7 +640,6 @@ static enum fio_q_status fio_http_queue(struct thread_data *td,
 	char url[1024];
 	long status;
 	CURLcode res;
-	int r = -1;
 
 	fio_ro_check(td, io_u);
 	memset(&_curl_stream, 0, sizeof(_curl_stream));
@@ -712,7 +711,7 @@ static enum fio_q_status fio_http_queue(struct thread_data *td,
 	log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
 
 err:
-	io_u->error = r;
+	io_u->error = EIO;
 	td_verror(td, io_u->error, "transfer");
 out:
 	curl_slist_free_all(slist);
diff --git a/engines/rdma.c b/engines/rdma.c
index ebdbcb1c..07336f3b 100644
--- a/engines/rdma.c
+++ b/engines/rdma.c
@@ -276,7 +276,6 @@ static int cq_event_handler(struct thread_data *td, enum ibv_wc_opcode opcode)
 	int i;
 
 	while ((ret = ibv_poll_cq(rd->cq, 1, &wc)) == 1) {
-		ret = 0;
 		compevnum++;
 
 		if (wc.status) {
diff --git a/server.c b/server.c
index 06eac584..b9f0e2ac 100644
--- a/server.c
+++ b/server.c
@@ -1883,7 +1883,6 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
 
 		offset = (char *)extended_buf_wp - (char *)extended_buf;
 		ptr->ts.ss_bw_data_offset = cpu_to_le64(offset);
-		extended_buf_wp = ss_bw + (int) ts->ss_dur;
 	}
 
 	fio_net_queue_cmd(FIO_NET_CMD_TS, extended_buf, extended_buf_size, NULL, SK_F_COPY);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-12-15 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-12-15 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit f53eaac02ec46bdf7f87058f30667be80975caf6:

  engines/io_uring_cmd: skip pi verify checks for error cases (2023-12-12 09:39:06 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 4e472f8806571ea5799bc898e44609697ba0e140:

  Merge branch 'master' of https://github.com/preichl/fio (2023-12-14 14:15:34 -0700)

----------------------------------------------------------------
Jens Axboe (2):
      Merge branch 'patch-3' of https://github.com/0mp/fio
      Merge branch 'master' of https://github.com/preichl/fio

Mateusz Piotrowski (1):
      doc: Reference geom(4) for FreeBSD users

Pavel Reichl (1):
      engines/http: Fix memory leak

 HOWTO.rst      | 2 +-
 engines/http.c | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index d173702b..847c0356 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -801,7 +801,7 @@ Target file/device
 
 	On Windows, disk devices are accessed as :file:`\\\\.\\PhysicalDrive0` for
 	the first device, :file:`\\\\.\\PhysicalDrive1` for the second etc.
-	Note: Windows and FreeBSD prevent write access to areas
+	Note: Windows and FreeBSD (refer to geom(4)) prevent write access to areas
 	of the disk containing in-use data (e.g. filesystems).
 
 	The filename "`-`" is a reserved name, meaning *stdin* or *stdout*.  Which
diff --git a/engines/http.c b/engines/http.c
index 56dc7d1b..83cfe8bb 100644
--- a/engines/http.c
+++ b/engines/http.c
@@ -250,6 +250,7 @@ static char *_aws_uriencode(const char *uri)
 	for (i = 0; (c = uri[i]); i++) {
 		if (n > bufsize-5) {
 			log_err("encoding the URL failed\n");
+			free(r);
 			return NULL;
 		}
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-12-13 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-12-13 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 63e6f55a9147cf9f76376c2e7e38a623c8832f23:

  Merge branch 'master' of https://github.com/bvanassche/fio (2023-12-11 16:27:21 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to f53eaac02ec46bdf7f87058f30667be80975caf6:

  engines/io_uring_cmd: skip pi verify checks for error cases (2023-12-12 09:39:06 -0500)

----------------------------------------------------------------
Ankit Kumar (1):
      engines/io_uring_cmd: skip pi verify checks for error cases

 engines/io_uring.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 5ae3135b..c0cb5a78 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -468,10 +468,12 @@ static struct io_u *fio_ioring_cmd_event(struct thread_data *td, int event)
 	cqe = &ld->cq_ring.cqes[index];
 	io_u = (struct io_u *) (uintptr_t) cqe->user_data;
 
-	if (cqe->res != 0)
+	if (cqe->res != 0) {
 		io_u->error = -cqe->res;
-	else
+		return io_u;
+	} else {
 		io_u->error = 0;
+	}
 
 	if (o->cmd_type == FIO_URING_CMD_NVME) {
 		data = FILE_ENG_DATA(io_u->file);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-12-12 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-12-12 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit bdf99b6836d75683cba5968c40f321748482ae86:

  Merge branch 'xnvme_includes' of https://github.com/safl/fio (2023-11-20 07:43:16 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 63e6f55a9147cf9f76376c2e7e38a623c8832f23:

  Merge branch 'master' of https://github.com/bvanassche/fio (2023-12-11 16:27:21 -0500)

----------------------------------------------------------------
Bart Van Assche (1):
      Fall back to F_SET_RW_HINT if F_SET_FILE_RW_HINT is not supported

Vincent Fu (2):
      engines/io_uring_cmd: friendlier bad bs error msg
      Merge branch 'master' of https://github.com/bvanassche/fio

 engines/io_uring.c | 19 +++++++++++++------
 ioengines.c        | 22 ++++++++++++----------
 2 files changed, 25 insertions(+), 16 deletions(-)

---

Diff of recent changes:

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 38c36fdc..5ae3135b 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -1279,14 +1279,21 @@ static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f)
 		lba_size = data->lba_ext ? data->lba_ext : data->lba_size;
 
 		for_each_rw_ddir(ddir) {
-			if (td->o.min_bs[ddir] % lba_size ||
-				td->o.max_bs[ddir] % lba_size) {
-				if (data->lba_ext)
-					log_err("%s: block size must be a multiple of (LBA data size + Metadata size)\n",
-						f->file_name);
-				else
+			if (td->o.min_bs[ddir] % lba_size || td->o.max_bs[ddir] % lba_size) {
+				if (data->lba_ext) {
+					log_err("%s: block size must be a multiple of %u "
+						"(LBA data size + Metadata size)\n", f->file_name, lba_size);
+					if (td->o.min_bs[ddir] == td->o.max_bs[ddir] &&
+					    !(td->o.min_bs[ddir] % data->lba_size)) {
+						/* fixed block size is actually a multiple of LBA data size */
+						unsigned long long suggestion = lba_size *
+							(td->o.min_bs[ddir] / data->lba_size);
+						log_err("Did you mean to use a block size of %llu?\n", suggestion);
+					}
+				} else {
 					log_err("%s: block size must be a multiple of LBA data size\n",
 						f->file_name);
+				}
 				td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
 				return 1;
 			}
diff --git a/ioengines.c b/ioengines.c
index 36172725..87cc2286 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -590,19 +590,21 @@ int td_io_open_file(struct thread_data *td, struct fio_file *f)
 	if (fio_option_is_set(&td->o, write_hint) &&
 	    (f->filetype == FIO_TYPE_BLOCK || f->filetype == FIO_TYPE_FILE)) {
 		uint64_t hint = td->o.write_hint;
-		int cmd;
+		int res;
 
 		/*
-		 * For direct IO, we just need/want to set the hint on
-		 * the file descriptor. For buffered IO, we need to set
-		 * it on the inode.
+		 * For direct IO, set the hint on the file descriptor if that is
+		 * supported. Otherwise set it on the inode. For buffered IO, we
+		 * need to set it on the inode.
 		 */
-		if (td->o.odirect)
-			cmd = F_SET_FILE_RW_HINT;
-		else
-			cmd = F_SET_RW_HINT;
-
-		if (fcntl(f->fd, cmd, &hint) < 0) {
+		if (td->o.odirect) {
+			res = fcntl(f->fd, F_SET_FILE_RW_HINT, &hint);
+			if (res < 0)
+				res = fcntl(f->fd, F_SET_RW_HINT, &hint);
+		} else {
+			res = fcntl(f->fd, F_SET_RW_HINT, &hint);
+		}
+		if (res < 0) {
 			td_verror(td, errno, "fcntl write hint");
 			goto err;
 		}

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-11-20 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-11-20 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit afdde534004397ba1fb00ccc6f5906fa50dd667f:

  t/jobs/t0012.fio: make this job time_based (2023-11-07 12:22:40 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to bdf99b6836d75683cba5968c40f321748482ae86:

  Merge branch 'xnvme_includes' of https://github.com/safl/fio (2023-11-20 07:43:16 -0500)

----------------------------------------------------------------
Simon A. F. Lund (1):
      engines/xnvme: only include entry-header ('libxnvme.h')

Vincent Fu (1):
      Merge branch 'xnvme_includes' of https://github.com/safl/fio

 engines/xnvme.c | 4 ----
 1 file changed, 4 deletions(-)

---

Diff of recent changes:

diff --git a/engines/xnvme.c b/engines/xnvme.c
index b7824013..2a0b3520 100644
--- a/engines/xnvme.c
+++ b/engines/xnvme.c
@@ -10,10 +10,6 @@
 #include <stdlib.h>
 #include <assert.h>
 #include <libxnvme.h>
-#include <libxnvme_libconf.h>
-#include <libxnvme_nvm.h>
-#include <libxnvme_znd.h>
-#include <libxnvme_spec_fs.h>
 #include "fio.h"
 #include "zbd_types.h"
 #include "fdp.h"

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-11-08 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-11-08 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 05fce19c7d2668adb38243636a1781c0f8fae523:

  docs: add warning to per_job_logs option (2023-11-06 13:46:30 -0500)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to afdde534004397ba1fb00ccc6f5906fa50dd667f:

  t/jobs/t0012.fio: make this job time_based (2023-11-07 12:22:40 -0500)

----------------------------------------------------------------
Vincent Fu (1):
      t/jobs/t0012.fio: make this job time_based

 t/jobs/t0012.fio | 1 +
 1 file changed, 1 insertion(+)

---

Diff of recent changes:

diff --git a/t/jobs/t0012.fio b/t/jobs/t0012.fio
index d7123966..e01d2b01 100644
--- a/t/jobs/t0012.fio
+++ b/t/jobs/t0012.fio
@@ -14,6 +14,7 @@ flow_sleep=100
 thread
 log_avg_msec=1000
 write_iops_log=t0012.fio
+time_based
 
 [flow1]
 flow=1

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-11-07 13:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-11-07 13:00 UTC (permalink / raw)
  To: fio

The following changes since commit 2c0b784a12172da1533dfd40b66a0e4e5609065f:

  Merge branch 'thinkcycles-parameter' of https://github.com/cloehle/fio (2023-11-03 11:21:22 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 05fce19c7d2668adb38243636a1781c0f8fae523:

  docs: add warning to per_job_logs option (2023-11-06 13:46:30 -0500)

----------------------------------------------------------------
Vincent Fu (2):
      client/server: enable per_job_logs option
      docs: add warning to per_job_logs option

 HOWTO.rst |  8 +++++---
 client.c  | 18 ++++++++++++++----
 fio.1     |  6 ++++--
 server.c  |  1 +
 server.h  |  3 ++-
 5 files changed, 26 insertions(+), 10 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 42b2b119..d173702b 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -3968,9 +3968,11 @@ Measurements and reporting
 
 .. option:: per_job_logs=bool
 
-	If set, this generates bw/clat/iops log with per file private filenames. If
-	not set, jobs with identical names will share the log filename. Default:
-	true.
+        If set to true, fio generates bw/clat/iops logs with per job unique
+        filenames. If set to false, jobs with identical names will share a log
+        filename. Note that when this option is set to false log files will be
+        opened in append mode and if log files already exist the previous
+        contents will not be overwritten. Default: true.
 
 .. option:: group_reporting
 
diff --git a/client.c b/client.c
index 345fa910..699a2e5b 100644
--- a/client.c
+++ b/client.c
@@ -1452,10 +1452,13 @@ static int fio_client_handle_iolog(struct fio_client *client,
 	if (store_direct) {
 		ssize_t wrote;
 		size_t sz;
-		int fd;
+		int fd, flags;
 
-		fd = open((const char *) log_pathname,
-				O_WRONLY | O_CREAT | O_TRUNC, 0644);
+		if (pdu->per_job_logs)
+			flags = O_WRONLY | O_CREAT | O_TRUNC;
+		else
+			flags = O_WRONLY | O_CREAT | O_APPEND;
+		fd = open((const char *) log_pathname, flags, 0644);
 		if (fd < 0) {
 			log_err("fio: open log %s: %s\n",
 				log_pathname, strerror(errno));
@@ -1476,7 +1479,13 @@ static int fio_client_handle_iolog(struct fio_client *client,
 		ret = 0;
 	} else {
 		FILE *f;
-		f = fopen((const char *) log_pathname, "w");
+		const char *mode;
+
+		if (pdu->per_job_logs)
+			mode = "w";
+		else
+			mode = "a";
+		f = fopen((const char *) log_pathname, mode);
 		if (!f) {
 			log_err("fio: fopen log %s : %s\n",
 				log_pathname, strerror(errno));
@@ -1695,6 +1704,7 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd,
 	ret->log_offset		= le32_to_cpu(ret->log_offset);
 	ret->log_prio		= le32_to_cpu(ret->log_prio);
 	ret->log_hist_coarseness = le32_to_cpu(ret->log_hist_coarseness);
+	ret->per_job_logs	= le32_to_cpu(ret->per_job_logs);
 
 	if (*store_direct)
 		return ret;
diff --git a/fio.1 b/fio.1
index d62da688..8f659f1d 100644
--- a/fio.1
+++ b/fio.1
@@ -3667,8 +3667,10 @@ interpreted in seconds.
 .SS "Measurements and reporting"
 .TP
 .BI per_job_logs \fR=\fPbool
-If set, this generates bw/clat/iops log with per file private filenames. If
-not set, jobs with identical names will share the log filename. Default:
+If set to true, fio generates bw/clat/iops logs with per job unique filenames.
+If set to false, jobs with identical names will share a log filename. Note that
+when this option is set to false log files will be opened in append mode and if
+log files already exist the previous contents will not be overwritten. Default:
 true.
 .TP
 .BI group_reporting
diff --git a/server.c b/server.c
index 27332e32..06eac584 100644
--- a/server.c
+++ b/server.c
@@ -2260,6 +2260,7 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
 		.thread_number		= cpu_to_le32(td->thread_number),
 		.log_type		= cpu_to_le32(log->log_type),
 		.log_hist_coarseness	= cpu_to_le32(log->hist_coarseness),
+		.per_job_logs		= cpu_to_le32(td->o.per_job_logs),
 	};
 	struct sk_entry *first;
 	struct flist_head *entry;
diff --git a/server.h b/server.h
index ad706118..0eb594ce 100644
--- a/server.h
+++ b/server.h
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 101,
+	FIO_SERVER_VER			= 102,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,
@@ -198,6 +198,7 @@ struct cmd_iolog_pdu {
 	uint32_t log_offset;
 	uint32_t log_prio;
 	uint32_t log_hist_coarseness;
+	uint32_t per_job_logs;
 	uint8_t name[FIO_NET_NAME_MAX];
 	struct io_sample samples[0];
 };

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-11-04 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-11-04 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 48cf0c63e5b867c8953f25deaa02466bf94a2eed:

  engines/xnvme: fix fdp support for userspace drivers (2023-11-02 06:08:13 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 2c0b784a12172da1533dfd40b66a0e4e5609065f:

  Merge branch 'thinkcycles-parameter' of https://github.com/cloehle/fio (2023-11-03 11:21:22 -0400)

----------------------------------------------------------------
Christian Loehle (1):
      fio: Introduce new constant thinkcycles option

Vincent Fu (1):
      Merge branch 'thinkcycles-parameter' of https://github.com/cloehle/fio

 HOWTO.rst        |  8 ++++++++
 backend.c        |  4 ++++
 cconv.c          |  2 ++
 fio.1            |  7 +++++++
 fio_time.h       |  1 +
 options.c        | 12 ++++++++++++
 thread_options.h |  8 ++++++--
 time.c           | 11 +++++++++++
 8 files changed, 51 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 34d6afdf..42b2b119 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -3209,6 +3209,14 @@ I/O depth
 I/O rate
 ~~~~~~~~
 
+.. option:: thinkcycles=int
+
+	Stall the job for the specified number of cycles after an I/O has completed before
+	issuing the next. May be used to simulate processing being done by an application.
+	This is not taken into account for the time to be waited on for  :option:`thinktime`.
+	Might not have any effect on some platforms, this can be checked by trying a setting
+	a high enough amount of thinkcycles.
+
 .. option:: thinktime=time
 
 	Stall the job for the specified period of time after an I/O has completed before issuing the
diff --git a/backend.c b/backend.c
index a5895fec..1fab467a 100644
--- a/backend.c
+++ b/backend.c
@@ -49,6 +49,7 @@
 #include "helper_thread.h"
 #include "pshared.h"
 #include "zone-dist.h"
+#include "fio_time.h"
 
 static struct fio_sem *startup_sem;
 static struct flist_head *cgroup_list;
@@ -1133,6 +1134,9 @@ reap:
 		if (ret < 0)
 			break;
 
+		if (ddir_rw(ddir) && td->o.thinkcycles)
+			cycles_spin(td->o.thinkcycles);
+
 		if (ddir_rw(ddir) && td->o.thinktime)
 			handle_thinktime(td, ddir, &comp_time);
 
diff --git a/cconv.c b/cconv.c
index 341388d4..c9298408 100644
--- a/cconv.c
+++ b/cconv.c
@@ -233,6 +233,7 @@ int convert_thread_options_to_cpu(struct thread_options *o,
 	o->random_generator = le32_to_cpu(top->random_generator);
 	o->hugepage_size = le32_to_cpu(top->hugepage_size);
 	o->rw_min_bs = le64_to_cpu(top->rw_min_bs);
+	o->thinkcycles = le32_to_cpu(top->thinkcycles);
 	o->thinktime = le32_to_cpu(top->thinktime);
 	o->thinktime_spin = le32_to_cpu(top->thinktime_spin);
 	o->thinktime_blocks = le32_to_cpu(top->thinktime_blocks);
@@ -472,6 +473,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->random_generator = cpu_to_le32(o->random_generator);
 	top->hugepage_size = cpu_to_le32(o->hugepage_size);
 	top->rw_min_bs = __cpu_to_le64(o->rw_min_bs);
+	top->thinkcycles = cpu_to_le32(o->thinkcycles);
 	top->thinktime = cpu_to_le32(o->thinktime);
 	top->thinktime_spin = cpu_to_le32(o->thinktime_spin);
 	top->thinktime_blocks = cpu_to_le32(o->thinktime_blocks);
diff --git a/fio.1 b/fio.1
index c4742aa9..d62da688 100644
--- a/fio.1
+++ b/fio.1
@@ -2962,6 +2962,13 @@ reporting if I/O gets backed up on the device side (the coordinated omission
 problem). Note that this option cannot reliably be used with async IO engines.
 .SS "I/O rate"
 .TP
+.BI thinkcycles \fR=\fPint
+Stall the job for the specified number of cycles after an I/O has completed before
+issuing the next. May be used to simulate processing being done by an application.
+This is not taken into account for the time to be waited on for \fBthinktime\fR.
+Might not have any effect on some platforms, this can be checked by trying a setting
+a high enough amount of thinkcycles.
+.TP
 .BI thinktime \fR=\fPtime
 Stall the job for the specified period of time after an I/O has completed before issuing the
 next. May be used to simulate processing being done by an application.
diff --git a/fio_time.h b/fio_time.h
index b20e734c..969ad68d 100644
--- a/fio_time.h
+++ b/fio_time.h
@@ -22,6 +22,7 @@ extern uint64_t time_since_now(const struct timespec *);
 extern uint64_t time_since_genesis(void);
 extern uint64_t mtime_since_genesis(void);
 extern uint64_t utime_since_genesis(void);
+extern void cycles_spin(unsigned int);
 extern uint64_t usec_spin(unsigned int);
 extern uint64_t usec_sleep(struct thread_data *, unsigned long);
 extern void fill_start_time(struct timespec *);
diff --git a/options.c b/options.c
index 6b2cb53f..53df03de 100644
--- a/options.c
+++ b/options.c
@@ -3875,6 +3875,18 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.category = FIO_OPT_C_IO,
 		.group	= FIO_OPT_G_THINKTIME,
 	},
+	{
+		.name	= "thinkcycles",
+		.lname	= "Think cycles",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct thread_options, thinkcycles),
+		.help	= "Spin for a constant amount of cycles between requests",
+		.def	= "0",
+		.parent	= "thinktime",
+		.hide	= 1,
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_THINKTIME,
+	},
 	{
 		.name	= "thinktime_blocks",
 		.lname	= "Thinktime blocks",
diff --git a/thread_options.h b/thread_options.h
index fdde055e..24f695fe 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -309,6 +309,8 @@ struct thread_options {
 	char *exec_prerun;
 	char *exec_postrun;
 
+	unsigned int thinkcycles;
+
 	unsigned int thinktime;
 	unsigned int thinktime_spin;
 	unsigned int thinktime_blocks;
@@ -355,8 +357,8 @@ struct thread_options {
 
 	unsigned long long latency_target;
 	unsigned long long latency_window;
-	fio_fp64_t latency_percentile;
 	uint32_t latency_run;
+	fio_fp64_t latency_percentile;
 
 	/*
 	 * flow support
@@ -626,6 +628,8 @@ struct thread_options_pack {
 	uint8_t exec_prerun[FIO_TOP_STR_MAX];
 	uint8_t exec_postrun[FIO_TOP_STR_MAX];
 
+	uint32_t thinkcycles;
+
 	uint32_t thinktime;
 	uint32_t thinktime_spin;
 	uint32_t thinktime_blocks;
@@ -671,8 +675,8 @@ struct thread_options_pack {
 	uint64_t latency_target;
 	uint64_t latency_window;
 	uint64_t max_latency[DDIR_RWDIR_CNT];
-	fio_fp64_t latency_percentile;
 	uint32_t latency_run;
+	fio_fp64_t latency_percentile;
 
 	/*
 	 * flow support
diff --git a/time.c b/time.c
index 7cbab6ff..7f85c8de 100644
--- a/time.c
+++ b/time.c
@@ -38,6 +38,17 @@ uint64_t usec_spin(unsigned int usec)
 	return t;
 }
 
+/*
+ * busy loop for a fixed amount of cycles
+ */
+void cycles_spin(unsigned int n)
+{
+	unsigned long i;
+
+	for (i=0; i < n; i++)
+		nop;
+}
+
 uint64_t usec_sleep(struct thread_data *td, unsigned long usec)
 {
 	struct timespec req;

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-11-03 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-11-03 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 95f4d3f054464e997ae1067dc7f4f8ec3f896ccc:

  Merge branch 'pi-perf' of https://github.com/ankit-sam/fio (2023-10-31 09:27:15 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 48cf0c63e5b867c8953f25deaa02466bf94a2eed:

  engines/xnvme: fix fdp support for userspace drivers (2023-11-02 06:08:13 -0600)

----------------------------------------------------------------
Ankit Kumar (1):
      engines/xnvme: fix fdp support for userspace drivers

 engines/xnvme.c        |  2 +-
 examples/xnvme-fdp.fio | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/engines/xnvme.c b/engines/xnvme.c
index ce7b2bdd..b7824013 100644
--- a/engines/xnvme.c
+++ b/engines/xnvme.c
@@ -964,7 +964,7 @@ static int xnvme_fioe_fetch_ruhs(struct thread_data *td, struct fio_file *f,
 	uint32_t nsid;
 	int err = 0, err_lock;
 
-	if (f->filetype != FIO_TYPE_CHAR) {
+	if (f->filetype != FIO_TYPE_CHAR && f->filetype != FIO_TYPE_FILE) {
 		log_err("ioeng->fdp_ruhs(): ignoring filetype: %d\n", f->filetype);
 		return -EINVAL;
 	}
diff --git a/examples/xnvme-fdp.fio b/examples/xnvme-fdp.fio
index 86fbe0d3..c50959f1 100644
--- a/examples/xnvme-fdp.fio
+++ b/examples/xnvme-fdp.fio
@@ -16,6 +16,26 @@
 ;   --xnvme_sync=nvme \
 ;   --filename=/dev/ng0n1
 ;
+; # Use the xNVMe io-engine engine with SPDK backend, note that you have to set the Namespace-id
+; fio examples/xnvme-fdp.fio \
+;   --section=default \
+;   --ioengine=xnvme \
+;   --xnvme_dev_nsid=1 \
+;   --filename=0000\\:01\\:00.0
+;
+; NOTE: The URI encoded in the filename above, the ":" must be escaped.
+;
+; On the command-line using two "\\":
+;
+; --filename=0000\\:01\\:00.0
+;
+; Within a fio-script using a single "\":
+;
+; filename=0000\:01\:00.0
+;
+; NOTE: If you want to override the default bs, iodepth, and workload, then
+; invoke it as:
+;
 ; FIO_BS="512" FIO_RW="read" FIO_IODEPTH=16 fio examples/xnvme-fdp.fio \
 ;   --section=override --ioengine=xnvme --xnvme_sync=nvme --filename=/dev/ng0n1
 ;

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-11-01 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-11-01 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 7a725c78547f7337dddb6fd391f80914f671e583:

  Merge branch 'englist' of https://github.com/vt-alt/fio (2023-10-25 17:53:40 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 95f4d3f054464e997ae1067dc7f4f8ec3f896ccc:

  Merge branch 'pi-perf' of https://github.com/ankit-sam/fio (2023-10-31 09:27:15 -0600)

----------------------------------------------------------------
Ankit Kumar (1):
      crct10: use isa-l for crc if available

Jens Axboe (1):
      Merge branch 'pi-perf' of https://github.com/ankit-sam/fio

 HOWTO.rst              |  4 ++++
 configure              | 29 +++++++++++++++++++++++++++++
 crc/crct10dif_common.c | 13 +++++++++++++
 fio.1                  |  4 ++++
 4 files changed, 50 insertions(+)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 6a8fb3e3..34d6afdf 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2507,6 +2507,10 @@ with the caveat that when used on the command line, they must come after the
 	If this is set to 0, fio generates protection information for
 	write case and verifies for read case. Default: 1.
 
+	For 16 bit CRC generation fio will use isa-l if available otherwise
+	it will use the default slower generator.
+	(see: https://github.com/intel/isa-l)
+
 .. option:: pi_chk=str[,str][,str] : [io_uring_cmd]
 
 	Controls the protection information check. This can take one or more
diff --git a/configure b/configure
index 3e3f8132..420d97db 100755
--- a/configure
+++ b/configure
@@ -189,6 +189,7 @@ libiscsi="no"
 libnbd="no"
 libnfs=""
 xnvme=""
+isal=""
 libblkio=""
 libzbc=""
 dfs=""
@@ -262,6 +263,8 @@ for opt do
   ;;
   --disable-xnvme) xnvme="no"
   ;;
+  --disable-isal) isal="no"
+  ;;
   --disable-libblkio) libblkio="no"
   ;;
   --disable-tcmalloc) disable_tcmalloc="yes"
@@ -322,6 +325,7 @@ if test "$show_help" = "yes" ; then
   echo "--enable-libiscsi       Enable iscsi support"
   echo "--enable-libnbd         Enable libnbd (NBD engine) support"
   echo "--disable-xnvme         Disable xnvme support even if found"
+  echo "--disable-isal          Disable isal support even if found"
   echo "--disable-libblkio      Disable libblkio support even if found"
   echo "--disable-libzbc        Disable libzbc even if found"
   echo "--disable-tcmalloc      Disable tcmalloc support"
@@ -2684,6 +2688,28 @@ if test "$xnvme" != "no" ; then
 fi
 print_config "xnvme engine" "$xnvme"
 
+if test "$targetos" = "Linux" ; then
+##########################################
+# Check ISA-L support
+cat > $TMPC << EOF
+#include <isa-l/crc.h>
+#include <stddef.h>
+int main(void)
+{
+  return crc16_t10dif(0, NULL, 4096);
+}
+EOF
+if test "$isal" != "no" ; then
+  if compile_prog "" "-lisal" "ISAL"; then
+    isal="yes"
+    LIBS="-lisal $LIBS"
+  else
+    isal="no"
+  fi
+fi
+print_config "isal" "$isal"
+fi
+
 ##########################################
 # Check if we have libblkio
 if test "$libblkio" != "no" ; then
@@ -3334,6 +3360,9 @@ if test "$xnvme" = "yes" ; then
   echo "LIBXNVME_CFLAGS=$xnvme_cflags" >> $config_host_mak
   echo "LIBXNVME_LIBS=$xnvme_libs" >> $config_host_mak
 fi
+if test "$isal" = "yes" ; then
+  output_sym "CONFIG_LIBISAL"
+fi
 if test "$libblkio" = "yes" ; then
   output_sym "CONFIG_LIBBLKIO"
   echo "LIBBLKIO_CFLAGS=$libblkio_cflags" >> $config_host_mak
diff --git a/crc/crct10dif_common.c b/crc/crct10dif_common.c
index cfb2a1b1..1763b1c6 100644
--- a/crc/crct10dif_common.c
+++ b/crc/crct10dif_common.c
@@ -24,6 +24,17 @@
  *
  */
 
+#ifdef CONFIG_LIBISAL
+#include <isa-l/crc.h>
+
+extern unsigned short fio_crc_t10dif(unsigned short crc,
+				     const unsigned char *buffer,
+				     unsigned int len)
+{
+	return crc16_t10dif(crc, buffer, len);
+}
+
+#else
 #include "crc-t10dif.h"
 
 /* Table generated using the following polynomium:
@@ -76,3 +87,5 @@ extern unsigned short fio_crc_t10dif(unsigned short crc,
 
 	return crc;
 }
+
+#endif
diff --git a/fio.1 b/fio.1
index a8dc8f6c..c4742aa9 100644
--- a/fio.1
+++ b/fio.1
@@ -2263,6 +2263,10 @@ size greater than protection information size, fio will not generate or verify
 the protection information portion of metadata for write or read case
 respectively. If this is set to 0, fio generates protection information for
 write case and verifies for read case. Default: 1.
+
+For 16 bit CRC generation fio will use isa-l if available otherwise it will
+use the default slower generator.
+(see: https://github.com/intel/isa-l)
 .TP
 .BI (io_uring_cmd)pi_chk \fR=\fPstr[,str][,str]
 Controls the protection information check. This can take one or more of these

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-10-26 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-10-26 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit c11e22e92f3796f21eb15eb6ddc1614d9fa4f99d:

  Merge branch 'spellingfixes-2023-10-23' of https://github.com/proact-de/fio (2023-10-23 08:32:46 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 7a725c78547f7337dddb6fd391f80914f671e583:

  Merge branch 'englist' of https://github.com/vt-alt/fio (2023-10-25 17:53:40 -0400)

----------------------------------------------------------------
Vincent Fu (2):
      engines/io_uring_cmd: allocate enough ranges for async trims
      Merge branch 'englist' of https://github.com/vt-alt/fio

Vitaly Chikunov (1):
      nfs: Fix incorrect engine registering for '--enghelp' list

 engines/io_uring.c | 2 +-
 engines/nfs.c      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 05703df8..38c36fdc 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -1196,7 +1196,7 @@ static int fio_ioring_init(struct thread_data *td)
 	    td->o.zone_mode == ZONE_MODE_ZBD)
 		td->io_ops->flags |= FIO_ASYNCIO_SYNC_TRIM;
 	else
-		ld->dsm = calloc(ld->iodepth, sizeof(*ld->dsm));
+		ld->dsm = calloc(td->o.iodepth, sizeof(*ld->dsm));
 
 	return 0;
 }
diff --git a/engines/nfs.c b/engines/nfs.c
index 970962a3..ce748d14 100644
--- a/engines/nfs.c
+++ b/engines/nfs.c
@@ -308,7 +308,7 @@ static int fio_libnfs_close(struct thread_data *td, struct fio_file *f)
 	return ret;
 }
 
-struct ioengine_ops ioengine = {
+static struct ioengine_ops ioengine = {
 	.name		= "nfs",
 	.version	= FIO_IOOPS_VERSION,
 	.setup		= fio_libnfs_setup,

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-10-24 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-10-24 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit d4fdbe3fa91bbcc9583886af35b56cc7b691f8fa:

  Merge branch 'fix-riscv64-cpu-clock' of https://github.com/gilbsgilbs/fio (2023-10-22 18:52:51 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to c11e22e92f3796f21eb15eb6ddc1614d9fa4f99d:

  Merge branch 'spellingfixes-2023-10-23' of https://github.com/proact-de/fio (2023-10-23 08:32:46 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'spellingfixes-2023-10-23' of https://github.com/proact-de/fio

Martin Steigerwald (1):
      Various spelling fixes.

 HOWTO.rst | 14 +++++++-------
 configure |  2 +-
 fio.1     |  8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index cc7124b1..6a8fb3e3 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2344,8 +2344,8 @@ with the caveat that when used on the command line, they must come after the
 
 		cmdprio_bssplit=blocksize/percentage/class/level/hint:...
 
-	This is an extension of the second accepted format that allows to also
-	specify a priority hint.
+	This is an extension of the second accepted format that allows one to
+	also specify a priority hint.
 
 	For all formats, only the read and write data directions are supported,
 	values for trim IOs are ignored. This option is mutually exclusive with
@@ -3014,7 +3014,7 @@ with the caveat that when used on the command line, they must come after the
 	**hugepage**
 		Use hugepages, instead of existing posix memory backend. The
 		memory backend uses hugetlbfs. This require users to allocate
-		hugepages, mount hugetlbfs and set an enviornment variable for
+		hugepages, mount hugetlbfs and set an environment variable for
 		XNVME_HUGETLB_PATH.
 	**spdk**
 		Uses SPDK's memory allocator.
@@ -3047,7 +3047,7 @@ with the caveat that when used on the command line, they must come after the
 	creating but before connecting the libblkio instance. Each property must
 	have the format ``<name>=<value>``. Colons can be escaped as ``\:``.
 	These are set after the engine sets any other properties, so those can
-	be overriden. Available properties depend on the libblkio version in use
+	be overridden. Available properties depend on the libblkio version in use
 	and are listed at
 	https://libblkio.gitlab.io/libblkio/blkio.html#properties
 
@@ -3071,7 +3071,7 @@ with the caveat that when used on the command line, they must come after the
 	connecting but before starting the libblkio instance. Each property must
 	have the format ``<name>=<value>``. Colons can be escaped as ``\:``.
 	These are set after the engine sets any other properties, so those can
-	be overriden. Available properties depend on the libblkio version in use
+	be overridden. Available properties depend on the libblkio version in use
 	and are listed at
 	https://libblkio.gitlab.io/libblkio/blkio.html#properties
 
@@ -3635,8 +3635,8 @@ Threads, processes and job synchronization
 	By default, fio will continue running all other jobs when one job finishes.
 	Sometimes this is not the desired action. Setting ``exitall`` will
 	instead make fio terminate all jobs in the same group. The option
-        ``exit_what`` allows to control which jobs get terminated when ``exitall`` is
-        enabled. The default is ``group`` and does not change the behaviour of
+        ``exit_what`` allows one to control which jobs get terminated when ``exitall``
+        is enabled. The default is ``group`` and does not change the behaviour of
         ``exitall``. The setting ``all`` terminates all jobs. The setting ``stonewall``
         terminates all currently running jobs across all groups and continues execution
         with the next stonewalled group.
diff --git a/configure b/configure
index 742cb7c5..3e3f8132 100755
--- a/configure
+++ b/configure
@@ -334,7 +334,7 @@ if test "$show_help" = "yes" ; then
 fi
 
 cross_prefix=${cross_prefix-${CROSS_COMPILE}}
-# Preferred compiler (can be overriden later after we know the platform):
+# Preferred compiler (can be overridden later after we know the platform):
 #  ${CC} (if set)
 #  ${cross_prefix}gcc (if cross-prefix specified)
 #  gcc if available
diff --git a/fio.1 b/fio.1
index 628e278d..a8dc8f6c 100644
--- a/fio.1
+++ b/fio.1
@@ -2142,7 +2142,7 @@ The third accepted format for this option is:
 cmdprio_bssplit=blocksize/percentage/class/level/hint:...
 .RE
 .P
-This is an extension of the second accepted format that allows to also
+This is an extension of the second accepted format that allows one to also
 specify a priority hint.
 .P
 For all formats, only the read and write data directions are supported, values
@@ -2774,7 +2774,7 @@ This is the default posix memory backend for linux NVMe driver.
 .BI hugepage
 Use hugepages, instead of existing posix memory backend. The memory backend
 uses hugetlbfs. This require users to allocate hugepages, mount hugetlbfs and
-set an enviornment variable for XNVME_HUGETLB_PATH.
+set an environment variable for XNVME_HUGETLB_PATH.
 .TP
 .BI spdk
 Uses SPDK's memory allocator.
@@ -2803,7 +2803,7 @@ support it; see \fIhttps://libblkio.gitlab.io/libblkio/blkio.html#drivers\fR
 A colon-separated list of additional libblkio properties to be set after
 creating but before connecting the libblkio instance. Each property must have
 the format \fB<name>=<value>\fR. Colons can be escaped as \fB\\:\fR. These are
-set after the engine sets any other properties, so those can be overriden.
+set after the engine sets any other properties, so those can be overridden.
 Available properties depend on the libblkio version in use and are listed at
 \fIhttps://libblkio.gitlab.io/libblkio/blkio.html#properties\fR
 .TP
@@ -2821,7 +2821,7 @@ may support it; see \fIhttps://libblkio.gitlab.io/libblkio/blkio.html#drivers\fR
 A colon-separated list of additional libblkio properties to be set after
 connecting but before starting the libblkio instance. Each property must have
 the format \fB<name>=<value>\fR. Colons can be escaped as \fB\\:\fR. These are
-set after the engine sets any other properties, so those can be overriden.
+set after the engine sets any other properties, so those can be overridden.
 Available properties depend on the libblkio version in use and are listed at
 \fIhttps://libblkio.gitlab.io/libblkio/blkio.html#properties\fR
 .TP

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-10-23 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-10-23 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit f8735bf1fb208bc1b6b1ca818413c9e41944e813:

  Merge branch 'master' of https://github.com/michalbiesek/fio (2023-10-20 04:32:39 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to d4fdbe3fa91bbcc9583886af35b56cc7b691f8fa:

  Merge branch 'fix-riscv64-cpu-clock' of https://github.com/gilbsgilbs/fio (2023-10-22 18:52:51 -0600)

----------------------------------------------------------------
Gilbert Gilb's (1):
      riscv64: get clock from `rdtime` instead of `rdcycle`

Jens Axboe (1):
      Merge branch 'fix-riscv64-cpu-clock' of https://github.com/gilbsgilbs/fio

 arch/arch-riscv64.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/arch/arch-riscv64.h b/arch/arch-riscv64.h
index 9b8fd001..8ac33fa3 100644
--- a/arch/arch-riscv64.h
+++ b/arch/arch-riscv64.h
@@ -16,7 +16,7 @@ static inline unsigned long long get_cpu_clock(void)
 {
 	unsigned long val;
 
-	asm volatile("rdcycle %0" : "=r"(val));
+	asm volatile("rdtime %0" : "=r"(val));
 	return val;
 }
 #define ARCH_HAVE_CPU_CLOCK

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-10-20 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-10-20 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit c5d8ce3fc736210ded83b126c71e3225c7ffd7c9:

  ci: explicitly install pygments and certifi on macos (2023-10-16 10:54:21 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to f8735bf1fb208bc1b6b1ca818413c9e41944e813:

  Merge branch 'master' of https://github.com/michalbiesek/fio (2023-10-20 04:32:39 -0600)

----------------------------------------------------------------
Jens Axboe (3):
      Merge branch 'fix_issue_1642' of https://github.com/zqs-Oppenauer/fio
      Fio 3.36
      Merge branch 'master' of https://github.com/michalbiesek/fio

Michal Biesek (1):
      riscv64: add syscall helpers

Shai Levy (2):
      configure: improve pthread_sigmask detection.
      helper_thread: fix pthread_sigmask typo.

Vincent Fu (1):
      Merge branch 'master' of https://github.com/shailevi23/fio

zhuqingsong.0909 (1):
      fix assert failed when timeout during call rate_ddir.

 FIO-VERSION-GEN     |  2 +-
 arch/arch-riscv64.h | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 configure           |  3 +-
 helper_thread.c     |  5 ++--
 io_ddir.h           |  1 +
 io_u.c              | 10 +++++--
 zbd.c               |  1 +
 7 files changed, 101 insertions(+), 7 deletions(-)

---

Diff of recent changes:

diff --git a/FIO-VERSION-GEN b/FIO-VERSION-GEN
index 4b0d56d0..cf8dbb0e 100755
--- a/FIO-VERSION-GEN
+++ b/FIO-VERSION-GEN
@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=FIO-VERSION-FILE
-DEF_VER=fio-3.35
+DEF_VER=fio-3.36
 
 LF='
 '
diff --git a/arch/arch-riscv64.h b/arch/arch-riscv64.h
index a74b7d47..9b8fd001 100644
--- a/arch/arch-riscv64.h
+++ b/arch/arch-riscv64.h
@@ -29,4 +29,90 @@ static inline int arch_init(char *envp[])
 	return 0;
 }
 
+#define __do_syscallM(...) ({						\
+	__asm__ volatile (						\
+		"ecall"							\
+		: "=r"(a0)						\
+		: __VA_ARGS__						\
+		: "memory", "a1");					\
+	(long) a0;							\
+})
+
+#define __do_syscallN(...) ({						\
+	__asm__ volatile (						\
+		"ecall"							\
+		: "=r"(a0)						\
+		: __VA_ARGS__						\
+		: "memory");					\
+	(long) a0;							\
+})
+
+#define __do_syscall0(__n) ({						\
+	register long a7 __asm__("a7") = __n;				\
+	register long a0 __asm__("a0");					\
+									\
+	__do_syscallM("r" (a7));					\
+})
+
+#define __do_syscall1(__n, __a) ({					\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+									\
+	__do_syscallM("r" (a7), "0" (a0));				\
+})
+
+#define __do_syscall2(__n, __a, __b) ({					\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1));			\
+})
+
+#define __do_syscall3(__n, __a, __b, __c) ({				\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+	register __typeof__(__c) a2 __asm__("a2") = __c;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2));		\
+})
+
+#define __do_syscall4(__n, __a, __b, __c, __d) ({			\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+	register __typeof__(__c) a2 __asm__("a2") = __c;		\
+	register __typeof__(__d) a3 __asm__("a3") = __d;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3));\
+})
+
+#define __do_syscall5(__n, __a, __b, __c, __d, __e) ({			\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+	register __typeof__(__c) a2 __asm__("a2") = __c;		\
+	register __typeof__(__d) a3 __asm__("a3") = __d;		\
+	register __typeof__(__e) a4 __asm__("a4") = __e;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3),	\
+			"r"(a4));					\
+})
+
+#define __do_syscall6(__n, __a, __b, __c, __d, __e, __f) ({		\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+	register __typeof__(__c) a2 __asm__("a2") = __c;		\
+	register __typeof__(__d) a3 __asm__("a3") = __d;		\
+	register __typeof__(__e) a4 __asm__("a4") = __e;		\
+	register __typeof__(__f) a5 __asm__("a5") = __f;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3),	\
+			"r" (a4), "r"(a5));				\
+})
+
+#define FIO_ARCH_HAS_SYSCALL
+
 #endif
diff --git a/configure b/configure
index 36184a58..742cb7c5 100755
--- a/configure
+++ b/configure
@@ -864,7 +864,8 @@ cat > $TMPC <<EOF
 #include <signal.h> /* pthread_sigmask() */
 int main(void)
 {
-  return pthread_sigmask(0, NULL, NULL);
+  sigset_t sigmask;
+  return pthread_sigmask(0, NULL, &sigmask);
 }
 EOF
 if compile_prog "" "$LIBS" "pthread_sigmask" ; then
diff --git a/helper_thread.c b/helper_thread.c
index 53dea44b..2a9dabf5 100644
--- a/helper_thread.c
+++ b/helper_thread.c
@@ -106,13 +106,14 @@ static int read_from_pipe(int fd, void *buf, size_t len)
 
 static void block_signals(void)
 {
-#ifdef HAVE_PTHREAD_SIGMASK
+#ifdef CONFIG_PTHREAD_SIGMASK
 	sigset_t sigmask;
 
+	int ret;
+
 	ret = pthread_sigmask(SIG_UNBLOCK, NULL, &sigmask);
 	assert(ret == 0);
 	ret = pthread_sigmask(SIG_BLOCK, &sigmask, NULL);
-	assert(ret == 0);
 #endif
 }
 
diff --git a/io_ddir.h b/io_ddir.h
index 217eb628..280c1e79 100644
--- a/io_ddir.h
+++ b/io_ddir.h
@@ -11,6 +11,7 @@ enum fio_ddir {
 	DDIR_WAIT,
 	DDIR_LAST,
 	DDIR_INVAL = -1,
+	DDIR_TIMEOUT = -2,
 
 	DDIR_RWDIR_CNT = 3,
 	DDIR_RWDIR_SYNC_CNT = 4,
diff --git a/io_u.c b/io_u.c
index 07e5bac5..13187882 100644
--- a/io_u.c
+++ b/io_u.c
@@ -717,7 +717,7 @@ static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir)
 		 * check if the usec is capable of taking negative values
 		 */
 		if (now > td->o.timeout) {
-			ddir = DDIR_INVAL;
+			ddir = DDIR_TIMEOUT;
 			return ddir;
 		}
 		usec = td->o.timeout - now;
@@ -726,7 +726,7 @@ static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir)
 
 	now = utime_since_now(&td->epoch);
 	if ((td->o.timeout && (now > td->o.timeout)) || td->terminate)
-		ddir = DDIR_INVAL;
+		ddir = DDIR_TIMEOUT;
 
 	return ddir;
 }
@@ -951,7 +951,7 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u)
 
 	set_rw_ddir(td, io_u);
 
-	if (io_u->ddir == DDIR_INVAL) {
+	if (io_u->ddir == DDIR_INVAL || io_u->ddir == DDIR_TIMEOUT) {
 		dprint(FD_IO, "invalid direction received ddir = %d", io_u->ddir);
 		return 1;
 	}
@@ -1419,6 +1419,10 @@ static long set_io_u_file(struct thread_data *td, struct io_u *io_u)
 		put_file_log(td, f);
 		td_io_close_file(td, f);
 		io_u->file = NULL;
+
+		if (io_u->ddir == DDIR_TIMEOUT)
+			return 1;
+
 		if (td->o.file_service_type & __FIO_FSERVICE_NONUNIFORM)
 			fio_file_reset(td, f);
 		else {
diff --git a/zbd.c b/zbd.c
index caac68bb..c4f7b12f 100644
--- a/zbd.c
+++ b/zbd.c
@@ -2171,6 +2171,7 @@ retry:
 	case DDIR_WAIT:
 	case DDIR_LAST:
 	case DDIR_INVAL:
+	case DDIR_TIMEOUT:
 		goto accept;
 	}
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-10-17 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-10-17 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 50b94305b08a746c21a2c644ffb3cb56915d86ee:

  t/zbd: avoid test case 45 failure (2023-10-13 17:31:47 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to c5d8ce3fc736210ded83b126c71e3225c7ffd7c9:

  ci: explicitly install pygments and certifi on macos (2023-10-16 10:54:21 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      ci: explicitly install pygments and certifi on macos

 ci/actions-install.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/ci/actions-install.sh b/ci/actions-install.sh
index 95241e78..76335fbc 100755
--- a/ci/actions-install.sh
+++ b/ci/actions-install.sh
@@ -86,7 +86,7 @@ install_macos() {
     #echo "Updating homebrew..."
     #brew update >/dev/null 2>&1
     echo "Installing packages..."
-    HOMEBREW_NO_AUTO_UPDATE=1 brew install cunit libnfs sphinx-doc
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install cunit libnfs sphinx-doc pygments python-certifi
     brew link sphinx-doc --force
     pip3 install scipy six statsmodels
 }

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-10-14 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-10-14 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 16b9f29dab1d105951da663474ec243942fda400:

  Merge branch 'fix-stat-overflow' of https://github.com/stilor/fio (2023-10-06 15:27:23 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 50b94305b08a746c21a2c644ffb3cb56915d86ee:

  t/zbd: avoid test case 45 failure (2023-10-13 17:31:47 -0400)

----------------------------------------------------------------
Shin'ichiro Kawasaki (1):
      t/zbd: avoid test case 45 failure

 t/zbd/test-zbd-support | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

---

Diff of recent changes:

diff --git a/t/zbd/test-zbd-support b/t/zbd/test-zbd-support
index 0436d319..2f15a191 100755
--- a/t/zbd/test-zbd-support
+++ b/t/zbd/test-zbd-support
@@ -1058,15 +1058,20 @@ test44() {
 
 test45() {
     local bs i
+    local grep_str="fio: first I/O failed. If .* is a zoned block device, consider --zonemode=zbd"
 
     require_zbd || return $SKIP_TESTCASE
     prep_write
     bs=$((min_seq_write_size))
-    run_one_fio_job "$(ioengine "psync")" --iodepth=1 --rw=randwrite --bs=$bs\
-		    --offset=$((first_sequential_zone_sector * 512)) \
-		    --size="$zone_size" --do_verify=1 --verify=md5 2>&1 |
-	tee -a "${logfile}.${test_number}" |
-	grep -q "fio: first I/O failed. If .* is a zoned block device, consider --zonemode=zbd"
+    for ((i = 0; i < 10; i++)); do
+	    run_one_fio_job "$(ioengine "psync")" --iodepth=1 --rw=randwrite \
+			    --offset=$((first_sequential_zone_sector * 512)) \
+			    --bs="$bs" --time_based --runtime=1s \
+			    --do_verify=1 --verify=md5 \
+		    >> "${logfile}.${test_number}" 2>&1
+	    grep -qe "$grep_str" "${logfile}.${test_number}" && return 0
+    done
+    return 1
 }
 
 # Random write to sequential zones, libaio, 8 jobs, queue depth 64 per job

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-10-07 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-10-07 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 6f9cdcfcc7598c7d7b19c4a5120a251a80dab183:

  iolog: don't truncate time values (2023-10-02 14:29:29 +0000)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 16b9f29dab1d105951da663474ec243942fda400:

  Merge branch 'fix-stat-overflow' of https://github.com/stilor/fio (2023-10-06 15:27:23 -0400)

----------------------------------------------------------------
Alexey Neyman (2):
      Change memcpy() calls to assignments
      Handle 32-bit overflows in disk utilization stats

Vincent Fu (1):
      Merge branch 'fix-stat-overflow' of https://github.com/stilor/fio

 diskutil.c | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

---

Diff of recent changes:

diff --git a/diskutil.c b/diskutil.c
index cf4ede85..69b3dd26 100644
--- a/diskutil.c
+++ b/diskutil.c
@@ -77,6 +77,23 @@ static int get_io_ticks(struct disk_util *du, struct disk_util_stat *dus)
 	return ret != 10;
 }
 
+static uint64_t safe_32bit_diff(uint64_t nval, uint64_t oval)
+{
+	/* Linux kernel prints some of the stat fields as 32-bit integers. It is
+	 * possible that the value overflows, but since fio uses unsigned 64-bit
+	 * arithmetic in update_io_tick_disk(), it instead results in a huge
+	 * bogus value being added to the respective accumulating field. Just
+	 * in case Linux starts reporting these metrics as 64-bit values in the
+	 * future, check that overflow actually happens around the 32-bit
+	 * unsigned boundary; assume overflow only happens once between
+	 * successive polls.
+	 */
+	if (oval <= nval || oval >= (1ull << 32))
+		return nval - oval;
+	else
+		return (1ull << 32) + nval - oval;
+}
+
 static void update_io_tick_disk(struct disk_util *du)
 {
 	struct disk_util_stat __dus, *dus, *ldus;
@@ -96,15 +113,16 @@ static void update_io_tick_disk(struct disk_util *du)
 	dus->s.ios[1] += (__dus.s.ios[1] - ldus->s.ios[1]);
 	dus->s.merges[0] += (__dus.s.merges[0] - ldus->s.merges[0]);
 	dus->s.merges[1] += (__dus.s.merges[1] - ldus->s.merges[1]);
-	dus->s.ticks[0] += (__dus.s.ticks[0] - ldus->s.ticks[0]);
-	dus->s.ticks[1] += (__dus.s.ticks[1] - ldus->s.ticks[1]);
-	dus->s.io_ticks += (__dus.s.io_ticks - ldus->s.io_ticks);
-	dus->s.time_in_queue += (__dus.s.time_in_queue - ldus->s.time_in_queue);
+	dus->s.ticks[0] += safe_32bit_diff(__dus.s.ticks[0], ldus->s.ticks[0]);
+	dus->s.ticks[1] += safe_32bit_diff(__dus.s.ticks[1], ldus->s.ticks[1]);
+	dus->s.io_ticks += safe_32bit_diff(__dus.s.io_ticks, ldus->s.io_ticks);
+	dus->s.time_in_queue +=
+			safe_32bit_diff(__dus.s.time_in_queue, ldus->s.time_in_queue);
 
 	fio_gettime(&t, NULL);
 	dus->s.msec += mtime_since(&du->time, &t);
-	memcpy(&du->time, &t, sizeof(t));
-	memcpy(&ldus->s, &__dus.s, sizeof(__dus.s));
+	du->time = t;
+	ldus->s = __dus.s;
 }
 
 int update_io_ticks(void)

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-10-03 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-10-03 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit c95b52caacc8ef5c1235fb3754186e981b109bdb:

  ci: switch macos runs from macos-12 to macos-13 (2023-09-29 11:51:10 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 6f9cdcfcc7598c7d7b19c4a5120a251a80dab183:

  iolog: don't truncate time values (2023-10-02 14:29:29 +0000)

----------------------------------------------------------------
Vincent Fu (1):
      iolog: don't truncate time values

 iolog.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

---

Diff of recent changes:

diff --git a/iolog.c b/iolog.c
index 97ba4396..5213c60f 100644
--- a/iolog.c
+++ b/iolog.c
@@ -1002,14 +1002,14 @@ void flush_samples(FILE *f, void *samples, uint64_t sample_size)
 
 	if (log_offset) {
 		if (log_prio)
-			fmt = "%lu, %" PRId64 ", %u, %llu, %llu, 0x%04x\n";
+			fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %llu, 0x%04x\n";
 		else
-			fmt = "%lu, %" PRId64 ", %u, %llu, %llu, %u\n";
+			fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %llu, %u\n";
 	} else {
 		if (log_prio)
-			fmt = "%lu, %" PRId64 ", %u, %llu, 0x%04x\n";
+			fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, 0x%04x\n";
 		else
-			fmt = "%lu, %" PRId64 ", %u, %llu, %u\n";
+			fmt = "%" PRIu64 ", %" PRId64 ", %u, %llu, %u\n";
 	}
 
 	nr_samples = sample_size / __log_entry_sz(log_offset);
@@ -1024,7 +1024,7 @@ void flush_samples(FILE *f, void *samples, uint64_t sample_size)
 
 		if (!log_offset) {
 			fprintf(f, fmt,
-				(unsigned long) s->time,
+				s->time,
 				s->data.val,
 				io_sample_ddir(s), (unsigned long long) s->bs,
 				prio_val);
@@ -1032,7 +1032,7 @@ void flush_samples(FILE *f, void *samples, uint64_t sample_size)
 			struct io_sample_offset *so = (void *) s;
 
 			fprintf(f, fmt,
-				(unsigned long) s->time,
+				s->time,
 				s->data.val,
 				io_sample_ddir(s), (unsigned long long) s->bs,
 				(unsigned long long) so->offset,

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-09-30 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-09-30 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 06812a4f0e4ff4847076e742557ab406a0e96848:

  Merge branch 'fix_verify_block_offset' of https://github.com/ipylypiv/fio (2023-09-29 00:05:10 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to c95b52caacc8ef5c1235fb3754186e981b109bdb:

  ci: switch macos runs from macos-12 to macos-13 (2023-09-29 11:51:10 -0400)

----------------------------------------------------------------
Vincent Fu (2):
      workqueue: handle nice better
      ci: switch macos runs from macos-12 to macos-13

 .github/workflows/ci.yml | 2 +-
 workqueue.c              | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 69fedf77..b8000024 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,7 +27,7 @@ jobs:
           os: ubuntu-22.04
           cc: clang
         - build: macos
-          os: macos-12
+          os: macos-13
         - build: linux-i686-gcc
           os: ubuntu-22.04
           arch: i686
diff --git a/workqueue.c b/workqueue.c
index 9e6c41ff..3636bc3a 100644
--- a/workqueue.c
+++ b/workqueue.c
@@ -136,7 +136,8 @@ static void *worker_thread(void *data)
 	sk_out_assign(sw->sk_out);
 
 	if (wq->ops.nice) {
-		if (nice(wq->ops.nice) < 0) {
+		errno = 0;
+		if (nice(wq->ops.nice) == -1 && errno != 0) {
 			log_err("workqueue: nice %s\n", strerror(errno));
 			ret = 1;
 		}

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-09-29 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-09-29 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 996ac91f54844e63ef43092472fc1f7610567b67:

  t/zbd: set mq-deadline scheduler to device-mapper destination devices (2023-09-26 09:00:13 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 06812a4f0e4ff4847076e742557ab406a0e96848:

  Merge branch 'fix_verify_block_offset' of https://github.com/ipylypiv/fio (2023-09-29 00:05:10 -0600)

----------------------------------------------------------------
Igor Pylypiv (1):
      verify: Fix the bad pattern block offset value

Jens Axboe (1):
      Merge branch 'fix_verify_block_offset' of https://github.com/ipylypiv/fio

 verify.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/verify.c b/verify.c
index f7355f30..78f333e6 100644
--- a/verify.c
+++ b/verify.c
@@ -398,7 +398,8 @@ static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc)
 				(unsigned char)buf[i],
 				(unsigned char)pattern[mod],
 				bits);
-			log_err("fio: bad pattern block offset %u\n", i);
+			log_err("fio: bad pattern block offset %u\n",
+				i + header_size);
 			vc->name = "pattern";
 			log_verify_failure(hdr, vc);
 			return EILSEQ;

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-09-27 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-09-27 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit a142e0df6c1483a76d92ff7f9d8c07242af9910e:

  Merge branch 'fio_client_server_doc_fix' of https://github.com/pcpartpicker/fio (2023-09-20 07:41:17 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 996ac91f54844e63ef43092472fc1f7610567b67:

  t/zbd: set mq-deadline scheduler to device-mapper destination devices (2023-09-26 09:00:13 -0400)

----------------------------------------------------------------
Shin'ichiro Kawasaki (1):
      t/zbd: set mq-deadline scheduler to device-mapper destination devices

 t/zbd/functions        | 11 +++++++++
 t/zbd/test-zbd-support | 61 +++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 71 insertions(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/t/zbd/functions b/t/zbd/functions
index 4faa45a9..028df404 100644
--- a/t/zbd/functions
+++ b/t/zbd/functions
@@ -27,6 +27,17 @@ blkzone_reports_capacity() {
 		"${blkzone}" report -c 1 -o 0 "${dev}" | grep -q 'cap '
 }
 
+has_command() {
+	local cmd="${1}"
+
+	cmd_path=$(type -p "${cmd}" 2>/dev/null)
+	if [ -z "${cmd_path}" ]; then
+		echo "${cmd} is not available"
+		return 1
+	fi
+	return 0
+}
+
 # Whether or not $1 (/dev/...) is a NVME ZNS device.
 is_nvme_zns() {
 	local s
diff --git a/t/zbd/test-zbd-support b/t/zbd/test-zbd-support
index c8f3eb61..0436d319 100755
--- a/t/zbd/test-zbd-support
+++ b/t/zbd/test-zbd-support
@@ -46,6 +46,55 @@ ioengine() {
 	fi
 }
 
+get_dev_path_by_id() {
+	for d in /sys/block/* /sys/block/*/*; do
+		if [[ ! -r "${d}/dev" ]]; then
+			continue
+		fi
+		if [[ "${1}" == "$(<"${d}/dev")" ]]; then
+			echo "/dev/${d##*/}"
+			return 0
+		fi
+	done
+	return 1
+}
+
+dm_destination_dev_set_io_scheduler() {
+	local dev=$1 sched=$2
+	local dest_dev_id dest_dev path
+
+	has_command dmsetup || return 1
+
+	while read -r dest_dev_id; do
+		if ! dest_dev=$(get_dev_path_by_id "${dest_dev_id}"); then
+			continue
+		fi
+		path=${dest_dev/dev/sys\/block}/queue/scheduler
+		if [[ ! -w ${path} ]]; then
+			echo "Can not set scheduler of device mapper destination: ${dest_dev}"
+			continue
+		fi
+		echo "${2}" > "${path}"
+	done < <(dmsetup table "$(<"/sys/block/$dev/dm/name")" |
+			 sed -n  's/.* \([0-9]*:[0-9]*\).*/\1/p')
+}
+
+dev_has_dm_map() {
+	local dev=${1} target_type=${2}
+	local dm_name
+
+	has_command dmsetup || return 1
+
+	dm_name=$(<"/sys/block/$dev/dm/name")
+	if ! dmsetup status "${dm_name}" | grep -qe "${target_type}"; then
+		return 1
+	fi
+	if dmsetup status "${dm_name}" | grep -v "${target_type}"; then
+		return 1
+	fi
+	return 0
+}
+
 set_io_scheduler() {
     local dev=$1 sched=$2
 
@@ -62,7 +111,17 @@ set_io_scheduler() {
 	esac
     fi
 
-    echo "$sched" >"/sys/block/$dev/queue/scheduler"
+    if [ -w "/sys/block/$dev/queue/scheduler" ]; then
+	echo "$sched" >"/sys/block/$dev/queue/scheduler"
+    elif [ -r  "/sys/block/$dev/dm/name" ] &&
+		 ( dev_has_dm_map "$dev" linear ||
+		   dev_has_dm_map "$dev" flakey ||
+		   dev_has_dm_map "$dev" crypt ); then
+	dm_destination_dev_set_io_scheduler "$dev" "$sched"
+    else
+	echo "can not set io scheduler"
+	exit 1
+    fi
 }
 
 check_read() {

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-09-20 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-09-20 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit e2c5f17e3559cc7c96706cd75c2609f12675c60b:

  verify: open state file in binary mode on Windows (2023-09-14 18:54:25 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to a142e0df6c1483a76d92ff7f9d8c07242af9910e:

  Merge branch 'fio_client_server_doc_fix' of https://github.com/pcpartpicker/fio (2023-09-20 07:41:17 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      Merge branch 'fio_client_server_doc_fix' of https://github.com/pcpartpicker/fio

aggieNick02 (1):
      Update docs to clarify how to pass job options in client mode

 HOWTO.rst | 3 +++
 fio.1     | 3 +++
 2 files changed, 6 insertions(+)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 7f26978a..cc7124b1 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -5105,6 +5105,9 @@ is the connect string, and `remote-args` and `job file(s)` are sent to the
 server. The `server` string follows the same format as it does on the server
 side, to allow IP/hostname/socket and port strings.
 
+Note that all job options must be defined in job files when running fio as a
+client. Any job options specified in `remote-args` will be ignored.
+
 Fio can connect to multiple servers this way::
 
     fio --client=<server1> <job file(s)> --client=<server2> <job file(s)>
diff --git a/fio.1 b/fio.1
index 8159caa4..628e278d 100644
--- a/fio.1
+++ b/fio.1
@@ -4838,6 +4838,9 @@ is the connect string, and `remote\-args' and `job file(s)' are sent to the
 server. The `server' string follows the same format as it does on the server
 side, to allow IP/hostname/socket and port strings.
 .P
+Note that all job options must be defined in job files when running fio as a
+client. Any job options specified in `remote\-args' will be ignored.
+.P
 Fio can connect to multiple servers this way:
 .RS
 .P

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-09-16 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-09-16 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit e4a9812dee084b058eca6ebde9634a3d573a0079:

  engines:nvme: fill command fields as per pi check bits (2023-09-11 10:55:56 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to e2c5f17e3559cc7c96706cd75c2609f12675c60b:

  verify: open state file in binary mode on Windows (2023-09-14 18:54:25 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      verify: open state file in binary mode on Windows

 verify.c | 4 ++++
 1 file changed, 4 insertions(+)

---

Diff of recent changes:

diff --git a/verify.c b/verify.c
index 2848b686..f7355f30 100644
--- a/verify.c
+++ b/verify.c
@@ -1648,6 +1648,10 @@ static int open_state_file(const char *name, const char *prefix, int num,
 	else
 		flags = O_RDONLY;
 
+#ifdef _WIN32
+	flags |= O_BINARY;
+#endif
+
 	verify_state_gen_name(out, sizeof(out), name, prefix, num);
 
 	fd = open(out, flags, 0644);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-09-12 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-09-12 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 904ee91c2831615a054a8dea9b164e96ae00abb3:

  Merge branch 'pcpp_parse_nr_fix' of https://github.com/PCPartPicker/fio (2023-09-02 07:35:49 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to e4a9812dee084b058eca6ebde9634a3d573a0079:

  engines:nvme: fill command fields as per pi check bits (2023-09-11 10:55:56 -0600)

----------------------------------------------------------------
Ankit Kumar (2):
      engines:io_uring_cmd: disallow verify for e2e pi with extended blocks
      engines:nvme: fill command fields as per pi check bits

Vincent Fu (1):
      Merge branch 'pcpp_epoch_fixing_2' of https://github.com/PCPartPicker/fio

aggieNick02 (2):
      Record job start time to fix time pain points
      Make log_unix_epoch an official alias of log_alternate_epoch

 HOWTO.rst          | 21 +++++++++++++++------
 backend.c          |  2 +-
 cconv.c            |  4 ++--
 client.c           |  1 +
 engines/io_uring.c | 14 ++++++++++++++
 engines/nvme.c     | 15 ++++++++++-----
 fio.1              | 23 +++++++++++++++++------
 fio.h              |  3 ++-
 fio_time.h         |  2 +-
 libfio.c           |  2 +-
 options.c          | 22 ++++++++++++----------
 rate-submit.c      |  2 +-
 server.c           |  1 +
 stat.c             |  6 +++++-
 stat.h             |  1 +
 thread_options.h   |  8 +++-----
 time.c             | 20 ++++++++++++++------
 17 files changed, 101 insertions(+), 46 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 89032941..7f26978a 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -755,6 +755,10 @@ Time related parameters
 	calls will be excluded from other uses. Fio will manually clear it from the
 	CPU mask of other jobs.
 
+.. option:: job_start_clock_id=int
+   The clock_id passed to the call to `clock_gettime` used to record job_start
+   in the `json` output format. Default is 0, or CLOCK_REALTIME.
+
 
 Target file/device
 ~~~~~~~~~~~~~~~~~~
@@ -3966,6 +3970,13 @@ Measurements and reporting
 	same reporting group, unless if separated by a :option:`stonewall`, or by
 	using :option:`new_group`.
 
+    NOTE: When :option: `group_reporting` is used along with `json` output,
+    there are certain per-job properties which can be different between jobs
+    but do not have a natural group-level equivalent. Examples include
+    `kb_base`, `unit_base`, `sig_figs`, `thread_number`, `pid`, and
+    `job_start`. For these properties, the values for the first job are
+    recorded for the group.
+
 .. option:: new_group
 
 	Start a new reporting group. See: :option:`group_reporting`.  If not given,
@@ -4103,9 +4114,7 @@ Measurements and reporting
 
 .. option:: log_unix_epoch=bool
 
-	If set, fio will log Unix timestamps to the log files produced by enabling
-	write_type_log for each log type, instead of the default zero-based
-	timestamps.
+	Backwards compatible alias for log_alternate_epoch.
 
 .. option:: log_alternate_epoch=bool
 
@@ -4116,9 +4125,9 @@ Measurements and reporting
 
 .. option:: log_alternate_epoch_clock_id=int
 
-	Specifies the clock_id to be used by clock_gettime to obtain the alternate epoch
-	if either log_unix_epoch or log_alternate_epoch are true. Otherwise has no
-	effect. Default value is 0, or CLOCK_REALTIME.
+    Specifies the clock_id to be used by clock_gettime to obtain the alternate
+    epoch if log_alternate_epoch is true. Otherwise has no effect. Default
+    value is 0, or CLOCK_REALTIME.
 
 .. option:: block_error_percentiles=bool
 
diff --git a/backend.c b/backend.c
index 5f074039..a5895fec 100644
--- a/backend.c
+++ b/backend.c
@@ -1858,7 +1858,7 @@ static void *thread_main(void *data)
 	if (rate_submit_init(td, sk_out))
 		goto err;
 
-	set_epoch_time(td, o->log_unix_epoch | o->log_alternate_epoch, o->log_alternate_epoch_clock_id);
+	set_epoch_time(td, o->log_alternate_epoch_clock_id, o->job_start_clock_id);
 	fio_getrusage(&td->ru_start);
 	memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch));
 	memcpy(&td->iops_sample_time, &td->epoch, sizeof(td->epoch));
diff --git a/cconv.c b/cconv.c
index ce6acbe6..341388d4 100644
--- a/cconv.c
+++ b/cconv.c
@@ -216,9 +216,9 @@ int convert_thread_options_to_cpu(struct thread_options *o,
 	o->log_prio = le32_to_cpu(top->log_prio);
 	o->log_gz = le32_to_cpu(top->log_gz);
 	o->log_gz_store = le32_to_cpu(top->log_gz_store);
-	o->log_unix_epoch = le32_to_cpu(top->log_unix_epoch);
 	o->log_alternate_epoch = le32_to_cpu(top->log_alternate_epoch);
 	o->log_alternate_epoch_clock_id = le32_to_cpu(top->log_alternate_epoch_clock_id);
+	o->job_start_clock_id = le32_to_cpu(top->job_start_clock_id);
 	o->norandommap = le32_to_cpu(top->norandommap);
 	o->softrandommap = le32_to_cpu(top->softrandommap);
 	o->bs_unaligned = le32_to_cpu(top->bs_unaligned);
@@ -455,9 +455,9 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->log_prio = cpu_to_le32(o->log_prio);
 	top->log_gz = cpu_to_le32(o->log_gz);
 	top->log_gz_store = cpu_to_le32(o->log_gz_store);
-	top->log_unix_epoch = cpu_to_le32(o->log_unix_epoch);
 	top->log_alternate_epoch = cpu_to_le32(o->log_alternate_epoch);
 	top->log_alternate_epoch_clock_id = cpu_to_le32(o->log_alternate_epoch_clock_id);
+	top->job_start_clock_id = cpu_to_le32(o->job_start_clock_id);
 	top->norandommap = cpu_to_le32(o->norandommap);
 	top->softrandommap = cpu_to_le32(o->softrandommap);
 	top->bs_unaligned = cpu_to_le32(o->bs_unaligned);
diff --git a/client.c b/client.c
index c257036b..345fa910 100644
--- a/client.c
+++ b/client.c
@@ -956,6 +956,7 @@ static void convert_ts(struct thread_stat *dst, struct thread_stat *src)
 	dst->error		= le32_to_cpu(src->error);
 	dst->thread_number	= le32_to_cpu(src->thread_number);
 	dst->groupid		= le32_to_cpu(src->groupid);
+	dst->job_start		= le64_to_cpu(src->job_start);
 	dst->pid		= le32_to_cpu(src->pid);
 	dst->members		= le32_to_cpu(src->members);
 	dst->unified_rw_rep	= le32_to_cpu(src->unified_rw_rep);
diff --git a/engines/io_uring.c b/engines/io_uring.c
index 6cdf1b4f..05703df8 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -18,6 +18,7 @@
 #include "../lib/memalign.h"
 #include "../lib/fls.h"
 #include "../lib/roundup.h"
+#include "../verify.h"
 
 #ifdef ARCH_HAVE_IOURING
 
@@ -1299,6 +1300,19 @@ static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f)
 				return 1;
 			}
                 }
+
+		/*
+		 * For extended logical block sizes we cannot use verify when
+		 * end to end data protection checks are enabled, as the PI
+		 * section of data buffer conflicts with verify.
+		 */
+		if (data->ms && data->pi_type && data->lba_ext &&
+		    td->o.verify != VERIFY_NONE) {
+			log_err("%s: for extended LBA, verify cannot be used when E2E data protection is enabled\n",
+				f->file_name);
+			td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
+			return 1;
+		}
 	}
 	if (!ld || !o->registerfiles)
 		return generic_open_file(td, f);
diff --git a/engines/nvme.c b/engines/nvme.c
index 08503b33..75a5e0c1 100644
--- a/engines/nvme.c
+++ b/engines/nvme.c
@@ -415,19 +415,24 @@ void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 	case NVME_NS_DPS_PI_TYPE2:
 		switch (data->guard_type) {
 		case NVME_NVM_NS_16B_GUARD:
-			cmd->cdw14 = (__u32)slba;
+			if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF)
+				cmd->cdw14 = (__u32)slba;
 			break;
 		case NVME_NVM_NS_64B_GUARD:
-			cmd->cdw14 = (__u32)slba;
-			cmd->cdw3 = ((slba >> 32) & 0xffff);
+			if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+				cmd->cdw14 = (__u32)slba;
+				cmd->cdw3 = ((slba >> 32) & 0xffff);
+			}
 			break;
 		default:
 			break;
 		}
-		cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
+			cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
 		break;
 	case NVME_NS_DPS_PI_TYPE3:
-		cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
+			cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
 		break;
 	case NVME_NS_DPS_PI_NONE:
 		break;
diff --git a/fio.1 b/fio.1
index f0dc49ab..8159caa4 100644
--- a/fio.1
+++ b/fio.1
@@ -537,6 +537,10 @@ copy that segment, instead of entering the kernel with a
 \fBgettimeofday\fR\|(2) call. The CPU set aside for doing these time
 calls will be excluded from other uses. Fio will manually clear it from the
 CPU mask of other jobs.
+.TP
+.BI job_start_clock_id \fR=\fPint
+The clock_id passed to the call to \fBclock_gettime\fR used to record job_start
+in the \fBjson\fR output format. Default is 0, or CLOCK_REALTIME.
 .SS "Target file/device"
 .TP
 .BI directory \fR=\fPstr
@@ -3664,6 +3668,15 @@ quickly becomes unwieldy. To see the final report per-group instead of
 per-job, use \fBgroup_reporting\fR. Jobs in a file will be part of the
 same reporting group, unless if separated by a \fBstonewall\fR, or by
 using \fBnew_group\fR.
+.RS
+.P
+NOTE: When \fBgroup_reporting\fR is used along with \fBjson\fR output, there
+are certain per-job properties which can be different between jobs but do not
+have a natural group-level equivalent. Examples include \fBkb_base\fR,
+\fBunit_base\fR, \fBsig_figs\fR, \fBthread_number\fR, \fBpid\fR, and
+\fBjob_start\fR. For these properties, the values for the first job are
+recorded for the group.
+.RE
 .TP
 .BI new_group
 Start a new reporting group. See: \fBgroup_reporting\fR. If not given,
@@ -3795,9 +3808,7 @@ decompressed with fio, using the \fB\-\-inflate\-log\fR command line
 parameter. The files will be stored with a `.fz' suffix.
 .TP
 .BI log_unix_epoch \fR=\fPbool
-If set, fio will log Unix timestamps to the log files produced by enabling
-write_type_log for each log type, instead of the default zero-based
-timestamps.
+Backward-compatible alias for \fBlog_alternate_epoch\fR.
 .TP
 .BI log_alternate_epoch \fR=\fPbool
 If set, fio will log timestamps based on the epoch used by the clock specified
@@ -3806,9 +3817,9 @@ enabling write_type_log for each log type, instead of the default zero-based
 timestamps.
 .TP
 .BI log_alternate_epoch_clock_id \fR=\fPint
-Specifies the clock_id to be used by clock_gettime to obtain the alternate epoch
-if either \fBBlog_unix_epoch\fR or \fBlog_alternate_epoch\fR are true. Otherwise has no
-effect. Default value is 0, or CLOCK_REALTIME.
+Specifies the clock_id to be used by clock_gettime to obtain the alternate
+epoch if \fBlog_alternate_epoch\fR is true. Otherwise has no effect. Default
+value is 0, or CLOCK_REALTIME.
 .TP
 .BI block_error_percentiles \fR=\fPbool
 If set, record errors in trim block-sized units from writes and trims and
diff --git a/fio.h b/fio.h
index a54f57c9..1322656f 100644
--- a/fio.h
+++ b/fio.h
@@ -388,7 +388,8 @@ struct thread_data {
 
 	struct timespec start;	/* start of this loop */
 	struct timespec epoch;	/* time job was started */
-	unsigned long long alternate_epoch; /* Time job was started, clock_gettime's clock_id epoch based. */
+	unsigned long long alternate_epoch; /* Time job was started, as clock_gettime(log_alternate_epoch_clock_id) */
+	unsigned long long job_start; /* Time job was started, as clock_gettime(job_start_clock_id) */
 	struct timespec last_issue;
 	long time_offset;
 	struct timespec ts_cache;
diff --git a/fio_time.h b/fio_time.h
index 62d92120..b20e734c 100644
--- a/fio_time.h
+++ b/fio_time.h
@@ -30,6 +30,6 @@ extern bool ramp_time_over(struct thread_data *);
 extern bool in_ramp_time(struct thread_data *);
 extern void fio_time_init(void);
 extern void timespec_add_msec(struct timespec *, unsigned int);
-extern void set_epoch_time(struct thread_data *, int, clockid_t);
+extern void set_epoch_time(struct thread_data *, clockid_t, clockid_t);
 
 #endif
diff --git a/libfio.c b/libfio.c
index 237ce34c..5c433277 100644
--- a/libfio.c
+++ b/libfio.c
@@ -149,7 +149,7 @@ void reset_all_stats(struct thread_data *td)
 		td->ts.runtime[i] = 0;
 	}
 
-	set_epoch_time(td, td->o.log_unix_epoch | td->o.log_alternate_epoch, td->o.log_alternate_epoch_clock_id);
+	set_epoch_time(td, td->o.log_alternate_epoch_clock_id, td->o.job_start_clock_id);
 	memcpy(&td->start, &td->epoch, sizeof(td->epoch));
 	memcpy(&td->iops_sample_time, &td->epoch, sizeof(td->epoch));
 	memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch));
diff --git a/options.c b/options.c
index 65b2813c..6b2cb53f 100644
--- a/options.c
+++ b/options.c
@@ -4612,17 +4612,9 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.help	= "Install libz-dev(el) to get compression support",
 	},
 #endif
-	{
-		.name = "log_unix_epoch",
-		.lname = "Log epoch unix",
-		.type = FIO_OPT_BOOL,
-		.off1 = offsetof(struct thread_options, log_unix_epoch),
-		.help = "Use Unix time in log files",
-		.category = FIO_OPT_C_LOG,
-		.group = FIO_OPT_G_INVALID,
-	},
 	{
 		.name = "log_alternate_epoch",
+		.alias = "log_unix_epoch",
 		.lname = "Log epoch alternate",
 		.type = FIO_OPT_BOOL,
 		.off1 = offsetof(struct thread_options, log_alternate_epoch),
@@ -4635,7 +4627,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.lname = "Log alternate epoch clock_id",
 		.type = FIO_OPT_INT,
 		.off1 = offsetof(struct thread_options, log_alternate_epoch_clock_id),
-		.help = "If log_alternate_epoch or log_unix_epoch is true, this option specifies the clock_id from clock_gettime whose epoch should be used. If neither of those is true, this option has no effect. Default value is 0, or CLOCK_REALTIME",
+		.help = "If log_alternate_epoch is true, this option specifies the clock_id from clock_gettime whose epoch should be used. If log_alternate_epoch is false, this option has no effect. Default value is 0, or CLOCK_REALTIME",
 		.category = FIO_OPT_C_LOG,
 		.group = FIO_OPT_G_INVALID,
 	},
@@ -4964,6 +4956,16 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.category = FIO_OPT_C_GENERAL,
 		.group	= FIO_OPT_G_CLOCK,
 	},
+	{
+		.name	= "job_start_clock_id",
+		.lname	= "Job start clock_id",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct thread_options, job_start_clock_id),
+		.help	= "The clock_id passed to the call to clock_gettime used to record job_start in the json output format. Default is 0, or CLOCK_REALTIME",
+		.verify	= gtod_cpu_verify,
+		.category = FIO_OPT_C_GENERAL,
+		.group	= FIO_OPT_G_CLOCK,
+	},
 	{
 		.name	= "unified_rw_reporting",
 		.lname	= "Unified RW Reporting",
diff --git a/rate-submit.c b/rate-submit.c
index 6f6d15bd..92be3df7 100644
--- a/rate-submit.c
+++ b/rate-submit.c
@@ -185,7 +185,7 @@ static int io_workqueue_init_worker_fn(struct submit_worker *sw)
 	if (td->io_ops->post_init && td->io_ops->post_init(td))
 		goto err_io_init;
 
-	set_epoch_time(td, td->o.log_unix_epoch | td->o.log_alternate_epoch, td->o.log_alternate_epoch_clock_id);
+	set_epoch_time(td, td->o.log_alternate_epoch_clock_id, td->o.job_start_clock_id);
 	fio_getrusage(&td->ru_start);
 	clear_io_state(td, 1);
 
diff --git a/server.c b/server.c
index bb423702..27332e32 100644
--- a/server.c
+++ b/server.c
@@ -1706,6 +1706,7 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
 	p.ts.error		= cpu_to_le32(ts->error);
 	p.ts.thread_number	= cpu_to_le32(ts->thread_number);
 	p.ts.groupid		= cpu_to_le32(ts->groupid);
+	p.ts.job_start		= cpu_to_le64(ts->job_start);
 	p.ts.pid		= cpu_to_le32(ts->pid);
 	p.ts.members		= cpu_to_le32(ts->members);
 	p.ts.unified_rw_rep	= cpu_to_le32(ts->unified_rw_rep);
diff --git a/stat.c b/stat.c
index 7b791628..7cf6bee1 100644
--- a/stat.c
+++ b/stat.c
@@ -1712,6 +1712,7 @@ static struct json_object *show_thread_status_json(struct thread_stat *ts,
 	root = json_create_object();
 	json_object_add_value_string(root, "jobname", ts->name);
 	json_object_add_value_int(root, "groupid", ts->groupid);
+	json_object_add_value_int(root, "job_start", ts->job_start);
 	json_object_add_value_int(root, "error", ts->error);
 
 	/* ETA Info */
@@ -2526,6 +2527,7 @@ void __show_run_stats(void)
 			 */
 			ts->thread_number = td->thread_number;
 			ts->groupid = td->groupid;
+			ts->job_start = td->job_start;
 
 			/*
 			 * first pid in group, not very useful...
@@ -3048,7 +3050,9 @@ static void __add_log_sample(struct io_log *iolog, union io_sample_data data,
 		s = get_sample(iolog, cur_log, cur_log->nr_samples);
 
 		s->data = data;
-		s->time = t + (iolog->td ? iolog->td->alternate_epoch : 0);
+		s->time = t;
+		if (iolog->td && iolog->td->o.log_alternate_epoch)
+			s->time += iolog->td->alternate_epoch;
 		io_sample_set_ddir(iolog, s, ddir);
 		s->bs = bs;
 		s->priority = priority;
diff --git a/stat.h b/stat.h
index 8ceabc48..bd986d4e 100644
--- a/stat.h
+++ b/stat.h
@@ -169,6 +169,7 @@ struct thread_stat {
 	uint32_t error;
 	uint32_t thread_number;
 	uint32_t groupid;
+	uint64_t job_start; /* Time job was started, as clock_gettime(job_start_clock_id) */
 	uint32_t pid;
 	char description[FIO_JOBDESC_SIZE];
 	uint32_t members;
diff --git a/thread_options.h b/thread_options.h
index 38a9993d..fdde055e 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -170,7 +170,6 @@ struct thread_options {
 	unsigned int log_offset;
 	unsigned int log_gz;
 	unsigned int log_gz_store;
-	unsigned int log_unix_epoch;
 	unsigned int log_alternate_epoch;
 	unsigned int log_alternate_epoch_clock_id;
 	unsigned int norandommap;
@@ -273,6 +272,7 @@ struct thread_options {
 	unsigned int unified_rw_rep;
 	unsigned int gtod_reduce;
 	unsigned int gtod_cpu;
+	unsigned int job_start_clock_id;
 	enum fio_cs clocksource;
 	unsigned int no_stall;
 	unsigned int trim_percentage;
@@ -422,7 +422,6 @@ struct thread_options_pack {
 	uint32_t iodepth_batch_complete_min;
 	uint32_t iodepth_batch_complete_max;
 	uint32_t serialize_overlap;
-	uint32_t pad;
 
 	uint64_t size;
 	uint64_t io_size;
@@ -433,13 +432,11 @@ struct thread_options_pack {
 	uint32_t fill_device;
 	uint32_t file_append;
 	uint32_t unique_filename;
-	uint32_t pad3;
 	uint64_t file_size_low;
 	uint64_t file_size_high;
 	uint64_t start_offset;
 	uint64_t start_offset_align;
 	uint32_t start_offset_nz;
-	uint32_t pad4;
 
 	uint64_t bs[DDIR_RWDIR_CNT];
 	uint64_t ba[DDIR_RWDIR_CNT];
@@ -494,7 +491,6 @@ struct thread_options_pack {
 	uint32_t log_offset;
 	uint32_t log_gz;
 	uint32_t log_gz_store;
-	uint32_t log_unix_epoch;
 	uint32_t log_alternate_epoch;
 	uint32_t log_alternate_epoch_clock_id;
 	uint32_t norandommap;
@@ -593,6 +589,7 @@ struct thread_options_pack {
 	uint32_t unified_rw_rep;
 	uint32_t gtod_reduce;
 	uint32_t gtod_cpu;
+	uint32_t job_start_clock_id;
 	uint32_t clocksource;
 	uint32_t no_stall;
 	uint32_t trim_percentage;
@@ -603,6 +600,7 @@ struct thread_options_pack {
 	uint32_t lat_percentiles;
 	uint32_t slat_percentiles;
 	uint32_t percentile_precision;
+	uint32_t pad;
 	fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
 
 	uint8_t read_iolog_file[FIO_TOP_STR_MAX];
diff --git a/time.c b/time.c
index 5c4d6de0..7cbab6ff 100644
--- a/time.c
+++ b/time.c
@@ -172,14 +172,22 @@ void set_genesis_time(void)
 	fio_gettime(&genesis, NULL);
 }
 
-void set_epoch_time(struct thread_data *td, int log_alternate_epoch, clockid_t clock_id)
+void set_epoch_time(struct thread_data *td, clockid_t log_alternate_epoch_clock_id, clockid_t job_start_clock_id)
 {
+	struct timespec ts;
 	fio_gettime(&td->epoch, NULL);
-	if (log_alternate_epoch) {
-		struct timespec ts;
-		clock_gettime(clock_id, &ts);
-		td->alternate_epoch = (unsigned long long)(ts.tv_sec) * 1000 +
-		                 (unsigned long long)(ts.tv_nsec) / 1000000;
+	clock_gettime(log_alternate_epoch_clock_id, &ts);
+	td->alternate_epoch = (unsigned long long)(ts.tv_sec) * 1000 +
+						  (unsigned long long)(ts.tv_nsec) / 1000000;
+	if (job_start_clock_id == log_alternate_epoch_clock_id)
+	{
+		td->job_start = td->alternate_epoch;
+	}
+	else
+	{
+		clock_gettime(job_start_clock_id, &ts);
+		td->job_start = (unsigned long long)(ts.tv_sec) * 1000 +
+						(unsigned long long)(ts.tv_nsec) / 1000000;
 	}
 }
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-09-03 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-09-03 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 4a0c766c69ddfe5231d65f2676e97333ba89ab2b:

  Merge branch 'master' of https://github.com/michalbiesek/fio (2023-08-23 08:21:39 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 904ee91c2831615a054a8dea9b164e96ae00abb3:

  Merge branch 'pcpp_parse_nr_fix' of https://github.com/PCPartPicker/fio (2023-09-02 07:35:49 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'pcpp_parse_nr_fix' of https://github.com/PCPartPicker/fio

aggieNick02 (1):
      Add basic error checking to parsing nr from rw=randrw:<nr>, etc

 options.c | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

---

Diff of recent changes:

diff --git a/options.c b/options.c
index 48aa0d7b..65b2813c 100644
--- a/options.c
+++ b/options.c
@@ -596,9 +596,21 @@ static int str_rw_cb(void *data, const char *str)
 	if (!nr)
 		return 0;
 
-	if (td_random(td))
-		o->ddir_seq_nr = atoi(nr);
-	else {
+	if (td_random(td)) {
+		long long val;
+
+		if (str_to_decimal(nr, &val, 1, o, 0, 0)) {
+			log_err("fio: randrw postfix parsing failed\n");
+			free(nr);
+			return 1;
+		}
+		if ((val <= 0) || (val > UINT_MAX)) {
+			log_err("fio: randrw postfix parsing out of range\n");
+			free(nr);
+			return 1;
+		}
+		o->ddir_seq_nr = (unsigned int) val;
+	} else {
 		long long val;
 
 		if (str_to_decimal(nr, &val, 1, o, 0, 0)) {

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-08-24 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-08-24 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit b311162c37a2867873e1222ce6b5f38c88be4d80:

  examples: add example and fiograph for protection information options (2023-08-16 09:34:46 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 4a0c766c69ddfe5231d65f2676e97333ba89ab2b:

  Merge branch 'master' of https://github.com/michalbiesek/fio (2023-08-23 08:21:39 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'master' of https://github.com/michalbiesek/fio

Michal Biesek (1):
      Add RISC-V 64 support

 arch/arch-riscv64.h   | 32 ++++++++++++++++++++++++++++++++
 arch/arch.h           |  3 +++
 configure             | 24 +++++++++++++++++++++++-
 libfio.c              |  1 +
 os/os-linux-syscall.h |  7 +++++++
 5 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 arch/arch-riscv64.h

---

Diff of recent changes:

diff --git a/arch/arch-riscv64.h b/arch/arch-riscv64.h
new file mode 100644
index 00000000..a74b7d47
--- /dev/null
+++ b/arch/arch-riscv64.h
@@ -0,0 +1,32 @@
+#ifndef ARCH_RISCV64_H
+#define ARCH_RISCV64_H
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#define FIO_ARCH	(arch_riscv64)
+
+#define nop		__asm__ __volatile__ ("nop")
+#define read_barrier()		__asm__ __volatile__("fence r, r": : :"memory")
+#define write_barrier()		__asm__ __volatile__("fence w, w": : :"memory")
+
+static inline unsigned long long get_cpu_clock(void)
+{
+	unsigned long val;
+
+	asm volatile("rdcycle %0" : "=r"(val));
+	return val;
+}
+#define ARCH_HAVE_CPU_CLOCK
+
+#define ARCH_HAVE_INIT
+extern bool tsc_reliable;
+static inline int arch_init(char *envp[])
+{
+	tsc_reliable = true;
+	return 0;
+}
+
+#endif
diff --git a/arch/arch.h b/arch/arch.h
index 6e476701..3ee9b053 100644
--- a/arch/arch.h
+++ b/arch/arch.h
@@ -24,6 +24,7 @@ enum {
 	arch_mips,
 	arch_aarch64,
 	arch_loongarch64,
+	arch_riscv64,
 
 	arch_generic,
 
@@ -100,6 +101,8 @@ extern unsigned long arch_flags;
 #include "arch-aarch64.h"
 #elif defined(__loongarch64)
 #include "arch-loongarch64.h"
+#elif defined(__riscv) && __riscv_xlen == 64
+#include "arch-riscv64.h"
 #else
 #warning "Unknown architecture, attempting to use generic model."
 #include "arch-generic.h"
diff --git a/configure b/configure
index 6c938251..36184a58 100755
--- a/configure
+++ b/configure
@@ -133,6 +133,20 @@ EOF
   compile_object
 }
 
+check_val() {
+    cat > $TMPC <<EOF
+#if $1 == $2
+int main(void)
+{
+  return 0;
+}
+#else
+#error $1 is not equal $2
+#endif
+EOF
+  compile_object
+}
+
 output_sym() {
   echo "$1=y" >> $config_host_mak
   echo "#define $1" >> $config_host_h
@@ -501,13 +515,21 @@ elif check_define __hppa__ ; then
   cpu="hppa"
 elif check_define __loongarch64 ; then
   cpu="loongarch64"
+elif check_define __riscv ; then
+  if check_val __riscv_xlen 32 ; then
+    cpu="riscv32"
+  elif check_val __riscv_xlen 64 ; then
+    cpu="riscv64"
+  elif check_val __riscv_xlen 128 ; then
+    cpu="riscv128"
+  fi
 else
   cpu=`uname -m`
 fi
 
 # Normalise host CPU name and set ARCH.
 case "$cpu" in
-  ia64|ppc|ppc64|s390|s390x|sparc64|loongarch64)
+  ia64|ppc|ppc64|s390|s390x|sparc64|loongarch64|riscv64)
     cpu="$cpu"
   ;;
   i386|i486|i586|i686|i86pc|BePC)
diff --git a/libfio.c b/libfio.c
index 5e3fd30b..237ce34c 100644
--- a/libfio.c
+++ b/libfio.c
@@ -75,6 +75,7 @@ static const char *fio_arch_strings[arch_nr] = {
 	"mips",
 	"aarch64",
 	"loongarch64",
+	"riscv64",
 	"generic"
 };
 
diff --git a/os/os-linux-syscall.h b/os/os-linux-syscall.h
index 67ee4d91..626330ad 100644
--- a/os/os-linux-syscall.h
+++ b/os/os-linux-syscall.h
@@ -286,6 +286,13 @@
 #define __NR_sys_tee          	77
 #define __NR_sys_vmsplice       75
 #endif
+
+/* Linux syscalls for riscv64 */
+#elif defined(ARCH_RISCV64_H)
+#ifndef __NR_ioprio_set
+#define __NR_ioprio_set		30
+#define __NR_ioprio_get		31
+#endif
 #else
 #warning "Unknown architecture"
 #endif

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-08-17 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-08-17 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 6795954bde09c8697e0accb865b4f438d62c601f:

  engines/io_uring: fix leak of 'ld' in error path (2023-08-14 19:59:20 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to b311162c37a2867873e1222ce6b5f38c88be4d80:

  examples: add example and fiograph for protection information options (2023-08-16 09:34:46 -0400)

----------------------------------------------------------------
Ankit Kumar (1):
      examples: add example and fiograph for protection information options

 examples/uring-cmd-pi-ext.fio |  31 +++++++++++++++++++++++++++++++
 examples/uring-cmd-pi-ext.png | Bin 0 -> 81014 bytes
 examples/uring-cmd-pi-sb.fio  |  32 ++++++++++++++++++++++++++++++++
 examples/uring-cmd-pi-sb.png  | Bin 0 -> 87357 bytes
 tools/fiograph/fiograph.conf  |   2 +-
 5 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 examples/uring-cmd-pi-ext.fio
 create mode 100644 examples/uring-cmd-pi-ext.png
 create mode 100644 examples/uring-cmd-pi-sb.fio
 create mode 100644 examples/uring-cmd-pi-sb.png

---

Diff of recent changes:

diff --git a/examples/uring-cmd-pi-ext.fio b/examples/uring-cmd-pi-ext.fio
new file mode 100644
index 00000000..e22ec062
--- /dev/null
+++ b/examples/uring-cmd-pi-ext.fio
@@ -0,0 +1,31 @@
+# Protection information test with io_uring_cmd I/O engine for nvme-ns generic
+# character device.
+#
+# This requires nvme device to be formatted with extended LBA data size and
+# protection information enabled. This can be done with nvme-cli utility.
+# Replace bs below with the correct extended LBA size.
+#
+# First we sequentially write to the device, without protection information
+# action being set. FIO will generate and send necessary protection
+# information data as per the protection information check option. Later on we
+# sequentially read and verify the device returned protection information data.
+#
+[global]
+filename=/dev/ng0n1
+ioengine=io_uring_cmd
+cmd_type=nvme
+size=1G
+iodepth=32
+bs=4160
+pi_act=0
+pi_chk=GUARD,APPTAG,REFTAG
+apptag=0x0888
+apptag_mask=0xFFFF
+thread=1
+stonewall=1
+
+[write]
+rw=write
+
+[read]
+rw=read
diff --git a/examples/uring-cmd-pi-ext.png b/examples/uring-cmd-pi-ext.png
new file mode 100644
index 00000000..a102fc1a
Binary files /dev/null and b/examples/uring-cmd-pi-ext.png differ
diff --git a/examples/uring-cmd-pi-sb.fio b/examples/uring-cmd-pi-sb.fio
new file mode 100644
index 00000000..b201a7ce
--- /dev/null
+++ b/examples/uring-cmd-pi-sb.fio
@@ -0,0 +1,32 @@
+# Protection information test with io_uring_cmd I/O engine for nvme-ns generic
+# character device.
+#
+# This requires nvme device to be formatted with separate metadata buffer and
+# protection information enabled. This can be done with nvme-cli utility.
+# Replace md_per_io_size as per the required metadata buffer size for each IO.
+#
+# First we sequentially write to the device, without protection information
+# action being set. FIO will generate and send necessary protection
+# information data as per the protection information check option. Later on we
+# sequentially read and verify the device returned protection information data.
+#
+[global]
+filename=/dev/ng0n1
+ioengine=io_uring_cmd
+cmd_type=nvme
+size=1G
+iodepth=32
+bs=4096
+md_per_io_size=64
+pi_act=0
+pi_chk=GUARD,APPTAG,REFTAG
+apptag=0x0888
+apptag_mask=0xFFFF
+thread=1
+stonewall=1
+
+[write]
+rw=write
+
+[read]
+rw=read
diff --git a/examples/uring-cmd-pi-sb.png b/examples/uring-cmd-pi-sb.png
new file mode 100644
index 00000000..dcdda8cd
Binary files /dev/null and b/examples/uring-cmd-pi-sb.png differ
diff --git a/tools/fiograph/fiograph.conf b/tools/fiograph/fiograph.conf
index 91c5fcfe..123c39ae 100644
--- a/tools/fiograph/fiograph.conf
+++ b/tools/fiograph/fiograph.conf
@@ -54,7 +54,7 @@ specific_options=ime_psync  ime_psyncv
 specific_options=hipri  cmdprio_percentage  cmdprio_class  cmdprio  cmdprio_bssplit  fixedbufs  registerfiles  sqthread_poll  sqthread_poll_cpu  nonvectored  uncached  nowait  force_async
 
 [ioengine_io_uring_cmd]
-specific_options=hipri  cmdprio_percentage  cmdprio_class  cmdprio  cmdprio_bssplit  fixedbufs  registerfiles  sqthread_poll  sqthread_poll_cpu  nonvectored  uncached  nowait  force_async  cmd_type
+specific_options=hipri  cmdprio_percentage  cmdprio_class  cmdprio  cmdprio_bssplit  fixedbufs  registerfiles  sqthread_poll  sqthread_poll_cpu  nonvectored  uncached  nowait  force_async  cmd_type  md_per_io_size  pi_act  pi_chk  apptag  apptag_mask
 
 [ioengine_libaio]
 specific_options=userspace_reap  cmdprio_percentage  cmdprio_class  cmdprio  cmdprio_bssplit  nowait

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-08-15 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-08-15 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 62f35562722f0c903567096d0f10a836d1ae2f60:

  eta: calculate aggregate bw statistics even when eta is disabled (2023-08-03 11:49:08 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 6795954bde09c8697e0accb865b4f438d62c601f:

  engines/io_uring: fix leak of 'ld' in error path (2023-08-14 19:59:20 -0600)

----------------------------------------------------------------
Ankit Kumar (10):
      engines:io_uring: add missing error during open file
      engines:io_uring: update arguments to fetch nvme data
      engines:io_uring: enable support for separate metadata buffer
      engines:io_uring: uring_cmd add support for protection info
      io_u: move engine data out of union
      crc: pull required crc16-t10 files from linux kernel
      engines:io_uring: generate and verify pi for 16b guard
      crc: pull required crc64 nvme apis from linux kernel
      engines:nvme: pull required 48 bit accessors from linux kernel
      engines:io_uring: generate and verify pi for 64b guard

Jens Axboe (1):
      engines/io_uring: fix leak of 'ld' in error path

Vincent Fu (2):
      t/fiotestlib: use config variable to skip test at runtime
      t/nvmept_pi: test script for protection information

 HOWTO.rst              |  39 ++
 crc/crc-t10dif.h       |   9 +
 crc/crc64.c            |  32 ++
 crc/crc64.h            |   3 +
 crc/crc64table.h       | 130 +++++++
 crc/crct10dif_common.c |  78 ++++
 engines/io_uring.c     | 228 ++++++++++--
 engines/nvme.c         | 466 ++++++++++++++++++++++--
 engines/nvme.h         | 230 +++++++++++-
 fio.1                  |  38 ++
 io_u.h                 |   2 +-
 t/fiotestlib.py        |   5 +-
 t/nvmept_pi.py         | 949 +++++++++++++++++++++++++++++++++++++++++++++++++
 13 files changed, 2154 insertions(+), 55 deletions(-)
 create mode 100644 crc/crc-t10dif.h
 create mode 100644 crc/crc64table.h
 create mode 100644 crc/crct10dif_common.c
 create mode 100755 t/nvmept_pi.py

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index ac8314f3..89032941 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2487,6 +2487,45 @@ with the caveat that when used on the command line, they must come after the
         want fio to use placement identifier only at indices 0, 2 and 5 specify
         ``fdp_pli=0,2,5``.
 
+.. option:: md_per_io_size=int : [io_uring_cmd]
+
+	Size in bytes for separate metadata buffer per IO. Default: 0.
+
+.. option:: pi_act=int : [io_uring_cmd]
+
+	Action to take when nvme namespace is formatted with protection
+	information. If this is set to 1 and namespace is formatted with
+	metadata size equal to protection information size, fio won't use
+	separate metadata buffer or extended logical block. If this is set to
+	1 and namespace is formatted with metadata size greater than protection
+	information size, fio will not generate or verify the protection
+	information portion of metadata for write or read case respectively.
+	If this is set to 0, fio generates protection information for
+	write case and verifies for read case. Default: 1.
+
+.. option:: pi_chk=str[,str][,str] : [io_uring_cmd]
+
+	Controls the protection information check. This can take one or more
+	of these values. Default: none.
+
+	**GUARD**
+		Enables protection information checking of guard field.
+	**REFTAG**
+		Enables protection information checking of logical block
+		reference tag field.
+	**APPTAG**
+		Enables protection information checking of application tag field.
+
+.. option:: apptag=int : [io_uring_cmd]
+
+	Specifies logical block application tag value, if namespace is
+	formatted to use end to end protection information. Default: 0x1234.
+
+.. option:: apptag_mask=int : [io_uring_cmd]
+
+	Specifies logical block application tag mask value, if namespace is
+	formatted to use end to end protection information. Default: 0xffff.
+
 .. option:: cpuload=int : [cpuio]
 
 	Attempt to use the specified percentage of CPU cycles. This is a mandatory
diff --git a/crc/crc-t10dif.h b/crc/crc-t10dif.h
new file mode 100644
index 00000000..fde4ccd7
--- /dev/null
+++ b/crc/crc-t10dif.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __CRC_T10DIF_H
+#define __CRC_T10DIF_H
+
+extern unsigned short fio_crc_t10dif(unsigned short crc,
+				     const unsigned char *buffer,
+				     unsigned int len);
+
+#endif
diff --git a/crc/crc64.c b/crc/crc64.c
index bf24a97b..c910e5b8 100644
--- a/crc/crc64.c
+++ b/crc/crc64.c
@@ -1,4 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * crc64nvme[256] table is from the generator polynomial specified by NVMe
+ * 64b CRC and is defined as,
+ *
+ * x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + x^47 +
+ * x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + x^26 + x^23 +
+ * x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + x^4 + x^3 + 1
+ *
+ */
+
 #include "crc64.h"
+#include "crc64table.h"
 
 /*
  * poly 0x95AC9329AC4BC9B5ULL and init 0xFFFFFFFFFFFFFFFFULL
@@ -102,3 +114,23 @@ unsigned long long fio_crc64(const unsigned char *buffer, unsigned long length)
 	return crc;
 }
 
+/**
+ * fio_crc64_nvme - Calculate bitwise NVMe CRC64
+ * @crc: seed value for computation. 0 for a new CRC calculation, or the
+ * 	 previous crc64 value if computing incrementally.
+ * @p: pointer to buffer over which CRC64 is run
+ * @len: length of buffer @p
+ */
+unsigned long long fio_crc64_nvme(unsigned long long crc, const void *p,
+				  unsigned int len)
+{
+	const unsigned char *_p = p;
+	unsigned int i;
+
+	crc = ~crc;
+
+	for (i = 0; i < len; i++)
+		crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *_p++];
+
+	return ~crc;
+}
diff --git a/crc/crc64.h b/crc/crc64.h
index fe9cad3e..e586edee 100644
--- a/crc/crc64.h
+++ b/crc/crc64.h
@@ -3,4 +3,7 @@
 
 unsigned long long fio_crc64(const unsigned char *, unsigned long);
 
+unsigned long long fio_crc64_nvme(unsigned long long crc, const void *p,
+				  unsigned int len);
+
 #endif
diff --git a/crc/crc64table.h b/crc/crc64table.h
new file mode 100644
index 00000000..04224d4f
--- /dev/null
+++ b/crc/crc64table.h
@@ -0,0 +1,130 @@
+static const unsigned long long crc64nvmetable[256] = {
+	0x0000000000000000ULL, 	0x7f6ef0c830358979ULL,
+	0xfedde190606b12f2ULL, 	0x81b31158505e9b8bULL,
+	0xc962e5739841b68fULL, 	0xb60c15bba8743ff6ULL,
+	0x37bf04e3f82aa47dULL, 	0x48d1f42bc81f2d04ULL,
+	0xa61cecb46814fe75ULL, 	0xd9721c7c5821770cULL,
+	0x58c10d24087fec87ULL, 	0x27affdec384a65feULL,
+	0x6f7e09c7f05548faULL, 	0x1010f90fc060c183ULL,
+	0x91a3e857903e5a08ULL, 	0xeecd189fa00bd371ULL,
+	0x78e0ff3b88be6f81ULL, 	0x078e0ff3b88be6f8ULL,
+	0x863d1eabe8d57d73ULL, 	0xf953ee63d8e0f40aULL,
+	0xb1821a4810ffd90eULL, 	0xceecea8020ca5077ULL,
+	0x4f5ffbd87094cbfcULL, 	0x30310b1040a14285ULL,
+	0xdefc138fe0aa91f4ULL, 	0xa192e347d09f188dULL,
+	0x2021f21f80c18306ULL, 	0x5f4f02d7b0f40a7fULL,
+	0x179ef6fc78eb277bULL, 	0x68f0063448deae02ULL,
+	0xe943176c18803589ULL, 	0x962de7a428b5bcf0ULL,
+	0xf1c1fe77117cdf02ULL, 	0x8eaf0ebf2149567bULL,
+	0x0f1c1fe77117cdf0ULL, 	0x7072ef2f41224489ULL,
+	0x38a31b04893d698dULL, 	0x47cdebccb908e0f4ULL,
+	0xc67efa94e9567b7fULL, 	0xb9100a5cd963f206ULL,
+	0x57dd12c379682177ULL, 	0x28b3e20b495da80eULL,
+	0xa900f35319033385ULL, 	0xd66e039b2936bafcULL,
+	0x9ebff7b0e12997f8ULL, 	0xe1d10778d11c1e81ULL,
+	0x606216208142850aULL, 	0x1f0ce6e8b1770c73ULL,
+	0x8921014c99c2b083ULL, 	0xf64ff184a9f739faULL,
+	0x77fce0dcf9a9a271ULL, 	0x08921014c99c2b08ULL,
+	0x4043e43f0183060cULL, 	0x3f2d14f731b68f75ULL,
+	0xbe9e05af61e814feULL, 	0xc1f0f56751dd9d87ULL,
+	0x2f3dedf8f1d64ef6ULL, 	0x50531d30c1e3c78fULL,
+	0xd1e00c6891bd5c04ULL, 	0xae8efca0a188d57dULL,
+	0xe65f088b6997f879ULL, 	0x9931f84359a27100ULL,
+	0x1882e91b09fcea8bULL, 	0x67ec19d339c963f2ULL,
+	0xd75adabd7a6e2d6fULL, 	0xa8342a754a5ba416ULL,
+	0x29873b2d1a053f9dULL, 	0x56e9cbe52a30b6e4ULL,
+	0x1e383fcee22f9be0ULL, 	0x6156cf06d21a1299ULL,
+	0xe0e5de5e82448912ULL, 	0x9f8b2e96b271006bULL,
+	0x71463609127ad31aULL, 	0x0e28c6c1224f5a63ULL,
+	0x8f9bd7997211c1e8ULL, 	0xf0f5275142244891ULL,
+	0xb824d37a8a3b6595ULL, 	0xc74a23b2ba0eececULL,
+	0x46f932eaea507767ULL, 	0x3997c222da65fe1eULL,
+	0xafba2586f2d042eeULL, 	0xd0d4d54ec2e5cb97ULL,
+	0x5167c41692bb501cULL, 	0x2e0934dea28ed965ULL,
+	0x66d8c0f56a91f461ULL, 	0x19b6303d5aa47d18ULL,
+	0x980521650afae693ULL, 	0xe76bd1ad3acf6feaULL,
+	0x09a6c9329ac4bc9bULL, 	0x76c839faaaf135e2ULL,
+	0xf77b28a2faafae69ULL, 	0x8815d86aca9a2710ULL,
+	0xc0c42c4102850a14ULL, 	0xbfaadc8932b0836dULL,
+	0x3e19cdd162ee18e6ULL, 	0x41773d1952db919fULL,
+	0x269b24ca6b12f26dULL, 	0x59f5d4025b277b14ULL,
+	0xd846c55a0b79e09fULL, 	0xa72835923b4c69e6ULL,
+	0xeff9c1b9f35344e2ULL, 	0x90973171c366cd9bULL,
+	0x1124202993385610ULL, 	0x6e4ad0e1a30ddf69ULL,
+	0x8087c87e03060c18ULL, 	0xffe938b633338561ULL,
+	0x7e5a29ee636d1eeaULL, 	0x0134d92653589793ULL,
+	0x49e52d0d9b47ba97ULL, 	0x368bddc5ab7233eeULL,
+	0xb738cc9dfb2ca865ULL, 	0xc8563c55cb19211cULL,
+	0x5e7bdbf1e3ac9decULL, 	0x21152b39d3991495ULL,
+	0xa0a63a6183c78f1eULL, 	0xdfc8caa9b3f20667ULL,
+	0x97193e827bed2b63ULL, 	0xe877ce4a4bd8a21aULL,
+	0x69c4df121b863991ULL, 	0x16aa2fda2bb3b0e8ULL,
+	0xf86737458bb86399ULL, 	0x8709c78dbb8deae0ULL,
+	0x06bad6d5ebd3716bULL, 	0x79d4261ddbe6f812ULL,
+	0x3105d23613f9d516ULL, 	0x4e6b22fe23cc5c6fULL,
+	0xcfd833a67392c7e4ULL, 	0xb0b6c36e43a74e9dULL,
+	0x9a6c9329ac4bc9b5ULL, 	0xe50263e19c7e40ccULL,
+	0x64b172b9cc20db47ULL, 	0x1bdf8271fc15523eULL,
+	0x530e765a340a7f3aULL, 	0x2c608692043ff643ULL,
+	0xadd397ca54616dc8ULL, 	0xd2bd67026454e4b1ULL,
+	0x3c707f9dc45f37c0ULL, 	0x431e8f55f46abeb9ULL,
+	0xc2ad9e0da4342532ULL, 	0xbdc36ec59401ac4bULL,
+	0xf5129aee5c1e814fULL, 	0x8a7c6a266c2b0836ULL,
+	0x0bcf7b7e3c7593bdULL, 	0x74a18bb60c401ac4ULL,
+	0xe28c6c1224f5a634ULL, 	0x9de29cda14c02f4dULL,
+	0x1c518d82449eb4c6ULL, 	0x633f7d4a74ab3dbfULL,
+	0x2bee8961bcb410bbULL, 	0x548079a98c8199c2ULL,
+	0xd53368f1dcdf0249ULL, 	0xaa5d9839ecea8b30ULL,
+	0x449080a64ce15841ULL, 	0x3bfe706e7cd4d138ULL,
+	0xba4d61362c8a4ab3ULL, 	0xc52391fe1cbfc3caULL,
+	0x8df265d5d4a0eeceULL, 	0xf29c951de49567b7ULL,
+	0x732f8445b4cbfc3cULL, 	0x0c41748d84fe7545ULL,
+	0x6bad6d5ebd3716b7ULL, 	0x14c39d968d029fceULL,
+	0x95708ccedd5c0445ULL, 	0xea1e7c06ed698d3cULL,
+	0xa2cf882d2576a038ULL, 	0xdda178e515432941ULL,
+	0x5c1269bd451db2caULL, 	0x237c997575283bb3ULL,
+	0xcdb181ead523e8c2ULL, 	0xb2df7122e51661bbULL,
+	0x336c607ab548fa30ULL, 	0x4c0290b2857d7349ULL,
+	0x04d364994d625e4dULL, 	0x7bbd94517d57d734ULL,
+	0xfa0e85092d094cbfULL, 	0x856075c11d3cc5c6ULL,
+	0x134d926535897936ULL, 	0x6c2362ad05bcf04fULL,
+	0xed9073f555e26bc4ULL, 	0x92fe833d65d7e2bdULL,
+	0xda2f7716adc8cfb9ULL, 	0xa54187de9dfd46c0ULL,
+	0x24f29686cda3dd4bULL, 	0x5b9c664efd965432ULL,
+	0xb5517ed15d9d8743ULL, 	0xca3f8e196da80e3aULL,
+	0x4b8c9f413df695b1ULL, 	0x34e26f890dc31cc8ULL,
+	0x7c339ba2c5dc31ccULL, 	0x035d6b6af5e9b8b5ULL,
+	0x82ee7a32a5b7233eULL, 	0xfd808afa9582aa47ULL,
+	0x4d364994d625e4daULL, 	0x3258b95ce6106da3ULL,
+	0xb3eba804b64ef628ULL, 	0xcc8558cc867b7f51ULL,
+	0x8454ace74e645255ULL, 	0xfb3a5c2f7e51db2cULL,
+	0x7a894d772e0f40a7ULL, 	0x05e7bdbf1e3ac9deULL,
+	0xeb2aa520be311aafULL, 	0x944455e88e0493d6ULL,
+	0x15f744b0de5a085dULL, 	0x6a99b478ee6f8124ULL,
+	0x224840532670ac20ULL, 	0x5d26b09b16452559ULL,
+	0xdc95a1c3461bbed2ULL, 	0xa3fb510b762e37abULL,
+	0x35d6b6af5e9b8b5bULL, 	0x4ab846676eae0222ULL,
+	0xcb0b573f3ef099a9ULL, 	0xb465a7f70ec510d0ULL,
+	0xfcb453dcc6da3dd4ULL, 	0x83daa314f6efb4adULL,
+	0x0269b24ca6b12f26ULL, 	0x7d0742849684a65fULL,
+	0x93ca5a1b368f752eULL, 	0xeca4aad306bafc57ULL,
+	0x6d17bb8b56e467dcULL, 	0x12794b4366d1eea5ULL,
+	0x5aa8bf68aecec3a1ULL, 	0x25c64fa09efb4ad8ULL,
+	0xa4755ef8cea5d153ULL, 	0xdb1bae30fe90582aULL,
+	0xbcf7b7e3c7593bd8ULL, 	0xc399472bf76cb2a1ULL,
+	0x422a5673a732292aULL, 	0x3d44a6bb9707a053ULL,
+	0x759552905f188d57ULL, 	0x0afba2586f2d042eULL,
+	0x8b48b3003f739fa5ULL, 	0xf42643c80f4616dcULL,
+	0x1aeb5b57af4dc5adULL, 	0x6585ab9f9f784cd4ULL,
+	0xe436bac7cf26d75fULL, 	0x9b584a0fff135e26ULL,
+	0xd389be24370c7322ULL, 	0xace74eec0739fa5bULL,
+	0x2d545fb4576761d0ULL, 	0x523aaf7c6752e8a9ULL,
+	0xc41748d84fe75459ULL, 	0xbb79b8107fd2dd20ULL,
+	0x3acaa9482f8c46abULL, 	0x45a459801fb9cfd2ULL,
+	0x0d75adabd7a6e2d6ULL, 	0x721b5d63e7936bafULL,
+	0xf3a84c3bb7cdf024ULL, 	0x8cc6bcf387f8795dULL,
+	0x620ba46c27f3aa2cULL, 	0x1d6554a417c62355ULL,
+	0x9cd645fc4798b8deULL, 	0xe3b8b53477ad31a7ULL,
+	0xab69411fbfb21ca3ULL, 	0xd407b1d78f8795daULL,
+	0x55b4a08fdfd90e51ULL, 	0x2ada5047efec8728ULL,
+};
diff --git a/crc/crct10dif_common.c b/crc/crct10dif_common.c
new file mode 100644
index 00000000..cfb2a1b1
--- /dev/null
+++ b/crc/crct10dif_common.c
@@ -0,0 +1,78 @@
+/*
+ * Cryptographic API.
+ *
+ * T10 Data Integrity Field CRC16 Crypto Transform
+ *
+ * Copyright (c) 2007 Oracle Corporation.  All rights reserved.
+ * Written by Martin K. Petersen <martin.petersen@oracle.com>
+ * Copyright (C) 2013 Intel Corporation
+ * Author: Tim Chen <tim.c.chen@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include "crc-t10dif.h"
+
+/* Table generated using the following polynomium:
+ * x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1
+ * gt: 0x8bb7
+ */
+static const unsigned short t10_dif_crc_table[256] = {
+	0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
+	0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
+	0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
+	0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
+	0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
+	0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
+	0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
+	0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
+	0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
+	0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
+	0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
+	0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
+	0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
+	0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
+	0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
+	0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
+	0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
+	0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
+	0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
+	0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
+	0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
+	0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
+	0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
+	0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
+	0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
+	0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
+	0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
+	0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
+	0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
+	0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
+	0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
+	0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
+};
+
+extern unsigned short fio_crc_t10dif(unsigned short crc,
+				     const unsigned char *buffer,
+				     unsigned int len)
+{
+	unsigned int i;
+
+	for (i = 0 ; i < len ; i++)
+		crc = (crc << 8) ^ t10_dif_crc_table[((crc >> 8) ^ buffer[i]) & 0xff];
+
+	return crc;
+}
diff --git a/engines/io_uring.c b/engines/io_uring.c
index b361e6a5..6cdf1b4f 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -59,6 +59,7 @@ struct ioring_data {
 	int ring_fd;
 
 	struct io_u **io_u_index;
+	char *md_buf;
 
 	int *fds;
 
@@ -95,6 +96,12 @@ struct ioring_options {
 	unsigned int uncached;
 	unsigned int nowait;
 	unsigned int force_async;
+	unsigned int md_per_io_size;
+	unsigned int pi_act;
+	unsigned int apptag;
+	unsigned int apptag_mask;
+	unsigned int prchk;
+	char *pi_chk;
 	enum uring_cmd_type cmd_type;
 };
 
@@ -217,6 +224,56 @@ static struct fio_option options[] = {
 		.group	= FIO_OPT_G_IOURING,
 	},
 	CMDPRIO_OPTIONS(struct ioring_options, FIO_OPT_G_IOURING),
+	{
+		.name	= "md_per_io_size",
+		.lname	= "Separate Metadata Buffer Size per I/O",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct ioring_options, md_per_io_size),
+		.def	= "0",
+		.help	= "Size of separate metadata buffer per I/O (Default: 0)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_IOURING,
+	},
+	{
+		.name	= "pi_act",
+		.lname	= "Protection Information Action",
+		.type	= FIO_OPT_BOOL,
+		.off1	= offsetof(struct ioring_options, pi_act),
+		.def	= "1",
+		.help	= "Protection Information Action bit (pi_act=1 or pi_act=0)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_IOURING,
+	},
+	{
+		.name	= "pi_chk",
+		.lname	= "Protection Information Check",
+		.type	= FIO_OPT_STR_STORE,
+		.off1	= offsetof(struct ioring_options, pi_chk),
+		.def	= NULL,
+		.help	= "Control of Protection Information Checking (pi_chk=GUARD,REFTAG,APPTAG)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_IOURING,
+	},
+	{
+		.name	= "apptag",
+		.lname	= "Application Tag used in Protection Information",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct ioring_options, apptag),
+		.def	= "0x1234",
+		.help	= "Application Tag used in Protection Information field (Default: 0x1234)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_IOURING,
+	},
+	{
+		.name	= "apptag_mask",
+		.lname	= "Application Tag Mask",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct ioring_options, apptag_mask),
+		.def	= "0xffff",
+		.help	= "Application Tag Mask used with Application Tag (Default: 0xffff)",
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_IOURING,
+	},
 	{
 		.name	= NULL,
 	},
@@ -399,7 +456,9 @@ static struct io_u *fio_ioring_cmd_event(struct thread_data *td, int event)
 	struct ioring_options *o = td->eo;
 	struct io_uring_cqe *cqe;
 	struct io_u *io_u;
+	struct nvme_data *data;
 	unsigned index;
+	int ret;
 
 	index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
 	if (o->cmd_type == FIO_URING_CMD_NVME)
@@ -413,6 +472,15 @@ static struct io_u *fio_ioring_cmd_event(struct thread_data *td, int event)
 	else
 		io_u->error = 0;
 
+	if (o->cmd_type == FIO_URING_CMD_NVME) {
+		data = FILE_ENG_DATA(io_u->file);
+		if (data->pi_type && (io_u->ddir == DDIR_READ) && !o->pi_act) {
+			ret = fio_nvme_pi_verify(data, io_u);
+			if (ret)
+				io_u->error = ret;
+		}
+	}
+
 	return io_u;
 }
 
@@ -474,6 +542,33 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min,
 	return r < 0 ? r : events;
 }
 
+static inline void fio_ioring_cmd_nvme_pi(struct thread_data *td,
+					  struct io_u *io_u)
+{
+	struct ioring_data *ld = td->io_ops_data;
+	struct ioring_options *o = td->eo;
+	struct nvme_uring_cmd *cmd;
+	struct io_uring_sqe *sqe;
+	struct nvme_cmd_ext_io_opts ext_opts = {0};
+	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
+
+	if (io_u->ddir == DDIR_TRIM)
+		return;
+
+	sqe = &ld->sqes[(io_u->index) << 1];
+	cmd = (struct nvme_uring_cmd *)sqe->cmd;
+
+	if (data->pi_type) {
+		if (o->pi_act)
+			ext_opts.io_flags |= NVME_IO_PRINFO_PRACT;
+		ext_opts.io_flags |= o->prchk;
+		ext_opts.apptag = o->apptag;
+		ext_opts.apptag_mask = o->apptag_mask;
+	}
+
+	fio_nvme_pi_fill(cmd, io_u, &ext_opts);
+}
+
 static inline void fio_ioring_cmdprio_prep(struct thread_data *td,
 					   struct io_u *io_u)
 {
@@ -488,6 +583,7 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 					  struct io_u *io_u)
 {
 	struct ioring_data *ld = td->io_ops_data;
+	struct ioring_options *o = td->eo;
 	struct io_sq_ring *ring = &ld->sq_ring;
 	unsigned tail, next_tail;
 
@@ -515,6 +611,10 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 	if (ld->cmdprio.mode != CMDPRIO_MODE_NONE)
 		fio_ioring_cmdprio_prep(td, io_u);
 
+	if (!strcmp(td->io_ops->name, "io_uring_cmd") &&
+		o->cmd_type == FIO_URING_CMD_NVME)
+		fio_ioring_cmd_nvme_pi(td, io_u);
+
 	ring->array[tail & ld->sq_ring_mask] = io_u->index;
 	atomic_store_release(ring->tail, next_tail);
 
@@ -631,6 +731,7 @@ static void fio_ioring_cleanup(struct thread_data *td)
 
 		fio_cmdprio_cleanup(&ld->cmdprio);
 		free(ld->io_u_index);
+		free(ld->md_buf);
 		free(ld->iovecs);
 		free(ld->fds);
 		free(ld->dsm);
@@ -1012,10 +1113,24 @@ static int fio_ioring_cmd_post_init(struct thread_data *td)
 	return 0;
 }
 
+static void parse_prchk_flags(struct ioring_options *o)
+{
+	if (!o->pi_chk)
+		return;
+
+	if (strstr(o->pi_chk, "GUARD") != NULL)
+		o->prchk = NVME_IO_PRINFO_PRCHK_GUARD;
+	if (strstr(o->pi_chk, "REFTAG") != NULL)
+		o->prchk |= NVME_IO_PRINFO_PRCHK_REF;
+	if (strstr(o->pi_chk, "APPTAG") != NULL)
+		o->prchk |= NVME_IO_PRINFO_PRCHK_APP;
+}
+
 static int fio_ioring_init(struct thread_data *td)
 {
 	struct ioring_options *o = td->eo;
 	struct ioring_data *ld;
+	unsigned long long md_size;
 	int ret;
 
 	/* sqthread submission requires registered files */
@@ -1036,6 +1151,32 @@ static int fio_ioring_init(struct thread_data *td)
 
 	/* io_u index */
 	ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *));
+
+	/*
+	 * metadata buffer for nvme command.
+	 * We are only supporting iomem=malloc / mem=malloc as of now.
+	 */
+	if (!strcmp(td->io_ops->name, "io_uring_cmd") &&
+	    (o->cmd_type == FIO_URING_CMD_NVME) && o->md_per_io_size) {
+		md_size = (unsigned long long) o->md_per_io_size
+				* (unsigned long long) td->o.iodepth;
+		md_size += page_mask + td->o.mem_align;
+		if (td->o.mem_align && td->o.mem_align > page_size)
+			md_size += td->o.mem_align - page_size;
+		if (td->o.mem_type == MEM_MALLOC) {
+			ld->md_buf = malloc(md_size);
+			if (!ld->md_buf) {
+				free(ld);
+				return 1;
+			}
+		} else {
+			log_err("fio: Only iomem=malloc or mem=malloc is supported\n");
+			free(ld);
+			return 1;
+		}
+	}
+	parse_prchk_flags(o);
+
 	ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec));
 
 	td->io_ops_data = ld;
@@ -1062,11 +1203,42 @@ static int fio_ioring_init(struct thread_data *td)
 static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
 {
 	struct ioring_data *ld = td->io_ops_data;
+	struct ioring_options *o = td->eo;
+	struct nvme_pi_data *pi_data;
+	char *p;
 
 	ld->io_u_index[io_u->index] = io_u;
+
+	if (!strcmp(td->io_ops->name, "io_uring_cmd")) {
+		p = PTR_ALIGN(ld->md_buf, page_mask) + td->o.mem_align;
+		p += o->md_per_io_size * io_u->index;
+		io_u->mmap_data = p;
+
+		if (!o->pi_act) {
+			pi_data = calloc(1, sizeof(*pi_data));
+			pi_data->io_flags |= o->prchk;
+			pi_data->apptag_mask = o->apptag_mask;
+			pi_data->apptag = o->apptag;
+			io_u->engine_data = pi_data;
+		}
+	}
+
 	return 0;
 }
 
+static void fio_ioring_io_u_free(struct thread_data *td, struct io_u *io_u)
+{
+	struct ioring_options *o = td->eo;
+	struct nvme_pi *pi;
+
+	if (!strcmp(td->io_ops->name, "io_uring_cmd") &&
+	    (o->cmd_type == FIO_URING_CMD_NVME)) {
+		pi = io_u->engine_data;
+		free(pi);
+		io_u->engine_data = NULL;
+	}
+}
+
 static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f)
 {
 	struct ioring_data *ld = td->io_ops_data;
@@ -1086,39 +1258,44 @@ static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f)
 
 	if (o->cmd_type == FIO_URING_CMD_NVME) {
 		struct nvme_data *data = NULL;
-		unsigned int nsid, lba_size = 0;
-		__u32 ms = 0;
+		unsigned int lba_size = 0;
 		__u64 nlba = 0;
 		int ret;
 
 		/* Store the namespace-id and lba size. */
 		data = FILE_ENG_DATA(f);
 		if (data == NULL) {
-			ret = fio_nvme_get_info(f, &nsid, &lba_size, &ms, &nlba);
-			if (ret)
-				return ret;
-
 			data = calloc(1, sizeof(struct nvme_data));
-			data->nsid = nsid;
-			if (ms)
-				data->lba_ext = lba_size + ms;
-			else
-				data->lba_shift = ilog2(lba_size);
+			ret = fio_nvme_get_info(f, &nlba, o->pi_act, data);
+			if (ret) {
+				free(data);
+				return ret;
+			}
 
 			FILE_SET_ENG_DATA(f, data);
 		}
 
-		assert(data->lba_shift < 32);
-		lba_size = data->lba_ext ? data->lba_ext : (1U << data->lba_shift);
+		lba_size = data->lba_ext ? data->lba_ext : data->lba_size;
 
 		for_each_rw_ddir(ddir) {
 			if (td->o.min_bs[ddir] % lba_size ||
 				td->o.max_bs[ddir] % lba_size) {
 				if (data->lba_ext)
-					log_err("block size must be a multiple of "
-						"(LBA data size + Metadata size)\n");
+					log_err("%s: block size must be a multiple of (LBA data size + Metadata size)\n",
+						f->file_name);
 				else
-					log_err("block size must be a multiple of LBA data size\n");
+					log_err("%s: block size must be a multiple of LBA data size\n",
+						f->file_name);
+				td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
+				return 1;
+			}
+			if (data->ms && !data->lba_ext && ddir != DDIR_TRIM &&
+			    (o->md_per_io_size < ((td->o.max_bs[ddir] / data->lba_size) *
+						  data->ms))) {
+				log_err("%s: md_per_io_size should be at least %llu bytes\n",
+					f->file_name,
+					((td->o.max_bs[ddir] / data->lba_size) * data->ms));
+				td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
 				return 1;
 			}
                 }
@@ -1171,23 +1348,17 @@ static int fio_ioring_cmd_get_file_size(struct thread_data *td,
 
 	if (o->cmd_type == FIO_URING_CMD_NVME) {
 		struct nvme_data *data = NULL;
-		unsigned int nsid, lba_size = 0;
-		__u32 ms = 0;
 		__u64 nlba = 0;
 		int ret;
 
-		ret = fio_nvme_get_info(f, &nsid, &lba_size, &ms, &nlba);
-		if (ret)
-			return ret;
-
 		data = calloc(1, sizeof(struct nvme_data));
-		data->nsid = nsid;
-		if (ms)
-			data->lba_ext = lba_size + ms;
-		else
-			data->lba_shift = ilog2(lba_size);
+		ret = fio_nvme_get_info(f, &nlba, o->pi_act, data);
+		if (ret) {
+			free(data);
+			return ret;
+		}
 
-		f->real_file_size = lba_size * nlba;
+		f->real_file_size = data->lba_size * nlba;
 		fio_file_set_size_known(f);
 
 		FILE_SET_ENG_DATA(f, data);
@@ -1276,6 +1447,7 @@ static struct ioengine_ops ioengine_uring_cmd = {
 	.init			= fio_ioring_init,
 	.post_init		= fio_ioring_cmd_post_init,
 	.io_u_init		= fio_ioring_io_u_init,
+	.io_u_free		= fio_ioring_io_u_free,
 	.prep			= fio_ioring_cmd_prep,
 	.queue			= fio_ioring_queue,
 	.commit			= fio_ioring_commit,
diff --git a/engines/nvme.c b/engines/nvme.c
index b18ad4c2..08503b33 100644
--- a/engines/nvme.c
+++ b/engines/nvme.c
@@ -1,9 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * nvme structure declarations and helper functions for the
  * io_uring_cmd engine.
  */
 
 #include "nvme.h"
+#include "../crc/crc-t10dif.h"
+#include "../crc/crc64.h"
 
 static inline __u64 get_slba(struct nvme_data *data, struct io_u *io_u)
 {
@@ -21,6 +24,310 @@ static inline __u32 get_nlb(struct nvme_data *data, struct io_u *io_u)
 		return (io_u->xfer_buflen >> data->lba_shift) - 1;
 }
 
+static void fio_nvme_generate_pi_16b_guard(struct nvme_data *data,
+					   struct io_u *io_u,
+					   struct nvme_cmd_ext_io_opts *opts)
+{
+	struct nvme_pi_data *pi_data = io_u->engine_data;
+	struct nvme_16b_guard_pif *pi;
+	unsigned char *buf = io_u->xfer_buf;
+	unsigned char *md_buf = io_u->mmap_data;
+	__u64 slba = get_slba(data, io_u);
+	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u32 lba_num = 0;
+	__u16 guard = 0;
+
+	if (data->pi_loc) {
+		if (data->lba_ext)
+			pi_data->interval = data->lba_ext - data->ms;
+		else
+			pi_data->interval = 0;
+	} else {
+		if (data->lba_ext)
+			pi_data->interval = data->lba_ext - sizeof(struct nvme_16b_guard_pif);
+		else
+			pi_data->interval = data->ms - sizeof(struct nvme_16b_guard_pif);
+	}
+
+	if (io_u->ddir != DDIR_WRITE)
+		return;
+
+	while (lba_num < nlb) {
+		if (data->lba_ext)
+			pi = (struct nvme_16b_guard_pif *)(buf + pi_data->interval);
+		else
+			pi = (struct nvme_16b_guard_pif *)(md_buf + pi_data->interval);
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
+			if (data->lba_ext) {
+				guard = fio_crc_t10dif(0, buf, pi_data->interval);
+			} else {
+				guard = fio_crc_t10dif(0, buf, data->lba_size);
+				guard = fio_crc_t10dif(guard, md_buf, pi_data->interval);
+			}
+			pi->guard = cpu_to_be16(guard);
+		}
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
+			pi->apptag = cpu_to_be16(pi_data->apptag);
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+			switch (data->pi_type) {
+			case NVME_NS_DPS_PI_TYPE1:
+			case NVME_NS_DPS_PI_TYPE2:
+				pi->srtag = cpu_to_be32((__u32)slba + lba_num);
+				break;
+			case NVME_NS_DPS_PI_TYPE3:
+				break;
+			}
+		}
+		if (data->lba_ext) {
+			buf += data->lba_ext;
+		} else {
+			buf += data->lba_size;
+			md_buf += data->ms;
+		}
+		lba_num++;
+	}
+}
+
+static int fio_nvme_verify_pi_16b_guard(struct nvme_data *data,
+					struct io_u *io_u)
+{
+	struct nvme_pi_data *pi_data = io_u->engine_data;
+	struct nvme_16b_guard_pif *pi;
+	struct fio_file *f = io_u->file;
+	unsigned char *buf = io_u->xfer_buf;
+	unsigned char *md_buf = io_u->mmap_data;
+	__u64 slba = get_slba(data, io_u);
+	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u32 lba_num = 0;
+	__u16 unmask_app, unmask_app_exp, guard = 0;
+
+	while (lba_num < nlb) {
+		if (data->lba_ext)
+			pi = (struct nvme_16b_guard_pif *)(buf + pi_data->interval);
+		else
+			pi = (struct nvme_16b_guard_pif *)(md_buf + pi_data->interval);
+
+		if (data->pi_type == NVME_NS_DPS_PI_TYPE3) {
+			if (pi->apptag == NVME_PI_APP_DISABLE &&
+			    pi->srtag == NVME_PI_REF_DISABLE)
+				goto next;
+		} else if (data->pi_type == NVME_NS_DPS_PI_TYPE1 ||
+			   data->pi_type == NVME_NS_DPS_PI_TYPE2) {
+			if (pi->apptag == NVME_PI_APP_DISABLE)
+				goto next;
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
+			if (data->lba_ext) {
+				guard = fio_crc_t10dif(0, buf, pi_data->interval);
+			} else {
+				guard = fio_crc_t10dif(0, buf, data->lba_size);
+				guard = fio_crc_t10dif(guard, md_buf, pi_data->interval);
+			}
+			if (be16_to_cpu(pi->guard) != guard) {
+				log_err("%s: Guard compare error: LBA: %llu Expected=%x, Actual=%x\n",
+					f->file_name, (unsigned long long)slba,
+					guard, be16_to_cpu(pi->guard));
+				return -EIO;
+			}
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_APP) {
+			unmask_app = be16_to_cpu(pi->apptag) & pi_data->apptag_mask;
+			unmask_app_exp = pi_data->apptag & pi_data->apptag_mask;
+			if (unmask_app != unmask_app_exp) {
+				log_err("%s: APPTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
+					f->file_name, (unsigned long long)slba,
+					unmask_app_exp, unmask_app);
+				return -EIO;
+			}
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+			switch (data->pi_type) {
+			case NVME_NS_DPS_PI_TYPE1:
+			case NVME_NS_DPS_PI_TYPE2:
+				if (be32_to_cpu(pi->srtag) !=
+				    ((__u32)slba + lba_num)) {
+					log_err("%s: REFTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
+						f->file_name, (unsigned long long)slba,
+						(__u32)slba + lba_num,
+						be32_to_cpu(pi->srtag));
+					return -EIO;
+				}
+				break;
+			case NVME_NS_DPS_PI_TYPE3:
+				break;
+			}
+		}
+next:
+		if (data->lba_ext) {
+			buf += data->lba_ext;
+		} else {
+			buf += data->lba_size;
+			md_buf += data->ms;
+		}
+		lba_num++;
+	}
+
+	return 0;
+}
+
+static void fio_nvme_generate_pi_64b_guard(struct nvme_data *data,
+					   struct io_u *io_u,
+					   struct nvme_cmd_ext_io_opts *opts)
+{
+	struct nvme_pi_data *pi_data = io_u->engine_data;
+	struct nvme_64b_guard_pif *pi;
+	unsigned char *buf = io_u->xfer_buf;
+	unsigned char *md_buf = io_u->mmap_data;
+	uint64_t guard = 0;
+	__u64 slba = get_slba(data, io_u);
+	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u32 lba_num = 0;
+
+	if (data->pi_loc) {
+		if (data->lba_ext)
+			pi_data->interval = data->lba_ext - data->ms;
+		else
+			pi_data->interval = 0;
+	} else {
+		if (data->lba_ext)
+			pi_data->interval = data->lba_ext - sizeof(struct nvme_64b_guard_pif);
+		else
+			pi_data->interval = data->ms - sizeof(struct nvme_64b_guard_pif);
+	}
+
+	if (io_u->ddir != DDIR_WRITE)
+		return;
+
+	while (lba_num < nlb) {
+		if (data->lba_ext)
+			pi = (struct nvme_64b_guard_pif *)(buf + pi_data->interval);
+		else
+			pi = (struct nvme_64b_guard_pif *)(md_buf + pi_data->interval);
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
+			if (data->lba_ext) {
+				guard = fio_crc64_nvme(0, buf, pi_data->interval);
+			} else {
+				guard = fio_crc64_nvme(0, buf, data->lba_size);
+				guard = fio_crc64_nvme(guard, md_buf, pi_data->interval);
+			}
+			pi->guard = cpu_to_be64(guard);
+		}
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
+			pi->apptag = cpu_to_be16(pi_data->apptag);
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+			switch (data->pi_type) {
+			case NVME_NS_DPS_PI_TYPE1:
+			case NVME_NS_DPS_PI_TYPE2:
+				put_unaligned_be48(slba + lba_num, pi->srtag);
+				break;
+			case NVME_NS_DPS_PI_TYPE3:
+				break;
+			}
+		}
+		if (data->lba_ext) {
+			buf += data->lba_ext;
+		} else {
+			buf += data->lba_size;
+			md_buf += data->ms;
+		}
+		lba_num++;
+	}
+}
+
+static int fio_nvme_verify_pi_64b_guard(struct nvme_data *data,
+					struct io_u *io_u)
+{
+	struct nvme_pi_data *pi_data = io_u->engine_data;
+	struct nvme_64b_guard_pif *pi;
+	struct fio_file *f = io_u->file;
+	unsigned char *buf = io_u->xfer_buf;
+	unsigned char *md_buf = io_u->mmap_data;
+	__u64 slba = get_slba(data, io_u);
+	__u64 ref, ref_exp, guard = 0;
+	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u32 lba_num = 0;
+	__u16 unmask_app, unmask_app_exp;
+
+	while (lba_num < nlb) {
+		if (data->lba_ext)
+			pi = (struct nvme_64b_guard_pif *)(buf + pi_data->interval);
+		else
+			pi = (struct nvme_64b_guard_pif *)(md_buf + pi_data->interval);
+
+		if (data->pi_type == NVME_NS_DPS_PI_TYPE3) {
+			if (pi->apptag == NVME_PI_APP_DISABLE &&
+			    fio_nvme_pi_ref_escape(pi->srtag))
+				goto next;
+		} else if (data->pi_type == NVME_NS_DPS_PI_TYPE1 ||
+			   data->pi_type == NVME_NS_DPS_PI_TYPE2) {
+			if (pi->apptag == NVME_PI_APP_DISABLE)
+				goto next;
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
+			if (data->lba_ext) {
+				guard = fio_crc64_nvme(0, buf, pi_data->interval);
+			} else {
+				guard = fio_crc64_nvme(0, buf, data->lba_size);
+				guard = fio_crc64_nvme(guard, md_buf, pi_data->interval);
+			}
+			if (be64_to_cpu((uint64_t)pi->guard) != guard) {
+				log_err("%s: Guard compare error: LBA: %llu Expected=%llx, Actual=%llx\n",
+					f->file_name, (unsigned long long)slba,
+					guard, be64_to_cpu((uint64_t)pi->guard));
+				return -EIO;
+			}
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_APP) {
+			unmask_app = be16_to_cpu(pi->apptag) & pi_data->apptag_mask;
+			unmask_app_exp = pi_data->apptag & pi_data->apptag_mask;
+			if (unmask_app != unmask_app_exp) {
+				log_err("%s: APPTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
+					f->file_name, (unsigned long long)slba,
+					unmask_app_exp, unmask_app);
+				return -EIO;
+			}
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+			switch (data->pi_type) {
+			case NVME_NS_DPS_PI_TYPE1:
+			case NVME_NS_DPS_PI_TYPE2:
+				ref = get_unaligned_be48(pi->srtag);
+				ref_exp = (slba + lba_num) & ((1ULL << 48) - 1);
+				if (ref != ref_exp) {
+					log_err("%s: REFTAG compare error: LBA: %llu Expected=%llx, Actual=%llx\n",
+						f->file_name, (unsigned long long)slba,
+						ref_exp, ref);
+					return -EIO;
+				}
+				break;
+			case NVME_NS_DPS_PI_TYPE3:
+				break;
+			}
+		}
+next:
+		if (data->lba_ext) {
+			buf += data->lba_ext;
+		} else {
+			buf += data->lba_size;
+			md_buf += data->ms;
+		}
+		lba_num++;
+	}
+
+	return 0;
+}
 void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 				  struct nvme_dsm_range *dsm)
 {
@@ -79,10 +386,72 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 		cmd->addr = (__u64)(uintptr_t)io_u->xfer_buf;
 		cmd->data_len = io_u->xfer_buflen;
 	}
+	if (data->lba_shift && data->ms) {
+		cmd->metadata = (__u64)(uintptr_t)io_u->mmap_data;
+		cmd->metadata_len = (nlb + 1) * data->ms;
+	}
 	cmd->nsid = data->nsid;
 	return 0;
 }
 
+void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
+		      struct nvme_cmd_ext_io_opts *opts)
+{
+	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
+	__u64 slba;
+
+	slba = get_slba(data, io_u);
+	cmd->cdw12 |= opts->io_flags;
+
+	if (data->pi_type && !(opts->io_flags & NVME_IO_PRINFO_PRACT)) {
+		if (data->guard_type == NVME_NVM_NS_16B_GUARD)
+			fio_nvme_generate_pi_16b_guard(data, io_u, opts);
+		else if (data->guard_type == NVME_NVM_NS_64B_GUARD)
+			fio_nvme_generate_pi_64b_guard(data, io_u, opts);
+	}
+
+	switch (data->pi_type) {
+	case NVME_NS_DPS_PI_TYPE1:
+	case NVME_NS_DPS_PI_TYPE2:
+		switch (data->guard_type) {
+		case NVME_NVM_NS_16B_GUARD:
+			cmd->cdw14 = (__u32)slba;
+			break;
+		case NVME_NVM_NS_64B_GUARD:
+			cmd->cdw14 = (__u32)slba;
+			cmd->cdw3 = ((slba >> 32) & 0xffff);
+			break;
+		default:
+			break;
+		}
+		cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
+		break;
+	case NVME_NS_DPS_PI_TYPE3:
+		cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
+		break;
+	case NVME_NS_DPS_PI_NONE:
+		break;
+	}
+}
+
+int fio_nvme_pi_verify(struct nvme_data *data, struct io_u *io_u)
+{
+	int ret = 0;
+
+	switch (data->guard_type) {
+	case NVME_NVM_NS_16B_GUARD:
+		ret = fio_nvme_verify_pi_16b_guard(data, io_u);
+		break;
+	case NVME_NVM_NS_64B_GUARD:
+		ret = fio_nvme_verify_pi_64b_guard(data, io_u);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
 static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
 			 enum nvme_csi csi, void *data)
 {
@@ -99,13 +468,15 @@ static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
 	return ioctl(fd, NVME_IOCTL_ADMIN_CMD, &cmd);
 }
 
-int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
-		      __u32 *ms, __u64 *nlba)
+int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
+		      struct nvme_data *data)
 {
 	struct nvme_id_ns ns;
+	struct nvme_id_ctrl ctrl;
+	struct nvme_nvm_id_ns nvm_ns;
 	int namespace_id;
 	int fd, err;
-	__u32 format_idx;
+	__u32 format_idx, elbaf;
 
 	if (f->filetype != FIO_TYPE_CHAR) {
 		log_err("ioengine io_uring_cmd only works with nvme ns "
@@ -124,6 +495,12 @@ int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
 		goto out;
 	}
 
+	err = nvme_identify(fd, 0, NVME_IDENTIFY_CNS_CTRL, NVME_CSI_NVM, &ctrl);
+	if (err) {
+		log_err("%s: failed to fetch identify ctrl\n", f->file_name);
+		goto out;
+	}
+
 	/*
 	 * Identify namespace to get namespace-id, namespace size in LBA's
 	 * and LBA data size.
@@ -133,11 +510,10 @@ int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
 	if (err) {
 		log_err("%s: failed to fetch identify namespace\n",
 			f->file_name);
-		close(fd);
-		return err;
+		goto out;
 	}
 
-	*nsid = namespace_id;
+	data->nsid = namespace_id;
 
 	/*
 	 * 16 or 64 as maximum number of supported LBA formats.
@@ -149,28 +525,74 @@ int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
 	else
 		format_idx = (ns.flbas & 0xf) + (((ns.flbas >> 5) & 0x3) << 4);
 
-	*lba_sz = 1 << ns.lbaf[format_idx].ds;
+	data->lba_size = 1 << ns.lbaf[format_idx].ds;
+	data->ms = le16_to_cpu(ns.lbaf[format_idx].ms);
+
+	/* Check for end to end data protection support */
+	if (data->ms && (ns.dps & NVME_NS_DPS_PI_MASK))
+		data->pi_type = (ns.dps & NVME_NS_DPS_PI_MASK);
+
+	if (!data->pi_type)
+		goto check_elba;
+
+	if (ctrl.ctratt & NVME_CTRL_CTRATT_ELBAS) {
+		err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_CSI_NS,
+					NVME_CSI_NVM, &nvm_ns);
+		if (err) {
+			log_err("%s: failed to fetch identify nvm namespace\n",
+				f->file_name);
+			goto out;
+		}
+
+		elbaf = le32_to_cpu(nvm_ns.elbaf[format_idx]);
+
+		/* Currently we don't support storage tags */
+		if (elbaf & NVME_ID_NS_NVM_STS_MASK) {
+			log_err("%s: Storage tag not supported\n",
+				f->file_name);
+			err = -ENOTSUP;
+			goto out;
+		}
+
+		data->guard_type = (elbaf >> NVME_ID_NS_NVM_GUARD_SHIFT) &
+				NVME_ID_NS_NVM_GUARD_MASK;
+
+		/* No 32 bit guard, as storage tag is mandatory for it */
+		switch (data->guard_type) {
+		case NVME_NVM_NS_16B_GUARD:
+			data->pi_size = sizeof(struct nvme_16b_guard_pif);
+			break;
+		case NVME_NVM_NS_64B_GUARD:
+			data->pi_size = sizeof(struct nvme_64b_guard_pif);
+			break;
+		default:
+			break;
+		}
+	} else {
+		data->guard_type = NVME_NVM_NS_16B_GUARD;
+		data->pi_size = sizeof(struct nvme_16b_guard_pif);
+	}
+
+	/*
+	 * when PRACT bit is set to 1, and metadata size is equal to protection
+	 * information size, controller inserts and removes PI for write and
+	 * read commands respectively.
+	 */
+	if (pi_act && data->ms == data->pi_size)
+		data->ms = 0;
+
+	data->pi_loc = (ns.dps & NVME_NS_DPS_PI_FIRST);
 
+check_elba:
 	/*
-	 * Only extended LBA can be supported.
 	 * Bit 4 for flbas indicates if metadata is transferred at the end of
 	 * logical block creating an extended LBA.
 	 */
-	*ms = le16_to_cpu(ns.lbaf[format_idx].ms);
-	if (*ms && !((ns.flbas >> 4) & 0x1)) {
-		log_err("%s: only extended logical block can be supported\n",
-			f->file_name);
-		err = -ENOTSUP;
-		goto out;
-	}
+	if (data->ms && ((ns.flbas >> 4) & 0x1))
+		data->lba_ext = data->lba_size + data->ms;
+	else
+		data->lba_shift = ilog2(data->lba_size);
 
-	/* Check for end to end data protection support */
-	if (ns.dps & 0x3) {
-		log_err("%s: end to end data protection not supported\n",
-			f->file_name);
-		err = -ENOTSUP;
-		goto out;
-	}
 	*nlba = ns.nsze;
 
 out:
diff --git a/engines/nvme.h b/engines/nvme.h
index 238471dd..792b35d8 100644
--- a/engines/nvme.h
+++ b/engines/nvme.h
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * nvme structure declarations and helper functions for the
  * io_uring_cmd engine.
@@ -42,6 +43,10 @@ struct nvme_uring_cmd {
 #define NVME_DEFAULT_IOCTL_TIMEOUT 0
 #define NVME_IDENTIFY_DATA_SIZE 4096
 #define NVME_IDENTIFY_CSI_SHIFT 24
+#define NVME_NQN_LENGTH	256
+
+#define NVME_PI_APP_DISABLE 0xFFFF
+#define NVME_PI_REF_DISABLE 0xFFFFFFFF
 
 #define NVME_ZNS_ZRA_REPORT_ZONES 0
 #define NVME_ZNS_ZRAS_FEAT_ERZ (1 << 16)
@@ -52,6 +57,7 @@ struct nvme_uring_cmd {
 
 enum nvme_identify_cns {
 	NVME_IDENTIFY_CNS_NS		= 0x00,
+	NVME_IDENTIFY_CNS_CTRL		= 0x01,
 	NVME_IDENTIFY_CNS_CSI_NS	= 0x05,
 	NVME_IDENTIFY_CNS_CSI_CTRL	= 0x06,
 };
@@ -85,10 +91,55 @@ enum nvme_zns_zs {
 	NVME_ZNS_ZS_OFFLINE		= 0xf,
 };
 
+enum nvme_id_ctrl_ctratt {
+	NVME_CTRL_CTRATT_ELBAS		= 1 << 15,
+};
+
+enum {
+	NVME_ID_NS_NVM_STS_MASK		= 0x7f,
+	NVME_ID_NS_NVM_GUARD_SHIFT	= 7,
+	NVME_ID_NS_NVM_GUARD_MASK	= 0x3,
+};
+
+enum {
+	NVME_NVM_NS_16B_GUARD		= 0,
+	NVME_NVM_NS_32B_GUARD		= 1,
+	NVME_NVM_NS_64B_GUARD		= 2,
+};
+
 struct nvme_data {
 	__u32 nsid;
 	__u32 lba_shift;
+	__u32 lba_size;
 	__u32 lba_ext;
+	__u16 ms;
+	__u16 pi_size;
+	__u8 pi_type;
+	__u8 guard_type;
+	__u8 pi_loc;
+};
+
+enum nvme_id_ns_dps {
+	NVME_NS_DPS_PI_NONE		= 0,
+	NVME_NS_DPS_PI_TYPE1		= 1,
+	NVME_NS_DPS_PI_TYPE2		= 2,
+	NVME_NS_DPS_PI_TYPE3		= 3,
+	NVME_NS_DPS_PI_MASK		= 7 << 0,
+	NVME_NS_DPS_PI_FIRST		= 1 << 3,
+};
+
+enum nvme_io_control_flags {
+	NVME_IO_PRINFO_PRCHK_REF	= 1U << 26,
+	NVME_IO_PRINFO_PRCHK_APP	= 1U << 27,
+	NVME_IO_PRINFO_PRCHK_GUARD	= 1U << 28,
+	NVME_IO_PRINFO_PRACT		= 1U << 29,
+};
+
+struct nvme_pi_data {
+	__u32 interval;
+	__u32 io_flags;
+	__u16 apptag;
+	__u16 apptag_mask;
 };
 
 struct nvme_lbaf {
@@ -97,6 +148,20 @@ struct nvme_lbaf {
 	__u8			rp;
 };
 
+/* 16 bit guard protection Information format */
+struct nvme_16b_guard_pif {
+	__be16 guard;
+	__be16 apptag;
+	__be32 srtag;
+};
+
+/* 64 bit guard protection Information format */
+struct nvme_64b_guard_pif {
+	__be64 guard;
+	__be16 apptag;
+	__u8 srtag[6];
+};
+
 struct nvme_id_ns {
 	__le64			nsze;
 	__le64			ncap;
@@ -139,6 +204,133 @@ struct nvme_id_ns {
 	__u8			vs[3712];
 };
 
+struct nvme_id_psd {
+	__le16			mp;
+	__u8			rsvd2;
+	__u8			flags;
+	__le32			enlat;
+	__le32			exlat;
+	__u8			rrt;
+	__u8			rrl;
+	__u8			rwt;
+	__u8			rwl;
+	__le16			idlp;
+	__u8			ips;
+	__u8			rsvd19;
+	__le16			actp;
+	__u8			apws;
+	__u8			rsvd23[9];
+};
+
+struct nvme_id_ctrl {
+	__le16			vid;
+	__le16			ssvid;
+	char			sn[20];
+	char			mn[40];
+	char			fr[8];
+	__u8			rab;
+	__u8			ieee[3];
+	__u8			cmic;
+	__u8			mdts;
+	__le16			cntlid;
+	__le32			ver;
+	__le32			rtd3r;
+	__le32			rtd3e;
+	__le32			oaes;
+	__le32			ctratt;
+	__le16			rrls;
+	__u8			rsvd102[9];
+	__u8			cntrltype;
+	__u8			fguid[16];
+	__le16			crdt1;
+	__le16			crdt2;
+	__le16			crdt3;
+	__u8			rsvd134[119];
+	__u8			nvmsr;
+	__u8			vwci;
+	__u8			mec;
+	__le16			oacs;
+	__u8			acl;
+	__u8			aerl;
+	__u8			frmw;
+	__u8			lpa;
+	__u8			elpe;
+	__u8			npss;
+	__u8			avscc;
+	__u8			apsta;
+	__le16			wctemp;
+	__le16			cctemp;
+	__le16			mtfa;
+	__le32			hmpre;
+	__le32			hmmin;
+	__u8			tnvmcap[16];
+	__u8			unvmcap[16];
+	__le32			rpmbs;
+	__le16			edstt;
+	__u8			dsto;
+	__u8			fwug;
+	__le16			kas;
+	__le16			hctma;
+	__le16			mntmt;
+	__le16			mxtmt;
+	__le32			sanicap;
+	__le32			hmminds;
+	__le16			hmmaxd;
+	__le16			nsetidmax;
+	__le16			endgidmax;
+	__u8			anatt;
+	__u8			anacap;
+	__le32			anagrpmax;
+	__le32			nanagrpid;
+	__le32			pels;
+	__le16			domainid;
+	__u8			rsvd358[10];
+	__u8			megcap[16];
+	__u8			rsvd384[128];
+	__u8			sqes;
+	__u8			cqes;
+	__le16			maxcmd;
+	__le32			nn;
+	__le16			oncs;
+	__le16			fuses;
+	__u8			fna;
+	__u8			vwc;
+	__le16			awun;
+	__le16			awupf;
+	__u8			icsvscc;
+	__u8			nwpc;
+	__le16			acwu;
+	__le16			ocfs;
+	__le32			sgls;
+	__le32			mnan;
+	__u8			maxdna[16];
+	__le32			maxcna;
+	__u8			rsvd564[204];
+	char			subnqn[NVME_NQN_LENGTH];
+	__u8			rsvd1024[768];
+
+	/* Fabrics Only */
+	__le32			ioccsz;
+	__le32			iorcsz;
+	__le16			icdoff;
+	__u8			fcatt;
+	__u8			msdbd;
+	__le16			ofcs;
+	__u8			dctype;
+	__u8			rsvd1807[241];
+
+	struct nvme_id_psd	psd[32];
+	__u8			vs[1024];
+};
+
+struct nvme_nvm_id_ns {
+	__le64			lbstm;
+	__u8			pic;
+	__u8			rsvd9[3];
+	__le32			elbaf[64];
+	__u8			rsvd268[3828];
+};
+
 static inline int ilog2(uint32_t i)
 {
 	int log = -1;
@@ -216,15 +408,26 @@ struct nvme_dsm_range {
 	__le64	slba;
 };
 
+struct nvme_cmd_ext_io_opts {
+	__u32 io_flags;
+	__u16 apptag;
+	__u16 apptag_mask;
+};
+
 int fio_nvme_iomgmt_ruhs(struct thread_data *td, struct fio_file *f,
 			 struct nvme_fdp_ruh_status *ruhs, __u32 bytes);
 
-int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
-		      __u32 *ms, __u64 *nlba);
+int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
+		      struct nvme_data *data);
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 			    struct iovec *iov, struct nvme_dsm_range *dsm);
 
+void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
+		      struct nvme_cmd_ext_io_opts *opts);
+
+int fio_nvme_pi_verify(struct nvme_data *data, struct io_u *io_u);
+
 int fio_nvme_get_zoned_model(struct thread_data *td, struct fio_file *f,
 			     enum zbd_zoned_model *model);
 
@@ -238,4 +441,27 @@ int fio_nvme_reset_wp(struct thread_data *td, struct fio_file *f,
 int fio_nvme_get_max_open_zones(struct thread_data *td, struct fio_file *f,
 				unsigned int *max_open_zones);
 
+static inline void put_unaligned_be48(__u64 val, __u8 *p)
+{
+	*p++ = val >> 40;
+	*p++ = val >> 32;
+	*p++ = val >> 24;
+	*p++ = val >> 16;
+	*p++ = val >> 8;
+	*p++ = val;
+}
+
+static inline __u64 get_unaligned_be48(__u8 *p)
+{
+	return (__u64)p[0] << 40 | (__u64)p[1] << 32 | (__u64)p[2] << 24 |
+		p[3] << 16 | p[4] << 8 | p[5];
+}
+
+static inline bool fio_nvme_pi_ref_escape(__u8 *reftag)
+{
+	__u8 ref_esc[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
+	return memcmp(reftag, ref_esc, sizeof(ref_esc)) == 0;
+}
+
 #endif
diff --git a/fio.1 b/fio.1
index f62617e7..f0dc49ab 100644
--- a/fio.1
+++ b/fio.1
@@ -2247,6 +2247,44 @@ By default, the job will cycle through all available Placement IDs, so use this
 to isolate these identifiers to specific jobs. If you want fio to use placement
 identifier only at indices 0, 2 and 5 specify, you would set `fdp_pli=0,2,5`.
 .TP
+.BI (io_uring_cmd)md_per_io_size \fR=\fPint
+Size in bytes for separate metadata buffer per IO. Default: 0.
+.TP
+.BI (io_uring_cmd)pi_act \fR=\fPint
+Action to take when nvme namespace is formatted with protection information.
+If this is set to 1 and namespace is formatted with metadata size equal to
+protection information size, fio won't use separate metadata buffer or extended
+logical block. If this is set to 1 and namespace is formatted with metadata
+size greater than protection information size, fio will not generate or verify
+the protection information portion of metadata for write or read case
+respectively. If this is set to 0, fio generates protection information for
+write case and verifies for read case. Default: 1.
+.TP
+.BI (io_uring_cmd)pi_chk \fR=\fPstr[,str][,str]
+Controls the protection information check. This can take one or more of these
+values. Default: none.
+.RS
+.RS
+.TP
+.B GUARD
+Enables protection information checking of guard field.
+.TP
+.B REFTAG
+Enables protection information checking of logical block reference tag field.
+.TP
+.B APPTAG
+Enables protection information checking of application tag field.
+.RE
+.RE
+.TP
+.BI (io_uring_cmd)apptag \fR=\fPint
+Specifies logical block application tag value, if namespace is formatted to use
+end to end protection information. Default: 0x1234.
+.TP
+.BI (io_uring_cmd)apptag_mask \fR=\fPint
+Specifies logical block application tag mask value, if namespace is formatted
+to use end to end protection information. Default: 0xffff.
+.TP
 .BI (cpuio)cpuload \fR=\fPint
 Attempt to use the specified percentage of CPU cycles. This is a mandatory
 option when using cpuio I/O engine.
diff --git a/io_u.h b/io_u.h
index b432a540..786251d5 100644
--- a/io_u.h
+++ b/io_u.h
@@ -89,8 +89,8 @@ struct io_u {
 	union {
 		unsigned int index;
 		unsigned int seen;
-		void *engine_data;
 	};
+	void *engine_data;
 
 	union {
 		struct flist_head verify_list;
diff --git a/t/fiotestlib.py b/t/fiotestlib.py
index 1f35de0a..a96338a3 100755
--- a/t/fiotestlib.py
+++ b/t/fiotestlib.py
@@ -382,9 +382,10 @@ def run_fio_tests(test_list, test_env, args):
 
     for config in test_list:
         if (args.skip and config['test_id'] in args.skip) or \
-           (args.run_only and config['test_id'] not in args.run_only):
+           (args.run_only and config['test_id'] not in args.run_only) or \
+           ('force_skip' in config and config['force_skip']):
             skipped = skipped + 1
-            print(f"Test {config['test_id']} SKIPPED (User request)")
+            print(f"Test {config['test_id']} SKIPPED (User request or override)")
             continue
 
         if issubclass(config['test_class'], FioJobFileTest):
diff --git a/t/nvmept_pi.py b/t/nvmept_pi.py
new file mode 100755
index 00000000..5de77c9d
--- /dev/null
+++ b/t/nvmept_pi.py
@@ -0,0 +1,949 @@
+#!/usr/bin/env python3
+"""
+# nvmept_pi.py
+#
+# Test fio's io_uring_cmd ioengine support for DIF/DIX end-to-end data
+# protection.
+#
+# USAGE
+# see python3 nvmept_pi.py --help
+#
+# EXAMPLES (THIS IS A DESTRUCTIVE TEST!!)
+# python3 t/nvmept_pi.py --dut /dev/ng0n1 -f ./fio
+# python3 t/nvmept_pi.py --dut /dev/ng0n1 -f ./fio --lbaf 1
+#
+# REQUIREMENTS
+# Python 3.6
+#
+"""
+import os
+import sys
+import json
+import time
+import locale
+import logging
+import argparse
+import itertools
+import subprocess
+from pathlib import Path
+from fiotestlib import FioJobCmdTest, run_fio_tests
+from fiotestcommon import SUCCESS_NONZERO
+
+NUMBER_IOS = 8192
+BS_LOW = 1
+BS_HIGH = 16
+
+class DifDixTest(FioJobCmdTest):
+    """
+    NVMe DIF/DIX test class.
+    """
+
+    def setup(self, parameters):
+        """Setup a test."""
+
+        fio_args = [
+            "--name=nvmept_pi",
+            "--ioengine=io_uring_cmd",
+            "--cmd_type=nvme",
+            f"--filename={self.fio_opts['filename']}",
+            f"--rw={self.fio_opts['rw']}",
+            f"--bsrange={self.fio_opts['bsrange']}",
+            f"--output={self.filenames['output']}",
+            f"--output-format={self.fio_opts['output-format']}",
+            f"--md_per_io_size={self.fio_opts['md_per_io_size']}",
+            f"--pi_act={self.fio_opts['pi_act']}",
+            f"--pi_chk={self.fio_opts['pi_chk']}",
+            f"--apptag={self.fio_opts['apptag']}",
+            f"--apptag_mask={self.fio_opts['apptag_mask']}",
+        ]
+        for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles',
+                    'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait',
+                    'time_based', 'runtime', 'verify', 'io_size', 'offset', 'number_ios']:
+            if opt in self.fio_opts:
+                option = f"--{opt}={self.fio_opts[opt]}"
+                fio_args.append(option)
+
+        super().setup(fio_args)
+
+
+TEST_LIST = [
+#
+# Write data with pi_act=1 and then read the data back (with both
+# pi_act=[0,1]).
+#
+    {
+        # Write workload with variable IO sizes
+        # pi_act=1
+        "test_id": 101,
+        "fio_opts": {
+            "rw": 'write',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            "pi_act": 1,
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with fixed small IO size
+        # pi_act=0
+        "test_id": 102,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_LOW,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with fixed small IO size
+        # pi_act=1
+        "test_id": 103,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_LOW,
+        "test_class": DifDixTest,
+    },
+    {
+        # Write workload with fixed large IO size
+        # Precondition for read workloads to follow
+        # pi_act=1
+        "test_id": 104,
+        "fio_opts": {
+            "rw": 'write',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            "pi_act": 1,
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_HIGH,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        "test_id": 105,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        "test_id": 106,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+#
+# Write data with pi_act=0 and then read the data back (with both
+# pi_act=[0,1]).
+#
+    {
+        # Write workload with variable IO sizes
+        # pi_act=0
+        "test_id": 201,
+        "fio_opts": {
+            "rw": 'write',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            "pi_act": 0,
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with fixed small IO size
+        # pi_act=0
+        "test_id": 202,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_LOW,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with fixed small IO size
+        # pi_act=1
+        "test_id": 203,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_LOW,
+        "test_class": DifDixTest,
+    },
+    {
+        # Write workload with fixed large IO sizes
+        # pi_act=0
+        "test_id": 204,
+        "fio_opts": {
+            "rw": 'write',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            "pi_act": 0,
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_HIGH,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        "test_id": 205,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        "test_id": 206,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x8888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+#
+# Test apptag errors.
+#
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # trigger an apptag error
+        "test_id": 301,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "success": SUCCESS_NONZERO,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        # trigger an apptag error
+        "test_id": 302,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "success": SUCCESS_NONZERO,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # trigger an apptag error
+        # same as above but with pi_chk=APPTAG only
+        "test_id": 303,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "success": SUCCESS_NONZERO,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        # trigger an apptag error
+        # same as above but with pi_chk=APPTAG only
+        "test_id": 304,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "success": SUCCESS_NONZERO,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # this case would trigger an apptag error, but pi_chk says to check
+        # only the Guard PI and reftag, so there should be no error
+        "test_id": 305,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        # this case would trigger an apptag error, but pi_chk says to check
+        # only the Guard PI and reftag, so there should be no error
+        "test_id": 306,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # this case would trigger an apptag error, but pi_chk says to check
+        # only the Guard PI, so there should be no error
+        "test_id": 307,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "GUARD",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        # this case would trigger an apptag error, but pi_chk says to check
+        # only the Guard PI, so there should be no error
+        "test_id": 308,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "GUARD",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # this case would trigger an apptag error, but pi_chk says to check
+        # only the reftag, so there should be no error
+        # This case will be skipped when the device is formatted with Type 3 PI
+        # since Type 3 PI ignores the reftag
+        "test_id": 309,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "skip": "type3",
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        # this case would trigger an apptag error, but pi_chk says to check
+        # only the reftag, so there should be no error
+        # This case will be skipped when the device is formatted with Type 3 PI
+        # since Type 3 PI ignores the reftag
+        "test_id": 310,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x0888",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "skip": "type3",
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # use apptag mask to ignore apptag mismatch
+        "test_id": 311,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x0888",
+            "apptag_mask": "0x0FFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        # use apptag mask to ignore apptag mismatch
+        "test_id": 312,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x0888",
+            "apptag_mask": "0x0FFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # use apptag mask to ignore apptag mismatch
+        "test_id": 313,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0xF888",
+            "apptag_mask": "0x0FFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        # use apptag mask to ignore apptag mismatch
+        "test_id": 314,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0xF888",
+            "apptag_mask": "0x0FFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "test_class": DifDixTest,
+    },
+    {
+        # Write workload with fixed large IO sizes
+        # Set apptag=0xFFFF to disable all checking for Type 1 and 2
+        # pi_act=1
+        "test_id": 315,
+        "fio_opts": {
+            "rw": 'write',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "apptag": "0xFFFF",
+            "apptag_mask": "0xFFFF",
+            "pi_act": 1,
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_HIGH,
+        "bs_high": BS_HIGH,
+        "skip": "type3",
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # Data was written with apptag=0xFFFF
+        # Reading the data back should disable all checking for Type 1 and 2
+        "test_id": 316,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x0101",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "skip": "type3",
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=1
+        # Data was written with apptag=0xFFFF
+        # Reading the data back should disable all checking for Type 1 and 2
+        "test_id": 317,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 1,
+            "apptag": "0x0000",
+            "apptag_mask": "0xFFFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "skip": "type3",
+        "test_class": DifDixTest,
+    },
+#
+# Error cases related to block size and metadata size
+#
+    {
+        # Use a min block size that is not a multiple of lba/elba size to
+        # trigger an error.
+        "test_id": 401,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x8888",
+            "apptag_mask": "0x0FFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW+0.5,
+        "bs_high": BS_HIGH,
+        "success": SUCCESS_NONZERO,
+        "test_class": DifDixTest,
+    },
+    {
+        # Use metadata size that is too small
+        "test_id": 402,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x8888",
+            "apptag_mask": "0x0FFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "mdsize_adjustment": -1,
+        "success": SUCCESS_NONZERO,
+        "skip": "elba",
+        "test_class": DifDixTest,
+    },
+    {
+        # Read workload with variable IO sizes
+        # pi_act=0
+        # Should still work even if metadata size is too large
+        "test_id": 403,
+        "fio_opts": {
+            "rw": 'read',
+            "number_ios": NUMBER_IOS,
+            "output-format": "json",
+            "pi_act": 0,
+            "apptag": "0x8888",
+            "apptag_mask": "0x0FFF",
+            },
+        "pi_chk": "APPTAG,GUARD,REFTAG",
+        "bs_low": BS_LOW,
+        "bs_high": BS_HIGH,
+        "mdsize_adjustment": 1,
+        "test_class": DifDixTest,
+    },
+]
+
+
+def parse_args():
+    """Parse command-line arguments."""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-d', '--debug', help='Enable debug messages', action='store_true')
+    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
+    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
+    parser.add_argument('-s', '--skip', nargs='+', type=int,
+                        help='list of test(s) to skip')
+    parser.add_argument('-o', '--run-only', nargs='+', type=int,
+                        help='list of test(s) to run, skipping all others')
+    parser.add_argument('--dut', help='target NVMe character device to test '
+                        '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
+    parser.add_argument('-l', '--lbaf', nargs='+', type=int,
+                        help='list of lba formats to test')
+    args = parser.parse_args()
+
+    return args
+
+
+def get_lbafs(args):
+    """
+    Determine which LBA formats to use. Use either the ones specified on the
+    command line or if none are specified query the device and use all lba
+    formats with metadata.
+    """
+    lbaf_list = []
+    id_ns_cmd = f"sudo nvme id-ns --output-format=json {args.dut}".split(' ')
+    id_ns_output = subprocess.check_output(id_ns_cmd)
+    lbafs = json.loads(id_ns_output)['lbafs']
+    if args.lbaf:
+        for lbaf in args.lbaf:
+            lbaf_list.append({'lbaf': lbaf, 'ds': 2 ** lbafs[lbaf]['ds'],
+                              'ms': lbafs[lbaf]['ms'], })
+            if lbafs[lbaf]['ms'] == 0:
+                print(f'Error: lbaf {lbaf} has metadata size zero')
+                sys.exit(1)
+    else:
+        for lbaf_num, lbaf in enumerate(lbafs):
+            if lbaf['ms'] != 0:
+                lbaf_list.append({'lbaf': lbaf_num, 'ds': 2 ** lbaf['ds'],
+                                  'ms': lbaf['ms'], })
+
+    return lbaf_list
+
+
+def get_guard_pi(lbaf_list, args):
+    """
+    Find out how many bits of guard protection information are associated with
+    each lbaf to be used. If this is not available assume 16-bit guard pi.
+    Also record the bytes of protection information associated with the number
+    of guard PI bits.
+    """
+    nvm_id_ns_cmd = f"sudo nvme nvm-id-ns --output-format=json {args.dut}".split(' ')
+    try:
+        nvm_id_ns_output = subprocess.check_output(nvm_id_ns_cmd)
+    except subprocess.CalledProcessError:
+        print(f"Non-zero return code from {' '.join(nvm_id_ns_cmd)}; " \
+                "assuming all lbafs use 16b Guard Protection Information")
+        for lbaf in lbaf_list:
+            lbaf['guard_pi_bits'] = 16
+    else:
+        elbafs = json.loads(nvm_id_ns_output)['elbafs']
+        for elbaf_num, elbaf in enumerate(elbafs):
+            for lbaf in lbaf_list:
+                if lbaf['lbaf'] == elbaf_num:
+                    lbaf['guard_pi_bits'] = 16 << elbaf['pif']
+
+    # For 16b Guard Protection Information, the PI requires 8 bytes
+    # For 32b and 64b Guard PI, the PI requires 16 bytes
+    for lbaf in lbaf_list:
+        if lbaf['guard_pi_bits'] == 16:
+            lbaf['pi_bytes'] = 8
+        else:
+            lbaf['pi_bytes'] = 16
+
+
+def get_capabilities(args):
+    """
+    Determine what end-to-end data protection features the device supports.
+    """
+    caps = { 'pil': [], 'pitype': [], 'elba': [] }
+    id_ns_cmd = f"sudo nvme id-ns --output-format=json {args.dut}".split(' ')
+    id_ns_output = subprocess.check_output(id_ns_cmd)
+    id_ns_json = json.loads(id_ns_output)
+
+    mc = id_ns_json['mc']
+    if mc & 1:
+        caps['elba'].append(1)
+    if mc & 2:
+        caps['elba'].append(0)
+
+    dpc = id_ns_json['dpc']
+    if dpc & 1:
+        caps['pitype'].append(1)
+    if dpc & 2:
+        caps['pitype'].append(2)
+    if dpc & 4:
+        caps['pitype'].append(3)
+    if dpc & 8:
+        caps['pil'].append(1)
+    if dpc & 16:
+        caps['pil'].append(0)
+
+    for _, value in caps.items():
+        if len(value) == 0:
+            logging.error("One or more end-to-end data protection features unsupported: %s", caps)
+            sys.exit(-1)
+
+    return caps
+
+
+def format_device(args, lbaf, pitype, pil, elba):
+    """
+    Format device using specified lba format with specified pitype, pil, and
+    elba values.
+    """
+
+    format_cmd = f"sudo nvme format {args.dut} --lbaf={lbaf['lbaf']} " \
+                 f"--pi={pitype} --pil={pil} --ms={elba} --force"
+    logging.debug("Format command: %s", format_cmd)
+    format_cmd = format_cmd.split(' ')
+    format_cmd_result = subprocess.run(format_cmd, capture_output=True, check=False,
+                                       encoding=locale.getpreferredencoding())
+
+    # Sometimes nvme-cli may format the device successfully but fail to
+    # rescan the namespaces after the format. Continue if this happens but
+    # abort if some other error occurs.
+    if format_cmd_result.returncode != 0:
+        if 'failed to rescan namespaces' not in format_cmd_result.stderr \
+                or 'Success formatting namespace' not in format_cmd_result.stdout:
+            logging.error(format_cmd_result.stdout)
+            logging.error(format_cmd_result.stderr)
+            print("Unable to format device; skipping this configuration")
+            return False
+
+    logging.debug(format_cmd_result.stdout)
+    return True
+
+
+def difdix_test(test_env, args, lbaf, pitype, elba):
+    """
+    Adjust test arguments based on values of lbaf, pitype, and elba.  Then run
+    the tests.
+    """
+    for test in TEST_LIST:
+        test['force_skip'] = False
+
+        blocksize = lbaf['ds']
+        # Set fio blocksize parameter at runtime
+        # If we formatted the device in extended LBA mode (e.g., 520-byte
+        # sectors), we usually need to add the lba data size and metadata size
+        # together for fio's bs parameter. However, if pi_act == 1 and the
+        # device is formatted so that the metadata is the same size as the PI,
+        # then the device will take care of everything and the application
+        # should just use regular power of 2 lba data size even when the device
+        # is in extended lba mode.
+        if elba:
+            if not test['fio_opts']['pi_act'] or lbaf['ms'] != lbaf['pi_bytes']:
+                blocksize += lbaf['ms']
+            test['fio_opts']['md_per_io_size'] = 0
+        else:
+        # If we are using a separate buffer for metadata, fio doesn't need to
+        # do anything when pi_act==1 and protection information size is equal to
+        # metadata size since the device is taking care of it all. If either of
+        # the two conditions do not hold, then we do need to allocate a
+        # separate metadata buffer.
+            if test['fio_opts']['pi_act'] and lbaf['ms'] == lbaf['pi_bytes']:
+                test['fio_opts']['md_per_io_size'] = 0
+            else:
+                test['fio_opts']['md_per_io_size'] = lbaf['ms'] * test['bs_high']
+
+        test['fio_opts']['bsrange'] = f"{blocksize * test['bs_low']}-{blocksize * test['bs_high']}"
+        if 'mdsize_adjustment' in test:
+            test['fio_opts']['md_per_io_size'] += test['mdsize_adjustment']
+
+        # Set fio pi_chk parameter at runtime. If the device is formatted
+        # with Type 3 protection information, this means that the reference
+        # tag is not checked and I/O commands may throw an error if they
+        # are submitted with the REFTAG bit set in pi_chk. Make sure fio
+        # does not set pi_chk's REFTAG bit if the device is formatted with
+        # Type 3 PI.
+        if 'pi_chk' in test:
+            if pitype == 3 and 'REFTAG' in test['pi_chk']:
+                test['fio_opts']['pi_chk'] = test['pi_chk'].replace('REFTAG','')
+                logging.debug("Type 3 PI: dropping REFTAG bit")
+            else:
+                test['fio_opts']['pi_chk'] = test['pi_chk']
+
+        if 'skip' in test:
+            if pitype == 3 and 'type3' in test['skip']:
+                test['force_skip'] = True
+                logging.debug("Type 3 PI: skipping test case")
+            if elba and 'elba' in test['skip']:
+                test['force_skip'] = True
+                logging.debug("extended lba format: skipping test case")
+
+        logging.debug("Test %d: pi_act=%d, bsrange=%s, md_per_io_size=%d", test['test_id'],
+                      test['fio_opts']['pi_act'], test['fio_opts']['bsrange'],
+                      test['fio_opts']['md_per_io_size'])
+
+    return run_fio_tests(TEST_LIST, test_env, args)
+
+
+def main():
+    """
+    Run tests using fio's io_uring_cmd ioengine to exercise end-to-end data
+    protection capabilities.
+    """
+
+    args = parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    artifact_root = args.artifact_root if args.artifact_root else \
+        f"nvmept_pi-test-{time.strftime('%Y%m%d-%H%M%S')}"
+    os.mkdir(artifact_root)
+    print(f"Artifact directory is {artifact_root}")
+
+    if args.fio:
+        fio_path = str(Path(args.fio).absolute())
+    else:
+        fio_path = 'fio'
+    print(f"fio path is {fio_path}")
+
+    lbaf_list = get_lbafs(args)
+    get_guard_pi(lbaf_list, args)
+    caps = get_capabilities(args)
+    print("Device capabilities:", caps)
+
+    for test in TEST_LIST:
+        test['fio_opts']['filename'] = args.dut
+
+    test_env = {
+              'fio_path': fio_path,
+              'fio_root': str(Path(__file__).absolute().parent.parent),
+              'artifact_root': artifact_root,
+              'basename': 'nvmept_pi',
+              }
+
+    total = { 'passed':  0, 'failed': 0, 'skipped': 0 }
+
+    try:
+        for lbaf, pil, pitype, elba in itertools.product(lbaf_list, caps['pil'], caps['pitype'],
+                                                         caps['elba']):
+            print(f"\nlbaf: {lbaf}, pil: {pil}, pitype: {pitype}, elba: {elba}")
+
+            if not format_device(args, lbaf, pitype, pil, elba):
+                continue
+
+            test_env['artifact_root'] = \
+                os.path.join(artifact_root, f"lbaf{lbaf['lbaf']}pil{pil}pitype{pitype}" \
+                    f"elba{elba}")
+            os.mkdir(test_env['artifact_root'])
+
+            passed, failed, skipped = difdix_test(test_env, args, lbaf, pitype, elba)
+
+            total['passed'] += passed
+            total['failed'] += failed
+            total['skipped'] += skipped
+    except KeyboardInterrupt:
+        pass
+
+    print(f"\n\n{total['passed']} test(s) passed, {total['failed']} failed, " \
+            f"{total['skipped']} skipped")
+    sys.exit(total['failed'])
+
+
+if __name__ == '__main__':
+    main()

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-08-04 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-08-04 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 7b57011427a8204bd63671b08dde56cd9e879d68:

  t/fiotestlib: make recorded command prettier (2023-08-02 12:58:16 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 62f35562722f0c903567096d0f10a836d1ae2f60:

  eta: calculate aggregate bw statistics even when eta is disabled (2023-08-03 11:49:08 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      eta: calculate aggregate bw statistics even when eta is disabled

 eta.c | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

---

Diff of recent changes:

diff --git a/eta.c b/eta.c
index af4027e0..cc342461 100644
--- a/eta.c
+++ b/eta.c
@@ -375,6 +375,22 @@ bool eta_time_within_slack(unsigned int time)
 	return time > ((eta_interval_msec * 95) / 100);
 }
 
+/*
+ * These are the conditions under which we might be able to skip the eta
+ * calculation.
+ */
+static bool skip_eta()
+{
+	if (!(output_format & FIO_OUTPUT_NORMAL) && f_out == stdout)
+		return true;
+	if (temp_stall_ts || eta_print == FIO_ETA_NEVER)
+		return true;
+	if (!isatty(STDOUT_FILENO) && eta_print != FIO_ETA_ALWAYS)
+		return true;
+
+	return false;
+}
+
 /*
  * Print status of the jobs we know about. This includes rate estimates,
  * ETA, thread state, etc.
@@ -393,14 +409,12 @@ bool calc_thread_status(struct jobs_eta *je, int force)
 	static unsigned long long disp_io_iops[DDIR_RWDIR_CNT];
 	static struct timespec rate_prev_time, disp_prev_time;
 
-	if (!force) {
-		if (!(output_format & FIO_OUTPUT_NORMAL) &&
-		    f_out == stdout)
-			return false;
-		if (temp_stall_ts || eta_print == FIO_ETA_NEVER)
-			return false;
+	bool ret = true;
 
-		if (!isatty(STDOUT_FILENO) && (eta_print != FIO_ETA_ALWAYS))
+	if (!force && skip_eta()) {
+		if (write_bw_log)
+			ret = false;
+		else
 			return false;
 	}
 
@@ -534,7 +548,7 @@ bool calc_thread_status(struct jobs_eta *je, int force)
 	je->nr_threads = thread_number;
 	update_condensed_str(__run_str, run_str);
 	memcpy(je->run_str, run_str, strlen(run_str));
-	return true;
+	return ret;
 }
 
 static int gen_eta_str(struct jobs_eta *je, char *p, size_t left,

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-08-03 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-08-03 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 1660df6601e24a17dda9e12cbc901337fd5fd925:

  Merge branch 'master' of https://github.com/min22/fio (2023-07-31 15:03:37 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 7b57011427a8204bd63671b08dde56cd9e879d68:

  t/fiotestlib: make recorded command prettier (2023-08-02 12:58:16 -0400)

----------------------------------------------------------------
Vincent Fu (2):
      t/nvmept: fix typo
      t/fiotestlib: make recorded command prettier

 t/fiotestlib.py | 2 +-
 t/nvmept.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/t/fiotestlib.py b/t/fiotestlib.py
index 0fe17b74..1f35de0a 100755
--- a/t/fiotestlib.py
+++ b/t/fiotestlib.py
@@ -75,7 +75,7 @@ class FioExeTest(FioTest):
         command = [self.paths['exe']] + self.parameters
         with open(self.filenames['cmd'], "w+",
                   encoding=locale.getpreferredencoding()) as command_file:
-            command_file.write(" ".join(command))
+            command_file.write(" \\\n ".join(command))
 
         try:
             with open(self.filenames['stdout'], "w+",
diff --git a/t/nvmept.py b/t/nvmept.py
index cc26d152..c08fb350 100755
--- a/t/nvmept.py
+++ b/t/nvmept.py
@@ -295,7 +295,7 @@ def main():
               'fio_path': fio_path,
               'fio_root': str(Path(__file__).absolute().parent.parent),
               'artifact_root': artifact_root,
-              'basename': 'readonly',
+              'basename': 'nvmept',
               }
 
     _, failed, _ = run_fio_tests(TEST_LIST, test_env, args)

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-08-01 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-08-01 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 824912be19542f94264e485a25d37b55a9f68f0e:

  Revert "correctly free thread_data options at the topmost parent process" (2023-07-28 11:32:22 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 1660df6601e24a17dda9e12cbc901337fd5fd925:

  Merge branch 'master' of https://github.com/min22/fio (2023-07-31 15:03:37 -0600)

----------------------------------------------------------------
Denis Pronin (1):
      use 'const' where it is required

Jens Axboe (2):
      Merge branch 'improment/constness' of https://github.com/dpronin/fio
      Merge branch 'master' of https://github.com/min22/fio

Kookoo Gu (1):
      iolog.c: fix inaccurate clat when replay trace

 client.c | 10 +++++-----
 client.h |  8 ++++----
 iolog.c  | 14 +++++++-------
 3 files changed, 16 insertions(+), 16 deletions(-)

---

Diff of recent changes:

diff --git a/client.c b/client.c
index 7cd2ba66..c257036b 100644
--- a/client.c
+++ b/client.c
@@ -34,7 +34,7 @@ static void handle_start(struct fio_client *client, struct fio_net_cmd *cmd);
 static void convert_text(struct fio_net_cmd *cmd);
 static void client_display_thread_status(struct jobs_eta *je);
 
-struct client_ops fio_client_ops = {
+struct client_ops const fio_client_ops = {
 	.text		= handle_text,
 	.disk_util	= handle_du,
 	.thread_status	= handle_ts,
@@ -446,7 +446,7 @@ int fio_client_add_ini_file(void *cookie, const char *ini_file, bool remote)
 	return 0;
 }
 
-int fio_client_add(struct client_ops *ops, const char *hostname, void **cookie)
+int fio_client_add(struct client_ops const *ops, const char *hostname, void **cookie)
 {
 	struct fio_client *existing = *cookie;
 	struct fio_client *client;
@@ -1772,7 +1772,7 @@ fail:
 
 int fio_handle_client(struct fio_client *client)
 {
-	struct client_ops *ops = client->ops;
+	struct client_ops const *ops = client->ops;
 	struct fio_net_cmd *cmd;
 
 	dprint(FD_NET, "client: handle %s\n", client->hostname);
@@ -1957,7 +1957,7 @@ int fio_clients_send_trigger(const char *cmd)
 	return 0;
 }
 
-static void request_client_etas(struct client_ops *ops)
+static void request_client_etas(struct client_ops const *ops)
 {
 	struct fio_client *client;
 	struct flist_head *entry;
@@ -2089,7 +2089,7 @@ static int fio_check_clients_timed_out(void)
 	return ret;
 }
 
-int fio_handle_clients(struct client_ops *ops)
+int fio_handle_clients(struct client_ops const *ops)
 {
 	struct pollfd *pfds;
 	int i, ret = 0, retval = 0;
diff --git a/client.h b/client.h
index 8033325e..d77b6076 100644
--- a/client.h
+++ b/client.h
@@ -69,7 +69,7 @@ struct fio_client {
 	uint16_t argc;
 	char **argv;
 
-	struct client_ops *ops;
+	struct client_ops const *ops;
 	void *client_data;
 
 	struct client_file *files;
@@ -84,7 +84,7 @@ typedef void (client_eta_op)(struct jobs_eta *je);
 typedef void (client_timed_out_op)(struct fio_client *);
 typedef void (client_jobs_eta_op)(struct fio_client *client, struct jobs_eta *je);
 
-extern struct client_ops fio_client_ops;
+extern struct client_ops const fio_client_ops;
 
 struct client_ops {
 	client_cmd_op		*text;
@@ -128,8 +128,8 @@ extern int fio_start_client(struct fio_client *);
 extern int fio_start_all_clients(void);
 extern int fio_clients_send_ini(const char *);
 extern int fio_client_send_ini(struct fio_client *, const char *, bool);
-extern int fio_handle_clients(struct client_ops *);
-extern int fio_client_add(struct client_ops *, const char *, void **);
+extern int fio_handle_clients(struct client_ops const*);
+extern int fio_client_add(struct client_ops const*, const char *, void **);
 extern struct fio_client *fio_client_add_explicit(struct client_ops *, const char *, int, int);
 extern void fio_client_add_cmd_option(void *, const char *);
 extern int fio_client_add_ini_file(void *, const char *, bool);
diff --git a/iolog.c b/iolog.c
index cc2cbc65..97ba4396 100644
--- a/iolog.c
+++ b/iolog.c
@@ -82,8 +82,8 @@ static void iolog_delay(struct thread_data *td, unsigned long delay)
 {
 	uint64_t usec = utime_since_now(&td->last_issue);
 	unsigned long orig_delay = delay;
-	uint64_t this_delay;
 	struct timespec ts;
+	int ret = 0;
 
 	if (delay < td->time_offset) {
 		td->time_offset = 0;
@@ -97,13 +97,13 @@ static void iolog_delay(struct thread_data *td, unsigned long delay)
 	delay -= usec;
 
 	fio_gettime(&ts, NULL);
-	while (delay && !td->terminate) {
-		this_delay = delay;
-		if (this_delay > 500000)
-			this_delay = 500000;
 
-		usec_sleep(td, this_delay);
-		delay -= this_delay;
+	while (delay && !td->terminate) {
+		ret = io_u_queued_complete(td, 0);
+		if (ret < 0)
+			td_verror(td, -ret, "io_u_queued_complete");
+		if (utime_since_now(&ts) > delay)
+			break;
 	}
 
 	usec = utime_since_now(&ts);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-29 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-29 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 45eb1cf5ce883ae3b170f102db38204616c8e4b1:

  Merge branch 'helper_thread-fix-missing-stdbool-header' of https://github.com/dpronin/fio (2023-07-27 13:48:26 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 824912be19542f94264e485a25d37b55a9f68f0e:

  Revert "correctly free thread_data options at the topmost parent process" (2023-07-28 11:32:22 -0600)

----------------------------------------------------------------
Denis Pronin (3):
      fix missing headers in multiple files
      correctly free thread_data options at the topmost parent process
      io_uring engine: 'atomic_load_relaxed' instead of 'atomic_load_acquire'

Jens Axboe (4):
      Merge branch 'io_uring' of https://github.com/dpronin/fio
      Merge branch 'master' of https://github.com/dpronin/fio
      Merge branch 'td-eo-double-free-fix' of https://github.com/dpronin/fio
      Revert "correctly free thread_data options at the topmost parent process"

 cairo_text_helpers.c | 2 ++
 cairo_text_helpers.h | 2 ++
 engines/io_uring.c   | 4 ++--
 goptions.h           | 2 ++
 log.c                | 2 ++
 5 files changed, 10 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/cairo_text_helpers.c b/cairo_text_helpers.c
index 19fb8e03..5bdd6021 100644
--- a/cairo_text_helpers.c
+++ b/cairo_text_helpers.c
@@ -1,3 +1,5 @@
+#include "cairo_text_helpers.h"
+
 #include <cairo.h>
 #include <gtk/gtk.h>
 #include <math.h>
diff --git a/cairo_text_helpers.h b/cairo_text_helpers.h
index 014001ad..d0f52d51 100644
--- a/cairo_text_helpers.h
+++ b/cairo_text_helpers.h
@@ -1,6 +1,8 @@
 #ifndef CAIRO_TEXT_HELPERS_H
 #define CAIRO_TEXT_HELPERS_H
 
+#include <cairo.h>
+
 void draw_centered_text(cairo_t *cr, const char *font, double x, double y,
 			       double fontsize, const char *text);
 
diff --git a/engines/io_uring.c b/engines/io_uring.c
index e1abf688..b361e6a5 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -509,7 +509,7 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 
 	tail = *ring->tail;
 	next_tail = tail + 1;
-	if (next_tail == atomic_load_acquire(ring->head))
+	if (next_tail == atomic_load_relaxed(ring->head))
 		return FIO_Q_BUSY;
 
 	if (ld->cmdprio.mode != CMDPRIO_MODE_NONE)
@@ -569,7 +569,7 @@ static int fio_ioring_commit(struct thread_data *td)
 		unsigned start = *ld->sq_ring.tail - ld->queued;
 		unsigned flags;
 
-		flags = atomic_load_acquire(ring->flags);
+		flags = atomic_load_relaxed(ring->flags);
 		if (flags & IORING_SQ_NEED_WAKEUP)
 			io_uring_enter(ld, ld->queued, 0,
 					IORING_ENTER_SQ_WAKEUP);
diff --git a/goptions.h b/goptions.h
index a225a8d1..03617509 100644
--- a/goptions.h
+++ b/goptions.h
@@ -1,6 +1,8 @@
 #ifndef GFIO_OPTIONS_H
 #define GFIO_OPTIONS_H
 
+#include <gtk/gtk.h>
+
 void gopt_get_options_window(GtkWidget *window, struct gfio_client *gc);
 void gopt_init(void);
 void gopt_exit(void);
diff --git a/log.c b/log.c
index 237bac28..df58ea07 100644
--- a/log.c
+++ b/log.c
@@ -1,3 +1,5 @@
+#include "log.h"
+
 #include <unistd.h>
 #include <string.h>
 #include <stdarg.h>

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-28 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-28 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 0b47b2cf3dab1d26d72f52ed8c19f782a8277d3a:

  Merge branch 'prio-hints' (2023-07-21 15:23:40 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 45eb1cf5ce883ae3b170f102db38204616c8e4b1:

  Merge branch 'helper_thread-fix-missing-stdbool-header' of https://github.com/dpronin/fio (2023-07-27 13:48:26 -0600)

----------------------------------------------------------------
Denis Pronin (3):
      diskutil.h: fix missing headers wanted by the header
      helper_thread.h: include missing stdbool.h because 'bool' type is used
      helper_thread.h: forwardly declare structures fio_sem and sk_out

Jens Axboe (2):
      Merge branch 'diskutil-fix-missing-headers' of https://github.com/dpronin/fio
      Merge branch 'helper_thread-fix-missing-stdbool-header' of https://github.com/dpronin/fio

 diskutil.h      | 3 +++
 helper_thread.h | 5 +++++
 2 files changed, 8 insertions(+)

---

Diff of recent changes:

diff --git a/diskutil.h b/diskutil.h
index 9dca42c4..9b283799 100644
--- a/diskutil.h
+++ b/diskutil.h
@@ -2,10 +2,13 @@
 #define FIO_DISKUTIL_H
 #define FIO_DU_NAME_SZ		64
 
+#include <stdint.h>
 #include <limits.h>
 
 #include "helper_thread.h"
 #include "fio_sem.h"
+#include "flist.h"
+#include "lib/ieee754.h"
 
 /**
  * @ios: Number of I/O operations that have been completed successfully.
diff --git a/helper_thread.h b/helper_thread.h
index d7df6c4d..1c8167e8 100644
--- a/helper_thread.h
+++ b/helper_thread.h
@@ -1,6 +1,11 @@
 #ifndef FIO_HELPER_THREAD_H
 #define FIO_HELPER_THREAD_H
 
+#include <stdbool.h>
+
+struct fio_sem;
+struct sk_out;
+
 extern void helper_reset(void);
 extern void helper_do_stat(void);
 extern bool helper_should_exit(void);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-22 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-22 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit caf7ac7ef000097765b1c56404adb5e68b227977:

  t/zbd: add max_active configs to run-tests-against-nullb (2023-07-20 09:52:37 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 0b47b2cf3dab1d26d72f52ed8c19f782a8277d3a:

  Merge branch 'prio-hints' (2023-07-21 15:23:40 -0600)

----------------------------------------------------------------
Damien Le Moal (6):
      os-linux: Cleanup IO priority class and value macros
      cmdprio: Introduce generic option definitions
      os-linux: add initial support for IO priority hints
      options: add priohint option
      cmdprio: Add support for per I/O priority hint
      stats: Add hint information to per priority level stats

Jens Axboe (1):
      Merge branch 'prio-hints'

Shin'ichiro Kawasaki (1):
      backend: clear IO_U_F_FLIGHT flag in zero byte read path

 HOWTO.rst          |  37 +++++++++++++++++--
 backend.c          |  11 ++++--
 cconv.c            |   2 +
 engines/cmdprio.c  |   9 +++--
 engines/cmdprio.h  | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 engines/io_uring.c |  86 ++-----------------------------------------
 engines/libaio.c   |  82 +----------------------------------------
 fio.1              |  33 +++++++++++++++--
 options.c          |  31 ++++++++++++++--
 os/os-dragonfly.h  |   4 +-
 os/os-linux.h      |  27 ++++++++++----
 os/os.h            |   7 +++-
 server.h           |   2 +-
 stat.c             |  10 +++--
 thread_options.h   |   3 +-
 15 files changed, 252 insertions(+), 198 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 7fe70fbd..ac8314f3 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2287,6 +2287,16 @@ with the caveat that when used on the command line, they must come after the
 	reads and writes. See :manpage:`ionice(1)`. See also the
 	:option:`prioclass` option.
 
+.. option:: cmdprio_hint=int[,int] : [io_uring] [libaio]
+
+	Set the I/O priority hint to use for I/Os that must be issued with
+	a priority when :option:`cmdprio_percentage` or
+	:option:`cmdprio_bssplit` is set. If not specified when
+	:option:`cmdprio_percentage` or :option:`cmdprio_bssplit` is set,
+	this defaults to 0 (no hint). A single value applies to reads and
+	writes. Comma-separated values may be specified for reads and writes.
+	See also the :option:`priohint` option.
+
 .. option:: cmdprio=int[,int] : [io_uring] [libaio]
 
 	Set the I/O priority value to use for I/Os that must be issued with
@@ -2313,9 +2323,9 @@ with the caveat that when used on the command line, they must come after the
 
 		cmdprio_bssplit=blocksize/percentage:blocksize/percentage
 
-	In this case, each entry will use the priority class and priority
-	level defined by the options :option:`cmdprio_class` and
-	:option:`cmdprio` respectively.
+	In this case, each entry will use the priority class, priority hint
+	and priority level defined by the options :option:`cmdprio_class`,
+        :option:`cmdprio` and :option:`cmdprio_hint` respectively.
 
 	The second accepted format for this option is:
 
@@ -2326,7 +2336,14 @@ with the caveat that when used on the command line, they must come after the
 	accepted format does not restrict all entries to have the same priority
 	class and priority level.
 
-	For both formats, only the read and write data directions are supported,
+	The third accepted format for this option is:
+
+		cmdprio_bssplit=blocksize/percentage/class/level/hint:...
+
+	This is an extension of the second accepted format that allows to also
+	specify a priority hint.
+
+	For all formats, only the read and write data directions are supported,
 	values for trim IOs are ignored. This option is mutually exclusive with
 	the :option:`cmdprio_percentage` option.
 
@@ -3436,6 +3453,18 @@ Threads, processes and job synchronization
 	priority setting, see I/O engine specific :option:`cmdprio_percentage`
 	and :option:`cmdprio_class` options.
 
+.. option:: priohint=int
+
+	Set the I/O priority hint. This is only applicable to platforms that
+	support I/O priority classes and to devices with features controlled
+	through priority hints, e.g. block devices supporting command duration
+	limits, or CDL. CDL is a way to indicate the desired maximum latency
+	of I/Os so that the device can optimize its internal command scheduling
+	according to the latency limits indicated by the user.
+
+	For per-I/O priority hint setting, see the I/O engine specific
+	:option:`cmdprio_hint` option.
+
 .. option:: cpus_allowed=str
 
 	Controls the same options as :option:`cpumask`, but accepts a textual
diff --git a/backend.c b/backend.c
index b06a11a5..5f074039 100644
--- a/backend.c
+++ b/backend.c
@@ -466,7 +466,7 @@ int io_queue_event(struct thread_data *td, struct io_u *io_u, int *ret,
 				if (!from_verify)
 					unlog_io_piece(td, io_u);
 				td_verror(td, EIO, "full resid");
-				put_io_u(td, io_u);
+				clear_io_u(td, io_u);
 				break;
 			}
 
@@ -1799,13 +1799,16 @@ static void *thread_main(void *data)
 
 	/* ioprio_set() has to be done before td_io_init() */
 	if (fio_option_is_set(o, ioprio) ||
-	    fio_option_is_set(o, ioprio_class)) {
-		ret = ioprio_set(IOPRIO_WHO_PROCESS, 0, o->ioprio_class, o->ioprio);
+	    fio_option_is_set(o, ioprio_class) ||
+	    fio_option_is_set(o, ioprio_hint)) {
+		ret = ioprio_set(IOPRIO_WHO_PROCESS, 0, o->ioprio_class,
+				 o->ioprio, o->ioprio_hint);
 		if (ret == -1) {
 			td_verror(td, errno, "ioprio_set");
 			goto err;
 		}
-		td->ioprio = ioprio_value(o->ioprio_class, o->ioprio);
+		td->ioprio = ioprio_value(o->ioprio_class, o->ioprio,
+					  o->ioprio_hint);
 		td->ts.ioprio = td->ioprio;
 	}
 
diff --git a/cconv.c b/cconv.c
index 1bfa770f..ce6acbe6 100644
--- a/cconv.c
+++ b/cconv.c
@@ -281,6 +281,7 @@ int convert_thread_options_to_cpu(struct thread_options *o,
 	o->nice = le32_to_cpu(top->nice);
 	o->ioprio = le32_to_cpu(top->ioprio);
 	o->ioprio_class = le32_to_cpu(top->ioprio_class);
+	o->ioprio_hint = le32_to_cpu(top->ioprio_hint);
 	o->file_service_type = le32_to_cpu(top->file_service_type);
 	o->group_reporting = le32_to_cpu(top->group_reporting);
 	o->stats = le32_to_cpu(top->stats);
@@ -496,6 +497,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->nice = cpu_to_le32(o->nice);
 	top->ioprio = cpu_to_le32(o->ioprio);
 	top->ioprio_class = cpu_to_le32(o->ioprio_class);
+	top->ioprio_hint = cpu_to_le32(o->ioprio_hint);
 	top->file_service_type = cpu_to_le32(o->file_service_type);
 	top->group_reporting = cpu_to_le32(o->group_reporting);
 	top->stats = cpu_to_le32(o->stats);
diff --git a/engines/cmdprio.c b/engines/cmdprio.c
index 979a81b6..153e3691 100644
--- a/engines/cmdprio.c
+++ b/engines/cmdprio.c
@@ -267,7 +267,8 @@ static int fio_cmdprio_percentage(struct cmdprio *cmdprio, struct io_u *io_u,
  * to be set. If the random percentage value is within the user specified
  * percentage of I/Os that should use a cmdprio priority value (rather than
  * the default priority), then this function updates the io_u with an ioprio
- * value as defined by the cmdprio/cmdprio_class or cmdprio_bssplit options.
+ * value as defined by the cmdprio/cmdprio_hint/cmdprio_class or
+ * cmdprio_bssplit options.
  *
  * Return true if the io_u ioprio was changed and false otherwise.
  */
@@ -342,7 +343,8 @@ static int fio_cmdprio_gen_perc(struct thread_data *td, struct cmdprio *cmdprio)
 		prio = &cmdprio->perc_entry[ddir];
 		prio->perc = options->percentage[ddir];
 		prio->prio = ioprio_value(options->class[ddir],
-					  options->level[ddir]);
+					  options->level[ddir],
+					  options->hint[ddir]);
 		assign_clat_prio_index(prio, &values[ddir]);
 
 		ret = init_ts_clat_prio(ts, ddir, &values[ddir]);
@@ -400,7 +402,8 @@ static int fio_cmdprio_parse_and_gen_bssplit(struct thread_data *td,
 			goto err;
 
 		implicit_cmdprio = ioprio_value(options->class[ddir],
-						options->level[ddir]);
+						options->level[ddir],
+						options->hint[ddir]);
 
 		ret = fio_cmdprio_generate_bsprio_desc(&cmdprio->bsprio_desc[ddir],
 						       &parse_res[ddir],
diff --git a/engines/cmdprio.h b/engines/cmdprio.h
index 755da8d0..81e6c390 100644
--- a/engines/cmdprio.h
+++ b/engines/cmdprio.h
@@ -7,6 +7,7 @@
 #define FIO_CMDPRIO_H
 
 #include "../fio.h"
+#include "../optgroup.h"
 
 /* read and writes only, no trim */
 #define CMDPRIO_RWDIR_CNT 2
@@ -39,9 +40,114 @@ struct cmdprio_options {
 	unsigned int percentage[CMDPRIO_RWDIR_CNT];
 	unsigned int class[CMDPRIO_RWDIR_CNT];
 	unsigned int level[CMDPRIO_RWDIR_CNT];
+	unsigned int hint[CMDPRIO_RWDIR_CNT];
 	char *bssplit_str;
 };
 
+#ifdef FIO_HAVE_IOPRIO_CLASS
+#define CMDPRIO_OPTIONS(opt_struct, opt_group)					\
+	{									\
+		.name	= "cmdprio_percentage",					\
+		.lname	= "high priority percentage",				\
+		.type	= FIO_OPT_INT,						\
+		.off1	= offsetof(opt_struct,					\
+				   cmdprio_options.percentage[DDIR_READ]),	\
+		.off2	= offsetof(opt_struct,					\
+				   cmdprio_options.percentage[DDIR_WRITE]),	\
+		.minval	= 0,							\
+		.maxval	= 100,							\
+		.help	= "Send high priority I/O this percentage of the time",	\
+		.category = FIO_OPT_C_ENGINE,					\
+		.group	= opt_group,						\
+	},									\
+	{									\
+		.name	= "cmdprio_class",					\
+		.lname	= "Asynchronous I/O priority class",			\
+		.type	= FIO_OPT_INT,						\
+		.off1	= offsetof(opt_struct,					\
+				   cmdprio_options.class[DDIR_READ]),		\
+		.off2	= offsetof(opt_struct,					\
+				   cmdprio_options.class[DDIR_WRITE]),		\
+		.help	= "Set asynchronous IO priority class",			\
+		.minval	= IOPRIO_MIN_PRIO_CLASS + 1,				\
+		.maxval	= IOPRIO_MAX_PRIO_CLASS,				\
+		.interval = 1,							\
+		.category = FIO_OPT_C_ENGINE,					\
+		.group	= opt_group,						\
+	},									\
+	{									\
+		.name	= "cmdprio_hint",					\
+		.lname	= "Asynchronous I/O priority hint",			\
+		.type	= FIO_OPT_INT,						\
+		.off1	= offsetof(opt_struct,					\
+				   cmdprio_options.hint[DDIR_READ]),		\
+		.off2	= offsetof(opt_struct,					\
+				   cmdprio_options.hint[DDIR_WRITE]),		\
+		.help	= "Set asynchronous IO priority hint",			\
+		.minval	= IOPRIO_MIN_PRIO_HINT,					\
+		.maxval	= IOPRIO_MAX_PRIO_HINT,					\
+		.interval = 1,							\
+		.category = FIO_OPT_C_ENGINE,					\
+		.group	= opt_group,						\
+	},									\
+	{									\
+		.name	= "cmdprio",						\
+		.lname	= "Asynchronous I/O priority level",			\
+		.type	= FIO_OPT_INT,						\
+		.off1	= offsetof(opt_struct,					\
+				   cmdprio_options.level[DDIR_READ]),		\
+		.off2	= offsetof(opt_struct,					\
+				   cmdprio_options.level[DDIR_WRITE]),		\
+		.help	= "Set asynchronous IO priority level",			\
+		.minval	= IOPRIO_MIN_PRIO,					\
+		.maxval	= IOPRIO_MAX_PRIO,					\
+		.interval = 1,							\
+		.category = FIO_OPT_C_ENGINE,					\
+		.group	= opt_group,						\
+	},									\
+	{									\
+		.name   = "cmdprio_bssplit",					\
+		.lname  = "Priority percentage block size split",		\
+		.type   = FIO_OPT_STR_STORE,					\
+		.off1   = offsetof(opt_struct, cmdprio_options.bssplit_str),	\
+		.help   = "Set priority percentages for different block sizes",	\
+		.category = FIO_OPT_C_ENGINE,					\
+		.group	= opt_group,						\
+	}
+#else
+#define CMDPRIO_OPTIONS(opt_struct, opt_group)					\
+	{									\
+		.name	= "cmdprio_percentage",					\
+		.lname	= "high priority percentage",				\
+		.type	= FIO_OPT_UNSUPPORTED,					\
+		.help	= "Platform does not support I/O priority classes",	\
+	},									\
+	{									\
+		.name	= "cmdprio_class",					\
+		.lname	= "Asynchronous I/O priority class",			\
+		.type	= FIO_OPT_UNSUPPORTED,					\
+		.help	= "Platform does not support I/O priority classes",	\
+	},									\
+	{									\
+		.name	= "cmdprio_hint",					\
+		.lname	= "Asynchronous I/O priority hint",			\
+		.type	= FIO_OPT_UNSUPPORTED,					\
+		.help	= "Platform does not support I/O priority classes",	\
+	},									\
+	{									\
+		.name	= "cmdprio",						\
+		.lname	= "Asynchronous I/O priority level",			\
+		.type	= FIO_OPT_UNSUPPORTED,					\
+		.help	= "Platform does not support I/O priority classes",	\
+	},									\
+	{									\
+		.name   = "cmdprio_bssplit",					\
+		.lname  = "Priority percentage block size split",		\
+		.type	= FIO_OPT_UNSUPPORTED,					\
+		.help	= "Platform does not support I/O priority classes",	\
+	}
+#endif
+
 struct cmdprio {
 	struct cmdprio_options *options;
 	struct cmdprio_prio perc_entry[CMDPRIO_RWDIR_CNT];
diff --git a/engines/io_uring.c b/engines/io_uring.c
index f30a3c00..e1abf688 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -127,87 +127,6 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group	= FIO_OPT_G_IOURING,
 	},
-#ifdef FIO_HAVE_IOPRIO_CLASS
-	{
-		.name	= "cmdprio_percentage",
-		.lname	= "high priority percentage",
-		.type	= FIO_OPT_INT,
-		.off1	= offsetof(struct ioring_options,
-				   cmdprio_options.percentage[DDIR_READ]),
-		.off2	= offsetof(struct ioring_options,
-				   cmdprio_options.percentage[DDIR_WRITE]),
-		.minval	= 0,
-		.maxval	= 100,
-		.help	= "Send high priority I/O this percentage of the time",
-		.category = FIO_OPT_C_ENGINE,
-		.group	= FIO_OPT_G_IOURING,
-	},
-	{
-		.name	= "cmdprio_class",
-		.lname	= "Asynchronous I/O priority class",
-		.type	= FIO_OPT_INT,
-		.off1	= offsetof(struct ioring_options,
-				   cmdprio_options.class[DDIR_READ]),
-		.off2	= offsetof(struct ioring_options,
-				   cmdprio_options.class[DDIR_WRITE]),
-		.help	= "Set asynchronous IO priority class",
-		.minval	= IOPRIO_MIN_PRIO_CLASS + 1,
-		.maxval	= IOPRIO_MAX_PRIO_CLASS,
-		.interval = 1,
-		.category = FIO_OPT_C_ENGINE,
-		.group	= FIO_OPT_G_IOURING,
-	},
-	{
-		.name	= "cmdprio",
-		.lname	= "Asynchronous I/O priority level",
-		.type	= FIO_OPT_INT,
-		.off1	= offsetof(struct ioring_options,
-				   cmdprio_options.level[DDIR_READ]),
-		.off2	= offsetof(struct ioring_options,
-				   cmdprio_options.level[DDIR_WRITE]),
-		.help	= "Set asynchronous IO priority level",
-		.minval	= IOPRIO_MIN_PRIO,
-		.maxval	= IOPRIO_MAX_PRIO,
-		.interval = 1,
-		.category = FIO_OPT_C_ENGINE,
-		.group	= FIO_OPT_G_IOURING,
-	},
-	{
-		.name   = "cmdprio_bssplit",
-		.lname  = "Priority percentage block size split",
-		.type   = FIO_OPT_STR_STORE,
-		.off1   = offsetof(struct ioring_options,
-				   cmdprio_options.bssplit_str),
-		.help   = "Set priority percentages for different block sizes",
-		.category = FIO_OPT_C_ENGINE,
-		.group	= FIO_OPT_G_IOURING,
-	},
-#else
-	{
-		.name	= "cmdprio_percentage",
-		.lname	= "high priority percentage",
-		.type	= FIO_OPT_UNSUPPORTED,
-		.help	= "Your platform does not support I/O priority classes",
-	},
-	{
-		.name	= "cmdprio_class",
-		.lname	= "Asynchronous I/O priority class",
-		.type	= FIO_OPT_UNSUPPORTED,
-		.help	= "Your platform does not support I/O priority classes",
-	},
-	{
-		.name	= "cmdprio",
-		.lname	= "Asynchronous I/O priority level",
-		.type	= FIO_OPT_UNSUPPORTED,
-		.help	= "Your platform does not support I/O priority classes",
-	},
-	{
-		.name   = "cmdprio_bssplit",
-		.lname  = "Priority percentage block size split",
-		.type	= FIO_OPT_UNSUPPORTED,
-		.help	= "Your platform does not support I/O priority classes",
-	},
-#endif
 	{
 		.name	= "fixedbufs",
 		.lname	= "Fixed (pre-mapped) IO buffers",
@@ -297,6 +216,7 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group	= FIO_OPT_G_IOURING,
 	},
+	CMDPRIO_OPTIONS(struct ioring_options, FIO_OPT_G_IOURING),
 	{
 		.name	= NULL,
 	},
@@ -365,8 +285,8 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
 		/*
 		 * Since io_uring can have a submission context (sqthread_poll)
 		 * that is different from the process context, we cannot rely on
-		 * the IO priority set by ioprio_set() (option prio/prioclass)
-		 * to be inherited.
+		 * the IO priority set by ioprio_set() (options prio, prioclass,
+		 * and priohint) to be inherited.
 		 * td->ioprio will have the value of the "default prio", so set
 		 * this unconditionally. This value might get overridden by
 		 * fio_ioring_cmdprio_prep() if the option cmdprio_percentage or
diff --git a/engines/libaio.c b/engines/libaio.c
index 6a0745aa..aaccc7ce 100644
--- a/engines/libaio.c
+++ b/engines/libaio.c
@@ -72,87 +72,6 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group	= FIO_OPT_G_LIBAIO,
 	},
-#ifdef FIO_HAVE_IOPRIO_CLASS
-	{
-		.name	= "cmdprio_percentage",
-		.lname	= "high priority percentage",
-		.type	= FIO_OPT_INT,
-		.off1	= offsetof(struct libaio_options,
-				   cmdprio_options.percentage[DDIR_READ]),
-		.off2	= offsetof(struct libaio_options,
-				   cmdprio_options.percentage[DDIR_WRITE]),
-		.minval	= 0,
-		.maxval	= 100,
-		.help	= "Send high priority I/O this percentage of the time",
-		.category = FIO_OPT_C_ENGINE,
-		.group	= FIO_OPT_G_LIBAIO,
-	},
-	{
-		.name	= "cmdprio_class",
-		.lname	= "Asynchronous I/O priority class",
-		.type	= FIO_OPT_INT,
-		.off1	= offsetof(struct libaio_options,
-				   cmdprio_options.class[DDIR_READ]),
-		.off2	= offsetof(struct libaio_options,
-				   cmdprio_options.class[DDIR_WRITE]),
-		.help	= "Set asynchronous IO priority class",
-		.minval	= IOPRIO_MIN_PRIO_CLASS + 1,
-		.maxval	= IOPRIO_MAX_PRIO_CLASS,
-		.interval = 1,
-		.category = FIO_OPT_C_ENGINE,
-		.group	= FIO_OPT_G_LIBAIO,
-	},
-	{
-		.name	= "cmdprio",
-		.lname	= "Asynchronous I/O priority level",
-		.type	= FIO_OPT_INT,
-		.off1	= offsetof(struct libaio_options,
-				   cmdprio_options.level[DDIR_READ]),
-		.off2	= offsetof(struct libaio_options,
-				   cmdprio_options.level[DDIR_WRITE]),
-		.help	= "Set asynchronous IO priority level",
-		.minval	= IOPRIO_MIN_PRIO,
-		.maxval	= IOPRIO_MAX_PRIO,
-		.interval = 1,
-		.category = FIO_OPT_C_ENGINE,
-		.group	= FIO_OPT_G_LIBAIO,
-	},
-	{
-		.name   = "cmdprio_bssplit",
-		.lname  = "Priority percentage block size split",
-		.type   = FIO_OPT_STR_STORE,
-		.off1   = offsetof(struct libaio_options,
-				   cmdprio_options.bssplit_str),
-		.help   = "Set priority percentages for different block sizes",
-		.category = FIO_OPT_C_ENGINE,
-		.group	= FIO_OPT_G_LIBAIO,
-	},
-#else
-	{
-		.name	= "cmdprio_percentage",
-		.lname	= "high priority percentage",
-		.type	= FIO_OPT_UNSUPPORTED,
-		.help	= "Your platform does not support I/O priority classes",
-	},
-	{
-		.name	= "cmdprio_class",
-		.lname	= "Asynchronous I/O priority class",
-		.type	= FIO_OPT_UNSUPPORTED,
-		.help	= "Your platform does not support I/O priority classes",
-	},
-	{
-		.name	= "cmdprio",
-		.lname	= "Asynchronous I/O priority level",
-		.type	= FIO_OPT_UNSUPPORTED,
-		.help	= "Your platform does not support I/O priority classes",
-	},
-	{
-		.name   = "cmdprio_bssplit",
-		.lname  = "Priority percentage block size split",
-		.type	= FIO_OPT_UNSUPPORTED,
-		.help	= "Your platform does not support I/O priority classes",
-	},
-#endif
 	{
 		.name	= "nowait",
 		.lname	= "RWF_NOWAIT",
@@ -162,6 +81,7 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group	= FIO_OPT_G_LIBAIO,
 	},
+	CMDPRIO_OPTIONS(struct libaio_options, FIO_OPT_G_LIBAIO),
 	{
 		.name	= NULL,
 	},
diff --git a/fio.1 b/fio.1
index 20acd081..f62617e7 100644
--- a/fio.1
+++ b/fio.1
@@ -2084,6 +2084,14 @@ is set, this defaults to the highest priority class. A single value applies
 to reads and writes. Comma-separated values may be specified for reads and
 writes. See man \fBionice\fR\|(1). See also the \fBprioclass\fR option.
 .TP
+.BI (io_uring,libaio)cmdprio_hint \fR=\fPint[,int]
+Set the I/O priority hint to use for I/Os that must be issued with a
+priority when \fBcmdprio_percentage\fR or \fBcmdprio_bssplit\fR is set.
+If not specified when \fBcmdprio_percentage\fR or \fBcmdprio_bssplit\fR
+is set, this defaults to 0 (no hint). A single value applies to reads and
+writes. Comma-separated values may be specified for reads and writes.
+See also the \fBpriohint\fR option.
+.TP
 .BI (io_uring,libaio)cmdprio \fR=\fPint[,int]
 Set the I/O priority value to use for I/Os that must be issued with a
 priority when \fBcmdprio_percentage\fR or \fBcmdprio_bssplit\fR is set.
@@ -2109,8 +2117,9 @@ The first accepted format for this option is the same as the format of the
 cmdprio_bssplit=blocksize/percentage:blocksize/percentage
 .RE
 .P
-In this case, each entry will use the priority class and priority level defined
-by the options \fBcmdprio_class\fR and \fBcmdprio\fR respectively.
+In this case, each entry will use the priority class, priority hint and
+priority level defined by the options \fBcmdprio_class\fR, \fBcmdprio\fR
+and \fBcmdprio_hint\fR respectively.
 .P
 The second accepted format for this option is:
 .RS
@@ -2123,7 +2132,16 @@ entry. In comparison with the first accepted format, the second accepted format
 does not restrict all entries to have the same priority class and priority
 level.
 .P
-For both formats, only the read and write data directions are supported, values
+The third accepted format for this option is:
+.RS
+.P
+cmdprio_bssplit=blocksize/percentage/class/level/hint:...
+.RE
+.P
+This is an extension of the second accepted format that allows to also
+specify a priority hint.
+.P
+For all formats, only the read and write data directions are supported, values
 for trim IOs are ignored. This option is mutually exclusive with the
 \fBcmdprio_percentage\fR option.
 .RE
@@ -3144,6 +3162,15 @@ Set the I/O priority class. See man \fBionice\fR\|(1). For per-command
 priority setting, see the I/O engine specific `cmdprio_percentage` and
 `cmdprio_class` options.
 .TP
+.BI priohint \fR=\fPint
+Set the I/O priority hint. This is only applicable to platforms that support
+I/O priority classes and to devices with features controlled through priority
+hints, e.g. block devices supporting command duration limits, or CDL. CDL is a
+way to indicate the desired maximum latency of I/Os so that the device can
+optimize its internal command scheduling according to the latency limits
+indicated by the user. For per-I/O priority hint setting, see the I/O engine
+specific \fBcmdprio_hint\fB option.
+.TP
 .BI cpus_allowed \fR=\fPstr
 Controls the same options as \fBcpumask\fR, but accepts a textual
 specification of the permitted CPUs instead and CPUs are indexed from 0. So
diff --git a/options.c b/options.c
index 0f739317..48aa0d7b 100644
--- a/options.c
+++ b/options.c
@@ -313,15 +313,17 @@ static int parse_cmdprio_bssplit_entry(struct thread_options *o,
 	int matches = 0;
 	char *bs_str = NULL;
 	long long bs_val;
-	unsigned int perc = 0, class, level;
+	unsigned int perc = 0, class, level, hint;
 
 	/*
 	 * valid entry formats:
 	 * bs/ - %s/ - set perc to 0, prio to -1.
 	 * bs/perc - %s/%u - set prio to -1.
 	 * bs/perc/class/level - %s/%u/%u/%u
+	 * bs/perc/class/level/hint - %s/%u/%u/%u/%u
 	 */
-	matches = sscanf(str, "%m[^/]/%u/%u/%u", &bs_str, &perc, &class, &level);
+	matches = sscanf(str, "%m[^/]/%u/%u/%u/%u",
+			 &bs_str, &perc, &class, &level, &hint);
 	if (matches < 1) {
 		log_err("fio: invalid cmdprio_bssplit format\n");
 		return 1;
@@ -342,9 +344,14 @@ static int parse_cmdprio_bssplit_entry(struct thread_options *o,
 	case 2: /* bs/perc case */
 		break;
 	case 4: /* bs/perc/class/level case */
+	case 5: /* bs/perc/class/level/hint case */
 		class = min(class, (unsigned int) IOPRIO_MAX_PRIO_CLASS);
 		level = min(level, (unsigned int) IOPRIO_MAX_PRIO);
-		entry->prio = ioprio_value(class, level);
+		if (matches == 5)
+			hint = min(hint, (unsigned int) IOPRIO_MAX_PRIO_HINT);
+		else
+			hint = 0;
+		entry->prio = ioprio_value(class, level, hint);
 		break;
 	default:
 		log_err("fio: invalid cmdprio_bssplit format\n");
@@ -3806,6 +3813,18 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.category = FIO_OPT_C_GENERAL,
 		.group	= FIO_OPT_G_CRED,
 	},
+	{
+		.name	= "priohint",
+		.lname	= "I/O nice priority hint",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct thread_options, ioprio_hint),
+		.help	= "Set job IO priority hint",
+		.minval	= IOPRIO_MIN_PRIO_HINT,
+		.maxval	= IOPRIO_MAX_PRIO_HINT,
+		.interval = 1,
+		.category = FIO_OPT_C_GENERAL,
+		.group	= FIO_OPT_G_CRED,
+	},
 #else
 	{
 		.name	= "prioclass",
@@ -3813,6 +3832,12 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.type	= FIO_OPT_UNSUPPORTED,
 		.help	= "Your platform does not support IO priority classes",
 	},
+	{
+		.name	= "priohint",
+		.lname	= "I/O nice priority hint",
+		.type	= FIO_OPT_UNSUPPORTED,
+		.help	= "Your platform does not support IO priority hints",
+	},
 #endif
 	{
 		.name	= "thinktime",
diff --git a/os/os-dragonfly.h b/os/os-dragonfly.h
index bde39101..4ce72539 100644
--- a/os/os-dragonfly.h
+++ b/os/os-dragonfly.h
@@ -171,8 +171,8 @@ static inline int fio_getaffinity(int pid, os_cpu_mask_t *mask)
  * ioprio_set() with 4 arguments, so define fio's ioprio_set() as a macro.
  * Note that there is no idea of class within ioprio_set(2) unlike Linux.
  */
-#define ioprio_value(ioprio_class, ioprio)	(ioprio)
-#define ioprio_set(which, who, ioprio_class, ioprio)	\
+#define ioprio_value(ioprio_class, ioprio, ioprio_hint)	(ioprio)
+#define ioprio_set(which, who, ioprio_class, ioprio, ioprio_hint)	\
 	ioprio_set(which, who, ioprio)
 
 #define ioprio(ioprio)		(ioprio)
diff --git a/os/os-linux.h b/os/os-linux.h
index 2f9f7e79..c5cd6515 100644
--- a/os/os-linux.h
+++ b/os/os-linux.h
@@ -125,13 +125,24 @@ enum {
 #define IOPRIO_BITS		16
 #define IOPRIO_CLASS_SHIFT	13
 
+#define IOPRIO_HINT_BITS	10
+#define IOPRIO_HINT_SHIFT	3
+
 #define IOPRIO_MIN_PRIO		0	/* highest priority */
 #define IOPRIO_MAX_PRIO		7	/* lowest priority */
 
 #define IOPRIO_MIN_PRIO_CLASS	0
 #define IOPRIO_MAX_PRIO_CLASS	3
 
-static inline int ioprio_value(int ioprio_class, int ioprio)
+#define IOPRIO_MIN_PRIO_HINT	0
+#define IOPRIO_MAX_PRIO_HINT	((1 << IOPRIO_HINT_BITS) - 1)
+
+#define ioprio_class(ioprio)	((ioprio) >> IOPRIO_CLASS_SHIFT)
+#define ioprio(ioprio)		((ioprio) & IOPRIO_MAX_PRIO)
+#define ioprio_hint(ioprio)	\
+	(((ioprio) >> IOPRIO_HINT_SHIFT) & IOPRIO_MAX_PRIO_HINT)
+
+static inline int ioprio_value(int ioprio_class, int ioprio, int ioprio_hint)
 {
 	/*
 	 * If no class is set, assume BE
@@ -139,23 +150,23 @@ static inline int ioprio_value(int ioprio_class, int ioprio)
         if (!ioprio_class)
                 ioprio_class = IOPRIO_CLASS_BE;
 
-	return (ioprio_class << IOPRIO_CLASS_SHIFT) | ioprio;
+	return (ioprio_class << IOPRIO_CLASS_SHIFT) |
+		(ioprio_hint << IOPRIO_HINT_SHIFT) |
+		ioprio;
 }
 
 static inline bool ioprio_value_is_class_rt(unsigned int priority)
 {
-	return (priority >> IOPRIO_CLASS_SHIFT) == IOPRIO_CLASS_RT;
+	return ioprio_class(priority) == IOPRIO_CLASS_RT;
 }
 
-static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio)
+static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio,
+			     int ioprio_hint)
 {
 	return syscall(__NR_ioprio_set, which, who,
-		       ioprio_value(ioprio_class, ioprio));
+		       ioprio_value(ioprio_class, ioprio, ioprio_hint));
 }
 
-#define ioprio_class(ioprio)	((ioprio) >> IOPRIO_CLASS_SHIFT)
-#define ioprio(ioprio)		((ioprio) & 7)
-
 #ifndef CONFIG_HAVE_GETTID
 static inline int gettid(void)
 {
diff --git a/os/os.h b/os/os.h
index 036fc233..0f182324 100644
--- a/os/os.h
+++ b/os/os.h
@@ -120,11 +120,14 @@ extern int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu);
 #define ioprio_value_is_class_rt(prio)	(false)
 #define IOPRIO_MIN_PRIO_CLASS		0
 #define IOPRIO_MAX_PRIO_CLASS		0
+#define ioprio_hint(prio)		0
+#define IOPRIO_MIN_PRIO_HINT		0
+#define IOPRIO_MAX_PRIO_HINT		0
 #endif
 #ifndef FIO_HAVE_IOPRIO
-#define ioprio_value(prioclass, prio)	(0)
+#define ioprio_value(prioclass, prio, priohint)	(0)
 #define ioprio(ioprio)			0
-#define ioprio_set(which, who, prioclass, prio)	(0)
+#define ioprio_set(which, who, prioclass, prio, priohint) (0)
 #define IOPRIO_MIN_PRIO			0
 #define IOPRIO_MAX_PRIO			0
 #endif
diff --git a/server.h b/server.h
index 601d3340..ad706118 100644
--- a/server.h
+++ b/server.h
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 100,
+	FIO_SERVER_VER			= 101,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,
diff --git a/stat.c b/stat.c
index 7fad73d1..7b791628 100644
--- a/stat.c
+++ b/stat.c
@@ -597,10 +597,11 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
 				continue;
 
 			snprintf(buf, sizeof(buf),
-				 "%s prio %u/%u",
+				 "%s prio %u/%u/%u",
 				 clat_type,
 				 ioprio_class(ts->clat_prio[ddir][i].ioprio),
-				 ioprio(ts->clat_prio[ddir][i].ioprio));
+				 ioprio(ts->clat_prio[ddir][i].ioprio),
+				 ioprio_hint(ts->clat_prio[ddir][i].ioprio));
 			display_lat(buf, min, max, mean, dev, out);
 		}
 	}
@@ -640,10 +641,11 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
 					continue;
 
 				snprintf(prio_name, sizeof(prio_name),
-					 "%s prio %u/%u (%.2f%% of IOs)",
+					 "%s prio %u/%u/%u (%.2f%% of IOs)",
 					 clat_type,
 					 ioprio_class(ts->clat_prio[ddir][i].ioprio),
 					 ioprio(ts->clat_prio[ddir][i].ioprio),
+					 ioprio_hint(ts->clat_prio[ddir][i].ioprio),
 					 100. * (double) prio_samples / (double) samples);
 				show_clat_percentiles(ts->clat_prio[ddir][i].io_u_plat,
 						prio_samples, ts->percentile_list,
@@ -1533,6 +1535,8 @@ static void add_ddir_status_json(struct thread_stat *ts,
 				ioprio_class(ts->clat_prio[ddir][i].ioprio));
 			json_object_add_value_int(obj, "prio",
 				ioprio(ts->clat_prio[ddir][i].ioprio));
+			json_object_add_value_int(obj, "priohint",
+				ioprio_hint(ts->clat_prio[ddir][i].ioprio));
 
 			tmp_object = add_ddir_lat_json(ts,
 					ts->clat_percentiles | ts->lat_percentiles,
diff --git a/thread_options.h b/thread_options.h
index 1715b36c..38a9993d 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -248,6 +248,7 @@ struct thread_options {
 	unsigned int nice;
 	unsigned int ioprio;
 	unsigned int ioprio_class;
+	unsigned int ioprio_hint;
 	unsigned int file_service_type;
 	unsigned int group_reporting;
 	unsigned int stats;
@@ -568,6 +569,7 @@ struct thread_options_pack {
 	uint32_t nice;
 	uint32_t ioprio;
 	uint32_t ioprio_class;
+	uint32_t ioprio_hint;
 	uint32_t file_service_type;
 	uint32_t group_reporting;
 	uint32_t stats;
@@ -601,7 +603,6 @@ struct thread_options_pack {
 	uint32_t lat_percentiles;
 	uint32_t slat_percentiles;
 	uint32_t percentile_precision;
-	uint32_t pad5;
 	fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
 
 	uint8_t read_iolog_file[FIO_TOP_STR_MAX];

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-21 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-21 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 04361e9a23d6e0448fd6fbbd4e14ecdfff60e314:

  Merge branch 'patch-3' of https://github.com/yangjueji/fio (2023-07-15 09:57:43 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to caf7ac7ef000097765b1c56404adb5e68b227977:

  t/zbd: add max_active configs to run-tests-against-nullb (2023-07-20 09:52:37 -0400)

----------------------------------------------------------------
Dmitry Fomichev (2):
      t/zbd: fix null_blk configuration in run-tests-against-nullb
      t/zbd: add max_active configs to run-tests-against-nullb

Shin'ichiro Kawasaki (11):
      zbd: get max_active_zones limit value from zoned devices
      zbd: write to closed zones on the devices with max_active_zones limit
      zbd: print max_active_zones limit error message
      docs: modify max_open_zones option description
      t/zbd: add close_zone helper function
      t/zbd: add max_active_zone variable
      t/zbd: add test case to check zones in closed condition
      t/zbd: add test case to check max_active_zones limit error message
      t/zbd: get max_open_zones from sysfs
      t/zbd: fix fio failure check and SG node failure in test case 31
      t/zbd: add missing prep_write for test cases with write workloads

 HOWTO.rst                     |  44 +++++----
 fio.1                         |  36 +++++---
 io_u.c                        |   2 +
 ioengines.h                   |   4 +-
 oslib/blkzoned.h              |   9 ++
 oslib/linux-blkzoned.c        |  23 +++++
 t/zbd/functions               |  33 ++++++-
 t/zbd/run-tests-against-nullb | 203 ++++++++++++++++++++++++++++++++++++++++--
 t/zbd/test-zbd-support        |  91 ++++++++++++++++++-
 zbd.c                         |  47 +++++++++-
 zbd.h                         |   5 ++
 11 files changed, 457 insertions(+), 40 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 7ae8ea7b..7fe70fbd 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -1056,22 +1056,34 @@ Target file/device
 
 .. option:: max_open_zones=int
 
-	A zone of a zoned block device is in the open state when it is partially
-	written (i.e. not all sectors of the zone have been written). Zoned
-	block devices may have a limit on the total number of zones that can
-	be simultaneously in the open state, that is, the number of zones that
-	can be written to simultaneously. The :option:`max_open_zones` parameter
-	limits the number of zones to which write commands are issued by all fio
-	jobs, that is, limits the number of zones that will be in the open
-	state. This parameter is relevant only if the :option:`zonemode` =zbd is
-	used. The default value is always equal to maximum number of open zones
-	of the target zoned block device and a value higher than this limit
-	cannot be specified by users unless the option
-	:option:`ignore_zone_limits` is specified. When
-	:option:`ignore_zone_limits` is specified or the target device has no
-	limit on the number of zones that can be in an open state,
-	:option:`max_open_zones` can specify 0 to disable any limit on the
-	number of zones that can be simultaneously written to by all jobs.
+	When a zone of a zoned block device is partially written (i.e. not all
+	sectors of the zone have been written), the zone is in one of three
+	conditions: 'implicit open', 'explicit open' or 'closed'. Zoned block
+	devices may have a limit called 'max_open_zones' (same name as the
+	parameter) on the total number of zones that can simultaneously be in
+	the 'implicit open' or 'explicit open' conditions. Zoned block devices
+	may have another limit called 'max_active_zones', on the total number of
+	zones that can simultaneously be in the three conditions. The
+	:option:`max_open_zones` parameter limits the number of zones to which
+	write commands are issued by all fio jobs, that is, limits the number of
+	zones that will be in the conditions. When the device has the
+	max_open_zones limit and does not have the max_active_zones limit, the
+	:option:`max_open_zones` parameter limits the number of zones in the two
+	open conditions up to the limit. In this case, fio includes zones in the
+	two open conditions to the write target zones at fio start. When the
+	device has both the max_open_zones and the max_active_zones limits, the
+	:option:`max_open_zones` parameter limits the number of zones in the
+	three conditions up to the limit. In this case, fio includes zones in
+	the three conditions to the write target zones at fio start.
+
+	This parameter is relevant only if the :option:`zonemode` =zbd is used.
+	The default value is always equal to the max_open_zones limit of the
+	target zoned block device and a value higher than this limit cannot be
+	specified by users unless the option :option:`ignore_zone_limits` is
+	specified. When :option:`ignore_zone_limits` is specified or the target
+	device does not have the max_open_zones limit, :option:`max_open_zones`
+	can specify 0 to disable any limit on the number of zones that can be
+	simultaneously written to by all jobs.
 
 .. option:: job_max_open_zones=int
 
diff --git a/fio.1 b/fio.1
index da875276..20acd081 100644
--- a/fio.1
+++ b/fio.1
@@ -832,18 +832,30 @@ numbers fio only reads beyond the write pointer if explicitly told to do
 so. Default: false.
 .TP
 .BI max_open_zones \fR=\fPint
-A zone of a zoned block device is in the open state when it is partially written
-(i.e. not all sectors of the zone have been written). Zoned block devices may
-have limit a on the total number of zones that can be simultaneously in the
-open state, that is, the number of zones that can be written to simultaneously.
-The \fBmax_open_zones\fR parameter limits the number of zones to which write
-commands are issued by all fio jobs, that is, limits the number of zones that
-will be in the open state. This parameter is relevant only if the
-\fBzonemode=zbd\fR is used. The default value is always equal to maximum number
-of open zones of the target zoned block device and a value higher than this
-limit cannot be specified by users unless the option \fBignore_zone_limits\fR is
-specified. When \fBignore_zone_limits\fR is specified or the target device has
-no limit on the number of zones that can be in an open state,
+When a zone of a zoned block device is partially written (i.e. not all sectors
+of the zone have been written), the zone is in one of three
+conditions: 'implicit open', 'explicit open' or 'closed'. Zoned block devices
+may have a limit called 'max_open_zones' (same name as the parameter) on the
+total number of zones that can simultaneously be in the 'implicit open'
+or 'explicit open' conditions. Zoned block devices may have another limit
+called 'max_active_zones', on the total number of zones that can simultaneously
+be in the three conditions. The \fBmax_open_zones\fR parameter limits
+the number of zones to which write commands are issued by all fio jobs, that is,
+limits the number of zones that will be in the conditions. When the device has
+the max_open_zones limit and does not have the max_active_zones limit, the
+\fBmax_open_zones\fR parameter limits the number of zones in the two open
+conditions up to the limit. In this case, fio includes zones in the two open
+conditions to the write target zones at fio start. When the device has both the
+max_open_zones and the max_active_zones limits, the \fBmax_open_zones\fR
+parameter limits the number of zones in the three conditions up to the limit.
+In this case, fio includes zones in the three conditions to the write target
+zones at fio start.
+
+This parameter is relevant only if the \fBzonemode=zbd\fR is used. The default
+value is always equal to the max_open_zones limit of the target zoned block
+device and a value higher than this limit cannot be specified by users unless
+the option \fBignore_zone_limits\fR is specified. When \fBignore_zone_limits\fR
+is specified or the target device does not have the max_open_zones limit,
 \fBmax_open_zones\fR can specify 0 to disable any limit on the number of zones
 that can be simultaneously written to by all jobs.
 .TP
diff --git a/io_u.c b/io_u.c
index 27b6c92a..07e5bac5 100644
--- a/io_u.c
+++ b/io_u.c
@@ -1879,6 +1879,8 @@ static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)
 		io_ddir_name(io_u->ddir),
 		io_u->offset, io_u->xfer_buflen);
 
+	zbd_log_err(td, io_u);
+
 	if (td->io_ops->errdetails) {
 		char *err = td->io_ops->errdetails(io_u);
 
diff --git a/ioengines.h b/ioengines.h
index 9484265e..4391b31e 100644
--- a/ioengines.h
+++ b/ioengines.h
@@ -9,7 +9,7 @@
 #include "zbd_types.h"
 #include "fdp.h"
 
-#define FIO_IOOPS_VERSION	32
+#define FIO_IOOPS_VERSION	33
 
 #ifndef CONFIG_DYNAMIC_ENGINES
 #define FIO_STATIC	static
@@ -62,6 +62,8 @@ struct ioengine_ops {
 			uint64_t, uint64_t);
 	int (*get_max_open_zones)(struct thread_data *, struct fio_file *,
 				  unsigned int *);
+	int (*get_max_active_zones)(struct thread_data *, struct fio_file *,
+				    unsigned int *);
 	int (*finish_zone)(struct thread_data *, struct fio_file *,
 			   uint64_t, uint64_t);
 	int (*fdp_fetch_ruhs)(struct thread_data *, struct fio_file *,
diff --git a/oslib/blkzoned.h b/oslib/blkzoned.h
index 29fb034f..e598bd4f 100644
--- a/oslib/blkzoned.h
+++ b/oslib/blkzoned.h
@@ -18,6 +18,9 @@ extern int blkzoned_reset_wp(struct thread_data *td, struct fio_file *f,
 				uint64_t offset, uint64_t length);
 extern int blkzoned_get_max_open_zones(struct thread_data *td, struct fio_file *f,
 				       unsigned int *max_open_zones);
+extern int blkzoned_get_max_active_zones(struct thread_data *td,
+					 struct fio_file *f,
+					 unsigned int *max_active_zones);
 extern int blkzoned_finish_zone(struct thread_data *td, struct fio_file *f,
 				uint64_t offset, uint64_t length);
 #else
@@ -53,6 +56,12 @@ static inline int blkzoned_get_max_open_zones(struct thread_data *td, struct fio
 {
 	return -EIO;
 }
+static inline int blkzoned_get_max_active_zones(struct thread_data *td,
+						struct fio_file *f,
+						unsigned int *max_open_zones)
+{
+	return -EIO;
+}
 static inline int blkzoned_finish_zone(struct thread_data *td,
 				       struct fio_file *f,
 				       uint64_t offset, uint64_t length)
diff --git a/oslib/linux-blkzoned.c b/oslib/linux-blkzoned.c
index 722e0992..2c3ecf33 100644
--- a/oslib/linux-blkzoned.c
+++ b/oslib/linux-blkzoned.c
@@ -186,6 +186,29 @@ int blkzoned_get_max_open_zones(struct thread_data *td, struct fio_file *f,
 	return 0;
 }
 
+int blkzoned_get_max_active_zones(struct thread_data *td, struct fio_file *f,
+				  unsigned int *max_active_zones)
+{
+	char *max_active_str;
+
+	if (f->filetype != FIO_TYPE_BLOCK)
+		return -EIO;
+
+	max_active_str = blkzoned_get_sysfs_attr(f->file_name, "queue/max_active_zones");
+	if (!max_active_str) {
+		*max_active_zones = 0;
+		return 0;
+	}
+
+	dprint(FD_ZBD, "%s: max active zones supported by device: %s\n",
+	       f->file_name, max_active_str);
+	*max_active_zones = atoll(max_active_str);
+
+	free(max_active_str);
+
+	return 0;
+}
+
 static uint64_t zone_capacity(struct blk_zone_report *hdr,
 			      struct blk_zone *blkz)
 {
diff --git a/t/zbd/functions b/t/zbd/functions
index 9a6d6999..4faa45a9 100644
--- a/t/zbd/functions
+++ b/t/zbd/functions
@@ -4,6 +4,7 @@ blkzone=$(type -p blkzone 2>/dev/null)
 sg_inq=$(type -p sg_inq 2>/dev/null)
 zbc_report_zones=$(type -p zbc_report_zones 2>/dev/null)
 zbc_reset_zone=$(type -p zbc_reset_zone 2>/dev/null)
+zbc_close_zone=$(type -p zbc_close_zone 2>/dev/null)
 zbc_info=$(type -p zbc_info 2>/dev/null)
 if [ -z "${blkzone}" ] &&
        { [ -z "${zbc_report_zones}" ] || [ -z "${zbc_reset_zone}" ]; }; then
@@ -211,8 +212,14 @@ last_online_zone() {
 # max_open_zones in sysfs, or which lacks zoned block device support completely.
 max_open_zones() {
     local dev=$1
+    local realdev syspath
 
-    if [ -n "${sg_inq}" ] && [ ! -n "${use_libzbc}" ]; then
+    realdev=$(readlink -f "$dev")
+    syspath=/sys/block/${realdev##*/}/queue/max_open_zones
+
+    if [ -b "${realdev}" ] && [ -r "${syspath}" ]; then
+	cat ${syspath}
+    elif [ -n "${sg_inq}" ] && [ ! -n "${use_libzbc}" ]; then
 	if ! ${sg_inq} -e --page=0xB6 --len=20 --hex "$dev" \
 		 > /dev/null 2>&1; then
 	    # When sg_inq can not get max open zones, specify 0 which indicates
@@ -238,6 +245,18 @@ max_open_zones() {
     fi
 }
 
+# If sysfs provides, get max_active_zones limit of the zoned block device.
+max_active_zones() {
+	local dev=$1
+	local sys_queue="/sys/block/${dev##*/}/queue/"
+
+	if [[ -e "$sys_queue/max_active_zones" ]]; then
+		cat "$sys_queue/max_active_zones"
+		return
+	fi
+	echo 0
+}
+
 # Get minimum block size to write to seq zones. Refer the sysfs attribute
 # zone_write_granularity which shows the valid minimum size regardless of zoned
 # block device type. If the sysfs attribute is not available, refer physical
@@ -304,6 +323,18 @@ reset_zone() {
     fi
 }
 
+# Close the zone on device $1 at offset $2. The offset must be specified in
+# units of 512 byte sectors.
+close_zone() {
+	local dev=$1 offset=$2
+
+	if [ -n "${blkzone}" ] && [ -z "${use_libzbc}" ]; then
+		${blkzone} close -o "${offset}" -c 1 "$dev"
+	else
+		${zbc_close_zone} -sector "$dev" "${offset}" >/dev/null
+	fi
+}
+
 # Extract the number of bytes that have been transferred from a line like
 # READ: bw=6847KiB/s (7011kB/s), 6847KiB/s-6847KiB/s (7011kB/s-7011kB/s), io=257MiB (269MB), run=38406-38406msec
 fio_io() {
diff --git a/t/zbd/run-tests-against-nullb b/t/zbd/run-tests-against-nullb
index 7d2c7fa8..97d29966 100755
--- a/t/zbd/run-tests-against-nullb
+++ b/t/zbd/run-tests-against-nullb
@@ -67,13 +67,27 @@ configure_nullb()
 			fi
 			echo "${zone_capacity}" > zone_capacity
 		fi
+
 		if ((conv_pcnt)); then
 			if ((!conv_supported)); then
 				echo "null_blk does not support conventional zones"
 				return 2
 			fi
 			nr_conv=$((dev_size/zone_size*conv_pcnt/100))
-			echo "${nr_conv}" > zone_nr_conv
+		else
+			nr_conv=0
+		fi
+		echo "${nr_conv}" > zone_nr_conv
+
+		if ((max_open)); then
+			echo "${max_open}" > zone_max_open
+			if ((max_active)); then
+				if ((!max_act_supported)); then
+					echo "null_blk does not support active zone counts"
+					return 2
+				fi
+				echo "${max_active}" > zone_max_active
+			fi
 		fi
 	fi
 
@@ -90,6 +104,11 @@ show_nullb_config()
 		echo "    $(printf "Zone Capacity: %d MB" ${zone_capacity})"
 		if ((max_open)); then
 			echo "    $(printf "Max Open: %d Zones" ${max_open})"
+			if ((max_active)); then
+				echo "    $(printf "Max Active: %d Zones" ${max_active})"
+			else
+				echo "    Max Active: Unlimited Zones"
+			fi
 		else
 			echo "    Max Open: Unlimited Zones"
 		fi
@@ -124,6 +143,7 @@ section3()
 	zone_size=4
 	zone_capacity=3
 	max_open=0
+	max_active=0
 }
 
 # Zoned device with mostly sequential zones, ZCAP == ZSIZE, unlimited MaxOpen.
@@ -133,6 +153,7 @@ section4()
 	zone_size=1
 	zone_capacity=1
 	max_open=0
+	max_active=0
 }
 
 # Zoned device with mostly sequential zones, ZCAP < ZSIZE, unlimited MaxOpen.
@@ -142,6 +163,7 @@ section5()
 	zone_size=4
 	zone_capacity=3
 	max_open=0
+	max_active=0
 }
 
 # Zoned device with mostly conventional zones, ZCAP == ZSIZE, unlimited MaxOpen.
@@ -151,6 +173,7 @@ section6()
 	zone_size=1
 	zone_capacity=1
 	max_open=0
+	max_active=0
 }
 
 # Zoned device with mostly conventional zones, ZCAP < ZSIZE, unlimited MaxOpen.
@@ -161,9 +184,11 @@ section7()
 	zone_size=4
 	zone_capacity=3
 	max_open=0
+	max_active=0
 }
 
-# Zoned device with no conventional zones, ZCAP == ZSIZE, limited MaxOpen.
+# Zoned device with no conventional zones, ZCAP == ZSIZE, limited MaxOpen,
+# unlimited MaxActive.
 section8()
 {
 	dev_size=1024
@@ -172,9 +197,11 @@ section8()
 	zone_capacity=1
 	max_open=${set_max_open}
 	zbd_test_opts+=("-o ${max_open}")
+	max_active=0
 }
 
-# Zoned device with no conventional zones, ZCAP < ZSIZE, limited MaxOpen.
+# Zoned device with no conventional zones, ZCAP < ZSIZE, limited MaxOpen,
+# unlimited MaxActive.
 section9()
 {
 	conv_pcnt=0
@@ -182,9 +209,11 @@ section9()
 	zone_capacity=3
 	max_open=${set_max_open}
 	zbd_test_opts+=("-o ${max_open}")
+	max_active=0
 }
 
-# Zoned device with mostly sequential zones, ZCAP == ZSIZE, limited MaxOpen.
+# Zoned device with mostly sequential zones, ZCAP == ZSIZE, limited MaxOpen,
+# unlimited MaxActive.
 section10()
 {
 	conv_pcnt=10
@@ -192,9 +221,11 @@ section10()
 	zone_capacity=1
 	max_open=${set_max_open}
 	zbd_test_opts+=("-o ${max_open}")
+	max_active=0
 }
 
-# Zoned device with mostly sequential zones, ZCAP < ZSIZE, limited MaxOpen.
+# Zoned device with mostly sequential zones, ZCAP < ZSIZE, limited MaxOpen,
+# unlimited MaxActive.
 section11()
 {
 	conv_pcnt=10
@@ -202,9 +233,11 @@ section11()
 	zone_capacity=3
 	max_open=${set_max_open}
 	zbd_test_opts+=("-o ${max_open}")
+	max_active=0
 }
 
-# Zoned device with mostly conventional zones, ZCAP == ZSIZE, limited MaxOpen.
+# Zoned device with mostly conventional zones, ZCAP == ZSIZE, limited MaxOpen,
+# unlimited MaxActive.
 section12()
 {
 	conv_pcnt=66
@@ -212,9 +245,11 @@ section12()
 	zone_capacity=1
 	max_open=${set_max_open}
 	zbd_test_opts+=("-o ${max_open}")
+	max_active=0
 }
 
-# Zoned device with mostly conventional zones, ZCAP < ZSIZE, limited MaxOpen.
+# Zoned device with mostly conventional zones, ZCAP < ZSIZE, limited MaxOpen,
+# unlimited MaxActive.
 section13()
 {
 	dev_size=2048
@@ -223,6 +258,155 @@ section13()
 	zone_capacity=3
 	max_open=${set_max_open}
 	zbd_test_opts+=("-o ${max_open}")
+	max_active=0
+}
+
+# Zoned device with no conventional zones, ZCAP == ZSIZE, limited MaxOpen,
+# MaxActive == MaxOpen.
+section14()
+{
+	dev_size=1024
+	conv_pcnt=0
+	zone_size=1
+	zone_capacity=1
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=${set_max_open}
+}
+
+# Zoned device with no conventional zones, ZCAP < ZSIZE, limited MaxOpen,
+# MaxActive == MaxOpen.
+section15()
+{
+	conv_pcnt=0
+	zone_size=4
+	zone_capacity=3
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=${set_max_open}
+}
+
+# Zoned device with mostly sequential zones, ZCAP == ZSIZE, limited MaxOpen,
+# MaxActive == MaxOpen.
+section16()
+{
+	conv_pcnt=10
+	zone_size=1
+	zone_capacity=1
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=${set_max_open}
+}
+
+# Zoned device with mostly sequential zones, ZCAP < ZSIZE, limited MaxOpen,
+# MaxActive == MaxOpen.
+section17()
+{
+	conv_pcnt=10
+	zone_size=4
+	zone_capacity=3
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=${set_max_open}
+}
+
+# Zoned device with mostly conventional zones, ZCAP == ZSIZE, limited MaxOpen,
+# MaxActive == MaxOpen.
+section18()
+{
+	conv_pcnt=66
+	zone_size=1
+	zone_capacity=1
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=${set_max_open}
+}
+
+# Zoned device with mostly conventional zones, ZCAP < ZSIZE, limited MaxOpen,
+# MaxActive == MaxOpen.
+section19()
+{
+	dev_size=2048
+	conv_pcnt=66
+	zone_size=4
+	zone_capacity=3
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=${set_max_open}
+}
+
+# Zoned device with no conventional zones, ZCAP == ZSIZE, limited MaxOpen,
+# MaxActive > MaxOpen.
+section20()
+{
+	dev_size=1024
+	conv_pcnt=0
+	zone_size=1
+	zone_capacity=1
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=$((set_max_open+set_extra_max_active))
+}
+
+# Zoned device with no conventional zones, ZCAP < ZSIZE, limited MaxOpen,
+# MaxActive > MaxOpen.
+section21()
+{
+	conv_pcnt=0
+	zone_size=4
+	zone_capacity=3
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=$((set_max_open+set_extra_max_active))
+}
+
+# Zoned device with mostly sequential zones, ZCAP == ZSIZE, limited MaxOpen,
+# MaxActive > MaxOpen.
+section22()
+{
+	conv_pcnt=10
+	zone_size=1
+	zone_capacity=1
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=$((set_max_open+set_extra_max_active))
+}
+
+# Zoned device with mostly sequential zones, ZCAP < ZSIZE, limited MaxOpen,
+# MaxActive > MaxOpen.
+section23()
+{
+	conv_pcnt=10
+	zone_size=4
+	zone_capacity=3
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=$((set_max_open+set_extra_max_active))
+}
+
+# Zoned device with mostly conventional zones, ZCAP == ZSIZE, limited MaxOpen,
+# MaxActive > MaxOpen.
+section24()
+{
+	conv_pcnt=66
+	zone_size=1
+	zone_capacity=1
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=$((set_max_open+set_extra_max_active))
+}
+
+# Zoned device with mostly conventional zones, ZCAP < ZSIZE, limited MaxOpen,
+# MaxActive > MaxOpen.
+section25()
+{
+	dev_size=2048
+	conv_pcnt=66
+	zone_size=4
+	zone_capacity=3
+	max_open=${set_max_open}
+	zbd_test_opts+=("-o ${max_open}")
+	max_active=$((set_max_open+set_extra_max_active))
 }
 
 #
@@ -233,10 +417,12 @@ scriptdir="$(cd "$(dirname "$0")" && pwd)"
 sections=()
 zcap_supported=1
 conv_supported=1
+max_act_supported=1
 list_only=0
 dev_size=1024
 dev_blocksize=4096
 set_max_open=8
+set_extra_max_active=2
 zbd_test_opts=()
 num_of_runs=1
 test_case=0
@@ -276,6 +462,9 @@ fi
 if ! cat /sys/kernel/config/nullb/features | grep -q zone_nr_conv; then
 	conv_supported=0
 fi
+if ! cat /sys/kernel/config/nullb/features | grep -q zone_max_active; then
+	max_act_supported=0
+fi
 
 rc=0
 test_rc=0
diff --git a/t/zbd/test-zbd-support b/t/zbd/test-zbd-support
index a3d37a7d..c8f3eb61 100755
--- a/t/zbd/test-zbd-support
+++ b/t/zbd/test-zbd-support
@@ -272,6 +272,20 @@ require_max_open_zones() {
 	return 0
 }
 
+require_max_active_zones() {
+	local min=${1}
+
+	if ((max_active_zones == 0)); then
+		SKIP_REASON="$dev does not have max_active_zones limit"
+		return 1
+	fi
+	if ((max_active_zones < min)); then
+		SKIP_REASON="max_active_zones of $dev is smaller than $min"
+		return 1
+	fi
+	return 0
+}
+
 # Check whether buffered writes are refused for block devices.
 test1() {
     require_block_dev || return $SKIP_TESTCASE
@@ -780,9 +794,10 @@ test31() {
     opts=("--name=$dev" "--filename=$dev" "--rw=write" "--bs=${bs}")
     opts+=("--offset=$off" "--size=$((inc * nz))" "--io_size=$((bs * nz))")
     opts+=("--zonemode=strided" "--zonesize=${bs}" "--zonerange=${inc}")
-    opts+=("--direct=1")
+    opts+=("--direct=1" "$(ioengine "psync")")
     echo "fio ${opts[@]}" >> "${logfile}.${test_number}"
-    "$(dirname "$0")/../../fio" "${opts[@]}" >> "${logfile}.${test_number}" 2>&1
+    "$(dirname "$0")/../../fio" "${opts[@]}" >> "${logfile}.${test_number}" \
+				2>&1 || return $?
 
     # Next, run the test.
     opts=("--name=$dev" "--filename=$dev" "--offset=$off" "--size=$size")
@@ -1182,6 +1197,7 @@ test54() {
 	require_zbd || return $SKIP_TESTCASE
 	require_seq_zones 8 || return $SKIP_TESTCASE
 
+	prep_write
 	run_fio --name=job --filename=${dev} "$(ioengine "libaio")" \
 		--time_based=1 --runtime=30s --continue_on_error=0 \
 		--offset=$((first_sequential_zone_sector * 512)) \
@@ -1203,6 +1219,7 @@ test55() {
 	# offset=1z + offset_increment=10z + size=2z
 	require_seq_zones 13 || return $SKIP_TESTCASE
 
+	prep_write
 	run_fio	--name=j		\
 		--filename=${dev}	\
 		--direct=1		\
@@ -1228,6 +1245,7 @@ test56() {
 	require_regular_block_dev || return $SKIP_TESTCASE
 	require_seq_zones 10 || return $SKIP_TESTCASE
 
+	prep_write
 	run_fio	--name=j		\
 		--filename=${dev}	\
 		--direct=1		\
@@ -1249,6 +1267,7 @@ test57() {
 
 	require_zbd || return $SKIP_TESTCASE
 
+	prep_write
 	bs=$((4096 * 7))
 	off=$((first_sequential_zone_sector * 512))
 
@@ -1413,6 +1432,71 @@ test65() {
 	check_written $((zone_size + capacity))
 }
 
+# Test closed zones are handled as open zones. This test case requires zoned
+# block devices which has same max_open_zones and max_active_zones.
+test66() {
+	local i off
+
+	require_zbd || return $SKIP_TESTCASE
+	require_max_active_zones 2 || return $SKIP_TESTCASE
+	require_max_open_zones "${max_active_zones}" || return $SKIP_TESTCASE
+	require_seq_zones $((max_active_zones * 16)) || return $SKIP_TESTCASE
+
+	reset_zone "$dev" -1
+
+	# Prepare max_active_zones in closed condition.
+	off=$((first_sequential_zone_sector * 512))
+	run_fio --name=w --filename="$dev" --zonemod=zbd --direct=1 \
+		--offset=$((off)) --zonesize="${zone_size}" --rw=randwrite \
+		--bs=4096 --size="$((zone_size * max_active_zones))" \
+		--io_size="${zone_size}" "$(ioengine "psync")" \
+		>> "${logfile}.${test_number}" 2>&1 || return $?
+	for ((i = 0; i < max_active_zones; i++)); do
+		close_zone "$dev" $((off / 512)) || return $?
+		off=$((off + zone_size))
+	done
+
+	# Run random write to the closed zones and empty zones. This confirms
+	# that fio handles closed zones as write target open zones. Otherwise,
+	# fio writes to the empty zones and hit the max_active_zones limit.
+	off=$((first_sequential_zone_sector * 512))
+	run_one_fio_job --zonemod=zbd --direct=1 \
+		       "$(ioengine "psync")" --rw=randwrite --bs=4096 \
+		       --max_open_zones="$max_active_zones" --offset=$((off)) \
+		       --size=$((max_active_zones * 16 * zone_size)) \
+		       --io_size=$((zone_size)) --zonesize="${zone_size}" \
+		       --time_based --runtime=5s \
+		       >> "${logfile}.${test_number}" 2>&1
+}
+
+# Test max_active_zones limit failure is reported with good error message.
+test67() {
+	local i off
+
+	require_zbd || return $SKIP_TESTCASE
+	require_max_active_zones 2 || return $SKIP_TESTCASE
+	require_max_open_zones "${max_active_zones}" || return $SKIP_TESTCASE
+	require_seq_zones $((max_active_zones + 1)) || return $SKIP_TESTCASE
+
+	reset_zone "$dev" -1
+
+	# Prepare max_active_zones in open condition.
+	off=$((first_sequential_zone_sector * 512))
+	run_fio --name=w --filename="$dev" --zonemod=zbd --direct=1 \
+		--offset=$((off)) --zonesize="${zone_size}" --rw=randwrite \
+		--bs=4096 --size="$((zone_size * max_active_zones))" \
+		--io_size="${zone_size}" "$(ioengine "psync")" \
+		>> "${logfile}.${test_number}" 2>&1 || return $?
+
+	# Write to antoher zone and trigger max_active_zones limit error.
+	off=$((off + zone_size * max_active_zones))
+	run_one_fio_job --zonemod=zbd --direct=1 "$(ioengine "psync")" \
+			--rw=write --bs=$min_seq_write_size --offset=$((off)) \
+			--size=$((zone_size)) --zonesize="${zone_size}" \
+			>> "${logfile}.${test_number}" 2>&1 && return $?
+	grep -q 'Exceeded max_active_zones limit' "${logfile}.${test_number}"
+}
+
 SECONDS=0
 tests=()
 dynamic_analyzer=()
@@ -1497,6 +1581,7 @@ if [[ -b "$realdev" ]]; then
 			echo "Failed to determine maximum number of open zones"
 			exit 1
 		fi
+		max_active_zones=$(max_active_zones "$dev")
 		set_io_scheduler "$basename" deadline || exit $?
 		if [ -n "$reset_all_zones" ]; then
 			reset_zone "$dev" -1
@@ -1508,6 +1593,7 @@ if [[ -b "$realdev" ]]; then
 		zone_size=$(max 65536 "$min_seq_write_size")
 		sectors_per_zone=$((zone_size / 512))
 		max_open_zones=128
+		max_active_zones=0
 		set_io_scheduler "$basename" none || exit $?
 		;;
 	esac
@@ -1543,6 +1629,7 @@ elif [[ -c "$realdev" ]]; then
 		echo "Failed to determine maximum number of open zones"
 		exit 1
 	fi
+	max_active_zones=0
 	if [ -n "$reset_all_zones" ]; then
 		reset_zone "$dev" -1
 	fi
diff --git a/zbd.c b/zbd.c
index d4565215..caac68bb 100644
--- a/zbd.c
+++ b/zbd.c
@@ -471,6 +471,34 @@ static int zbd_get_max_open_zones(struct thread_data *td, struct fio_file *f,
 	return ret;
 }
 
+/**
+ * zbd_get_max_active_zones - Get the maximum number of active zones
+ * @td: FIO thread data
+ * @f: FIO file for which to get max active zones
+ *
+ * Returns max_active_zones limit value of the target file if it is available.
+ * Otherwise return zero, which means no limit.
+ */
+static unsigned int zbd_get_max_active_zones(struct thread_data *td,
+					     struct fio_file *f)
+{
+	unsigned int max_active_zones;
+	int ret;
+
+	if (td->io_ops && td->io_ops->get_max_active_zones)
+		ret = td->io_ops->get_max_active_zones(td, f,
+						       &max_active_zones);
+	else
+		ret = blkzoned_get_max_active_zones(td, f, &max_active_zones);
+	if (ret < 0) {
+		dprint(FD_ZBD, "%s: max_active_zones is not available\n",
+		       f->file_name);
+		return 0;
+	}
+
+	return max_active_zones;
+}
+
 /**
  * __zbd_write_zone_get - Add a zone to the array of write zones.
  * @td: fio thread data.
@@ -927,6 +955,7 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f)
 	f->zbd_info->zone_size_log2 = is_power_of_2(zone_size) ?
 		ilog2(zone_size) : 0;
 	f->zbd_info->nr_zones = nr_zones;
+	f->zbd_info->max_active_zones = zbd_get_max_active_zones(td, f);
 
 	if (same_zone_cap)
 		dprint(FD_ZBD, "Zone capacity = %"PRIu64" KB\n",
@@ -1247,7 +1276,11 @@ int zbd_setup_files(struct thread_data *td)
 		for (zi = f->min_zone; zi < f->max_zone; zi++) {
 			z = &zbd->zone_info[zi];
 			if (z->cond != ZBD_ZONE_COND_IMP_OPEN &&
-			    z->cond != ZBD_ZONE_COND_EXP_OPEN)
+			    z->cond != ZBD_ZONE_COND_EXP_OPEN &&
+			    z->cond != ZBD_ZONE_COND_CLOSED)
+				continue;
+			if (!zbd->max_active_zones &&
+			    z->cond == ZBD_ZONE_COND_CLOSED)
 				continue;
 			if (__zbd_write_zone_get(td, f, z))
 				continue;
@@ -2210,3 +2243,15 @@ int zbd_do_io_u_trim(struct thread_data *td, struct io_u *io_u)
 
 	return io_u_completed;
 }
+
+void zbd_log_err(const struct thread_data *td, const struct io_u *io_u)
+{
+	const struct fio_file *f = io_u->file;
+
+	if (td->o.zone_mode != ZONE_MODE_ZBD)
+		return;
+
+	if (io_u->error == EOVERFLOW)
+		log_err("%s: Exceeded max_active_zones limit. Check conditions of zones out of I/O ranges.\n",
+			f->file_name);
+}
diff --git a/zbd.h b/zbd.h
index f0ac9876..5750a0b8 100644
--- a/zbd.h
+++ b/zbd.h
@@ -52,6 +52,9 @@ struct fio_zone_info {
  *      are simultaneously written. A zero value means unlimited zones of
  *      simultaneous writes and that write target zones will not be tracked in
  *      the write_zones array.
+ * @max_active_zones: device side limit on the number of sequential write zones
+ *	in open or closed conditions. A zero value means unlimited number of
+ *	zones in the conditions.
  * @mutex: Protects the modifiable members in this structure (refcount and
  *		num_open_zones).
  * @zone_size: size of a single zone in bytes.
@@ -75,6 +78,7 @@ struct fio_zone_info {
 struct zoned_block_device_info {
 	enum zbd_zoned_model	model;
 	uint32_t		max_write_zones;
+	uint32_t		max_active_zones;
 	pthread_mutex_t		mutex;
 	uint64_t		zone_size;
 	uint64_t		wp_valid_data_bytes;
@@ -101,6 +105,7 @@ enum fio_ddir zbd_adjust_ddir(struct thread_data *td, struct io_u *io_u,
 enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u);
 char *zbd_write_status(const struct thread_stat *ts);
 int zbd_do_io_u_trim(struct thread_data *td, struct io_u *io_u);
+void zbd_log_err(const struct thread_data *td, const struct io_u *io_u);
 
 static inline void zbd_close_file(struct fio_file *f)
 {

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-16 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-16 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 14adf6e31487aa2bc8e47cd037428036089a3834:

  thinktime: Avoid calculating a negative time left to wait (2023-07-14 14:03:34 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 04361e9a23d6e0448fd6fbbd4e14ecdfff60e314:

  Merge branch 'patch-3' of https://github.com/yangjueji/fio (2023-07-15 09:57:43 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'patch-3' of https://github.com/yangjueji/fio

Jueji Yang (1):
      fix: io_uring sqpoll issue_time empty when kernel not yet read sq

 engines/io_uring.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 407d65ce..f30a3c00 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -646,7 +646,7 @@ static int fio_ioring_commit(struct thread_data *td)
 	 */
 	if (o->sqpoll_thread) {
 		struct io_sq_ring *ring = &ld->sq_ring;
-		unsigned start = *ld->sq_ring.head;
+		unsigned start = *ld->sq_ring.tail - ld->queued;
 		unsigned flags;
 
 		flags = atomic_load_acquire(ring->flags);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-15 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-15 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 270316dd2566346a12cfdf3cbe9996a88307f87d:

  Merge branch 'master' of https://github.com/bvanassche/fio (2023-07-13 15:28:20 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 14adf6e31487aa2bc8e47cd037428036089a3834:

  thinktime: Avoid calculating a negative time left to wait (2023-07-14 14:03:34 -0400)

----------------------------------------------------------------
Michael Kelley (1):
      thinktime: Avoid calculating a negative time left to wait

Vincent Fu (2):
      stat: add new diskutil sectors to json output
      stat: add diskutil aggregated sectors to normal output

 backend.c | 11 ++++++++++-
 stat.c    | 14 +++++++++++---
 2 files changed, 21 insertions(+), 4 deletions(-)

---

Diff of recent changes:

diff --git a/backend.c b/backend.c
index d67a4a07..b06a11a5 100644
--- a/backend.c
+++ b/backend.c
@@ -897,7 +897,16 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir,
 	if (left)
 		total = usec_spin(left);
 
-	left = td->o.thinktime - total;
+	/*
+	 * usec_spin() might run for slightly longer than intended in a VM
+	 * where the vCPU could get descheduled or the hypervisor could steal
+	 * CPU time. Ensure "left" doesn't become negative.
+	 */
+	if (total < td->o.thinktime)
+		left = td->o.thinktime - total;
+	else
+		left = 0;
+
 	if (td->o.timeout) {
 		runtime_left = td->o.timeout - utime_since_now(&td->epoch);
 		if (runtime_left < (unsigned long long)left)
diff --git a/stat.c b/stat.c
index ced73645..7fad73d1 100644
--- a/stat.c
+++ b/stat.c
@@ -957,11 +957,13 @@ static void show_agg_stats(struct disk_util_agg *agg, int terse,
 		return;
 
 	if (!terse) {
-		log_buf(out, ", aggrios=%llu/%llu, aggrmerge=%llu/%llu, "
-			 "aggrticks=%llu/%llu, aggrin_queue=%llu, "
-			 "aggrutil=%3.2f%%",
+		log_buf(out, ", aggrios=%llu/%llu, aggsectors=%llu/%llu, "
+			 "aggrmerge=%llu/%llu, aggrticks=%llu/%llu, "
+			 "aggrin_queue=%llu, aggrutil=%3.2f%%",
 			(unsigned long long) agg->ios[0] / agg->slavecount,
 			(unsigned long long) agg->ios[1] / agg->slavecount,
+			(unsigned long long) agg->sectors[0] / agg->slavecount,
+			(unsigned long long) agg->sectors[1] / agg->slavecount,
 			(unsigned long long) agg->merges[0] / agg->slavecount,
 			(unsigned long long) agg->merges[1] / agg->slavecount,
 			(unsigned long long) agg->ticks[0] / agg->slavecount,
@@ -1084,6 +1086,8 @@ void json_array_add_disk_util(struct disk_util_stat *dus,
 	json_object_add_value_string(obj, "name", (const char *)dus->name);
 	json_object_add_value_int(obj, "read_ios", dus->s.ios[0]);
 	json_object_add_value_int(obj, "write_ios", dus->s.ios[1]);
+	json_object_add_value_int(obj, "read_sectors", dus->s.sectors[0]);
+	json_object_add_value_int(obj, "write_sectors", dus->s.sectors[1]);
 	json_object_add_value_int(obj, "read_merges", dus->s.merges[0]);
 	json_object_add_value_int(obj, "write_merges", dus->s.merges[1]);
 	json_object_add_value_int(obj, "read_ticks", dus->s.ticks[0]);
@@ -1101,6 +1105,10 @@ void json_array_add_disk_util(struct disk_util_stat *dus,
 				agg->ios[0] / agg->slavecount);
 	json_object_add_value_int(obj, "aggr_write_ios",
 				agg->ios[1] / agg->slavecount);
+	json_object_add_value_int(obj, "aggr_read_sectors",
+				agg->sectors[0] / agg->slavecount);
+	json_object_add_value_int(obj, "aggr_write_sectors",
+				agg->sectors[1] / agg->slavecount);
 	json_object_add_value_int(obj, "aggr_read_merges",
 				agg->merges[0] / agg->slavecount);
 	json_object_add_value_int(obj, "aggr_write_merge",

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-14 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-14 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 8e2b81b854286f32eae7951a434dddebd968f9d5:

  zbd: Support finishing zones on Android (2023-07-05 15:48:11 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 270316dd2566346a12cfdf3cbe9996a88307f87d:

  Merge branch 'master' of https://github.com/bvanassche/fio (2023-07-13 15:28:20 -0600)

----------------------------------------------------------------
Ankit Kumar (4):
      fdp: use macros
      fdp: fix placement id check
      fdp: support random placement id selection
      engines/xnvme: add support for fdp

Bart Van Assche (5):
      diskutil: Improve disk utilization data structure documentation
      diskutil: Remove casts from get_io_ticks()
      diskutil: Simplify get_io_ticks()
      diskutil: Fix a debug statement in get_io_ticks()
      diskutil: Report how many sectors have been read and written

Jens Axboe (1):
      Merge branch 'master' of https://github.com/bvanassche/fio

Vincent Fu (1):
      options: add code for FDP pli selection use in client/server mode

 HOWTO.rst              | 23 +++++++++++++--
 cconv.c                |  2 ++
 configure              |  2 +-
 diskutil.c             | 29 +++++++------------
 diskutil.h             | 12 +++++++-
 engines/io_uring.c     |  2 +-
 engines/xnvme.c        | 78 +++++++++++++++++++++++++++++++++++++++++++++++++-
 examples/xnvme-fdp.fio | 36 +++++++++++++++++++++++
 fdp.c                  | 22 ++++++++------
 fdp.h                  | 13 +++++++++
 fio.1                  | 22 ++++++++++++--
 fio.h                  |  2 ++
 init.c                 |  2 ++
 options.c              | 20 +++++++++++++
 stat.c                 |  7 +++--
 thread_options.h       |  2 ++
 16 files changed, 236 insertions(+), 38 deletions(-)
 create mode 100644 examples/xnvme-fdp.fio

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 2e1e55c2..7ae8ea7b 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2431,11 +2431,26 @@ with the caveat that when used on the command line, they must come after the
 	For direct I/O, requests will only succeed if cache invalidation isn't required,
 	file blocks are fully allocated and the disk request could be issued immediately.
 
-.. option:: fdp=bool : [io_uring_cmd]
+.. option:: fdp=bool : [io_uring_cmd] [xnvme]
 
 	Enable Flexible Data Placement mode for write commands.
 
-.. option:: fdp_pli=str : [io_uring_cmd]
+.. option:: fdp_pli_select=str : [io_uring_cmd] [xnvme]
+
+	Defines how fio decides which placement ID to use next. The following
+	types are defined:
+
+		**random**
+			Choose a placement ID at random (uniform).
+
+		**roundrobin**
+			Round robin over available placement IDs. This is the
+			default.
+
+	The available placement ID index/indices is defined by the option
+	:option:`fdp_pli`.
+
+.. option:: fdp_pli=str : [io_uring_cmd] [xnvme]
 
 	Select which Placement ID Index/Indicies this job is allowed to use for
 	writes. By default, the job will cycle through all available Placement
@@ -4513,13 +4528,15 @@ For each data direction it prints:
 And finally, the disk statistics are printed. This is Linux specific. They will look like this::
 
   Disk stats (read/write):
-    sda: ios=16398/16511, merge=30/162, ticks=6853/819634, in_queue=826487, util=100.00%
+    sda: ios=16398/16511, sectors=32321/65472, merge=30/162, ticks=6853/819634, in_queue=826487, util=100.00%
 
 Each value is printed for both reads and writes, with reads first. The
 numbers denote:
 
 **ios**
 		Number of I/Os performed by all groups.
+**sectors**
+		Amount of data transferred in units of 512 bytes for all groups.
 **merge**
 		Number of merges performed by the I/O scheduler.
 **ticks**
diff --git a/cconv.c b/cconv.c
index 9095d519..1bfa770f 100644
--- a/cconv.c
+++ b/cconv.c
@@ -351,6 +351,7 @@ int convert_thread_options_to_cpu(struct thread_options *o,
 		o->merge_blktrace_iters[i].u.f = fio_uint64_to_double(le64_to_cpu(top->merge_blktrace_iters[i].u.i));
 
 	o->fdp = le32_to_cpu(top->fdp);
+	o->fdp_pli_select = le32_to_cpu(top->fdp_pli_select);
 	o->fdp_nrpli = le32_to_cpu(top->fdp_nrpli);
 	for (i = 0; i < o->fdp_nrpli; i++)
 		o->fdp_plis[i] = le32_to_cpu(top->fdp_plis[i]);
@@ -645,6 +646,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 		top->merge_blktrace_iters[i].u.i = __cpu_to_le64(fio_double_to_uint64(o->merge_blktrace_iters[i].u.f));
 
 	top->fdp = cpu_to_le32(o->fdp);
+	top->fdp_pli_select = cpu_to_le32(o->fdp_pli_select);
 	top->fdp_nrpli = cpu_to_le32(o->fdp_nrpli);
 	for (i = 0; i < o->fdp_nrpli; i++)
 		top->fdp_plis[i] = cpu_to_le32(o->fdp_plis[i]);
diff --git a/configure b/configure
index 74416fd4..6c938251 100755
--- a/configure
+++ b/configure
@@ -2651,7 +2651,7 @@ fi
 ##########################################
 # Check if we have xnvme
 if test "$xnvme" != "no" ; then
-  if check_min_lib_version xnvme 0.2.0; then
+  if check_min_lib_version xnvme 0.7.0; then
     xnvme="yes"
     xnvme_cflags=$(pkg-config --cflags xnvme)
     xnvme_libs=$(pkg-config --libs xnvme)
diff --git a/diskutil.c b/diskutil.c
index ace7af3d..cf4ede85 100644
--- a/diskutil.c
+++ b/diskutil.c
@@ -1,3 +1,4 @@
+#include <inttypes.h>
 #include <stdio.h>
 #include <string.h>
 #include <sys/types.h>
@@ -44,8 +45,6 @@ static void disk_util_free(struct disk_util *du)
 
 static int get_io_ticks(struct disk_util *du, struct disk_util_stat *dus)
 {
-	unsigned in_flight;
-	unsigned long long sectors[2];
 	char line[256];
 	FILE *f;
 	char *p;
@@ -65,23 +64,17 @@ static int get_io_ticks(struct disk_util *du, struct disk_util_stat *dus)
 
 	dprint(FD_DISKUTIL, "%s: %s", du->path, p);
 
-	ret = sscanf(p, "%llu %llu %llu %llu %llu %llu %llu %llu %u %llu %llu\n",
-				(unsigned long long *) &dus->s.ios[0],
-				(unsigned long long *) &dus->s.merges[0],
-				&sectors[0],
-				(unsigned long long *) &dus->s.ticks[0],
-				(unsigned long long *) &dus->s.ios[1],
-				(unsigned long long *) &dus->s.merges[1],
-				&sectors[1],
-				(unsigned long long *) &dus->s.ticks[1],
-				&in_flight,
-				(unsigned long long *) &dus->s.io_ticks,
-				(unsigned long long *) &dus->s.time_in_queue);
+	ret = sscanf(p, "%"SCNu64" %"SCNu64" %"SCNu64" %"SCNu64" "
+		     "%"SCNu64" %"SCNu64" %"SCNu64" %"SCNu64" "
+		     "%*u %"SCNu64" %"SCNu64"\n",
+		     &dus->s.ios[0], &dus->s.merges[0], &dus->s.sectors[0],
+		     &dus->s.ticks[0],
+		     &dus->s.ios[1], &dus->s.merges[1], &dus->s.sectors[1],
+		     &dus->s.ticks[1],
+		     &dus->s.io_ticks, &dus->s.time_in_queue);
 	fclose(f);
-	dprint(FD_DISKUTIL, "%s: stat read ok? %d\n", du->path, ret == 1);
-	dus->s.sectors[0] = sectors[0];
-	dus->s.sectors[1] = sectors[1];
-	return ret != 11;
+	dprint(FD_DISKUTIL, "%s: stat read ok? %d\n", du->path, ret == 10);
+	return ret != 10;
 }
 
 static void update_io_tick_disk(struct disk_util *du)
diff --git a/diskutil.h b/diskutil.h
index 7d7ef802..9dca42c4 100644
--- a/diskutil.h
+++ b/diskutil.h
@@ -7,6 +7,16 @@
 #include "helper_thread.h"
 #include "fio_sem.h"
 
+/**
+ * @ios: Number of I/O operations that have been completed successfully.
+ * @merges: Number of I/O operations that have been merged.
+ * @sectors: I/O size in 512-byte units.
+ * @ticks: Time spent on I/O in milliseconds.
+ * @io_ticks: CPU time spent on I/O in milliseconds.
+ * @time_in_queue: Weighted time spent doing I/O in milliseconds.
+ *
+ * For the array members, index 0 refers to reads and index 1 refers to writes.
+ */
 struct disk_util_stats {
 	uint64_t ios[2];
 	uint64_t merges[2];
@@ -18,7 +28,7 @@ struct disk_util_stats {
 };
 
 /*
- * Disk utils as read in /sys/block/<dev>/stat
+ * Disk utilization as read from /sys/block/<dev>/stat
  */
 struct disk_util_stat {
 	uint8_t name[FIO_DU_NAME_SZ];
diff --git a/engines/io_uring.c b/engines/io_uring.c
index 5021239e..407d65ce 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -1310,7 +1310,7 @@ static int fio_ioring_cmd_fetch_ruhs(struct thread_data *td, struct fio_file *f,
 	struct nvme_fdp_ruh_status *ruhs;
 	int bytes, ret, i;
 
-	bytes = sizeof(*ruhs) + 128 * sizeof(struct nvme_fdp_ruh_status_desc);
+	bytes = sizeof(*ruhs) + FDP_MAX_RUHS * sizeof(struct nvme_fdp_ruh_status_desc);
 	ruhs = scalloc(1, bytes);
 	if (!ruhs)
 		return -ENOMEM;
diff --git a/engines/xnvme.c b/engines/xnvme.c
index bb92a121..ce7b2bdd 100644
--- a/engines/xnvme.c
+++ b/engines/xnvme.c
@@ -16,6 +16,7 @@
 #include <libxnvme_spec_fs.h>
 #include "fio.h"
 #include "zbd_types.h"
+#include "fdp.h"
 #include "optgroup.h"
 
 static pthread_mutex_t g_serialize = PTHREAD_MUTEX_INITIALIZER;
@@ -509,6 +510,7 @@ static enum fio_q_status xnvme_fioe_queue(struct thread_data *td, struct io_u *i
 	uint16_t nlb;
 	int err;
 	bool vectored_io = ((struct xnvme_fioe_options *)td->eo)->xnvme_iovec;
+	uint32_t dir = io_u->dtype;
 
 	fio_ro_check(td, io_u);
 
@@ -524,6 +526,10 @@ static enum fio_q_status xnvme_fioe_queue(struct thread_data *td, struct io_u *i
 	ctx->cmd.common.nsid = nsid;
 	ctx->cmd.nvm.slba = slba;
 	ctx->cmd.nvm.nlb = nlb;
+	if (dir) {
+		ctx->cmd.nvm.dtype = io_u->dtype;
+		ctx->cmd.nvm.cdw13.dspec = io_u->dspec;
+	}
 
 	switch (io_u->ddir) {
 	case DDIR_READ:
@@ -947,6 +953,72 @@ exit:
 	return err;
 }
 
+static int xnvme_fioe_fetch_ruhs(struct thread_data *td, struct fio_file *f,
+				 struct fio_ruhs_info *fruhs_info)
+{
+	struct xnvme_opts opts = xnvme_opts_from_fioe(td);
+	struct xnvme_dev *dev;
+	struct xnvme_spec_ruhs *ruhs;
+	struct xnvme_cmd_ctx ctx;
+	uint32_t ruhs_nbytes;
+	uint32_t nsid;
+	int err = 0, err_lock;
+
+	if (f->filetype != FIO_TYPE_CHAR) {
+		log_err("ioeng->fdp_ruhs(): ignoring filetype: %d\n", f->filetype);
+		return -EINVAL;
+	}
+
+	err = pthread_mutex_lock(&g_serialize);
+	if (err) {
+		log_err("ioeng->fdp_ruhs(): pthread_mutex_lock(), err(%d)\n", err);
+		return -err;
+	}
+
+	dev = xnvme_dev_open(f->file_name, &opts);
+	if (!dev) {
+		log_err("ioeng->fdp_ruhs(): xnvme_dev_open(%s) failed, errno: %d\n",
+			f->file_name, errno);
+		err = -errno;
+		goto exit;
+	}
+
+	ruhs_nbytes = sizeof(*ruhs) + (FDP_MAX_RUHS * sizeof(struct xnvme_spec_ruhs_desc));
+	ruhs = xnvme_buf_alloc(dev, ruhs_nbytes);
+	if (!ruhs) {
+		err = -errno;
+		goto exit;
+	}
+	memset(ruhs, 0, ruhs_nbytes);
+
+	ctx = xnvme_cmd_ctx_from_dev(dev);
+	nsid = xnvme_dev_get_nsid(dev);
+
+	err = xnvme_nvm_mgmt_recv(&ctx, nsid, XNVME_SPEC_IO_MGMT_RECV_RUHS, 0, ruhs, ruhs_nbytes);
+
+	if (err || xnvme_cmd_ctx_cpl_status(&ctx)) {
+		err = err ? err : -EIO;
+		log_err("ioeng->fdp_ruhs(): err(%d), sc(%d)", err, ctx.cpl.status.sc);
+		goto free_buffer;
+	}
+
+	fruhs_info->nr_ruhs = ruhs->nruhsd;
+	for (uint32_t idx = 0; idx < fruhs_info->nr_ruhs; ++idx) {
+		fruhs_info->plis[idx] = le16_to_cpu(ruhs->desc[idx].pi);
+	}
+
+free_buffer:
+	xnvme_buf_free(dev, ruhs);
+exit:
+	xnvme_dev_close(dev);
+
+	err_lock = pthread_mutex_unlock(&g_serialize);
+	if (err_lock)
+		log_err("ioeng->fdp_ruhs(): pthread_mutex_unlock(), err(%d)\n", err_lock);
+
+	return err;
+}
+
 static int xnvme_fioe_get_file_size(struct thread_data *td, struct fio_file *f)
 {
 	struct xnvme_opts opts = xnvme_opts_from_fioe(td);
@@ -971,7 +1043,9 @@ static int xnvme_fioe_get_file_size(struct thread_data *td, struct fio_file *f)
 
 	f->real_file_size = xnvme_dev_get_geo(dev)->tbytes;
 	fio_file_set_size_known(f);
-	f->filetype = FIO_TYPE_BLOCK;
+
+	if (td->o.zone_mode == ZONE_MODE_ZBD)
+		f->filetype = FIO_TYPE_BLOCK;
 
 exit:
 	xnvme_dev_close(dev);
@@ -1011,6 +1085,8 @@ FIO_STATIC struct ioengine_ops ioengine = {
 	.get_zoned_model = xnvme_fioe_get_zoned_model,
 	.report_zones = xnvme_fioe_report_zones,
 	.reset_wp = xnvme_fioe_reset_wp,
+
+	.fdp_fetch_ruhs = xnvme_fioe_fetch_ruhs,
 };
 
 static void fio_init fio_xnvme_register(void)
diff --git a/examples/xnvme-fdp.fio b/examples/xnvme-fdp.fio
new file mode 100644
index 00000000..86fbe0d3
--- /dev/null
+++ b/examples/xnvme-fdp.fio
@@ -0,0 +1,36 @@
+; README
+;
+; This job-file is intended to be used either as:
+;
+; # Use the xNVMe io-engine engine io_uring_cmd async. impl.
+; fio examples/xnvme-fdp.fio \
+;   --section=default \
+;   --ioengine=xnvme \
+;   --xnvme_async=io_uring_cmd \
+;   --filename=/dev/ng0n1
+;
+; # Use the xNVMe io-engine engine with nvme sync. impl.
+; fio examples/xnvme-fdp.fio \
+;   --section=default \
+;   --ioengine=xnvme \
+;   --xnvme_sync=nvme \
+;   --filename=/dev/ng0n1
+;
+; FIO_BS="512" FIO_RW="read" FIO_IODEPTH=16 fio examples/xnvme-fdp.fio \
+;   --section=override --ioengine=xnvme --xnvme_sync=nvme --filename=/dev/ng0n1
+;
+[global]
+rw=randwrite
+size=2M
+iodepth=1
+bs=4K
+thread=1
+fdp=1
+fdp_pli=4,5
+
+[default]
+
+[override]
+rw=${FIO_RW}
+iodepth=${FIO_IODEPTH}
+bs=${FIO_BS}
diff --git a/fdp.c b/fdp.c
index d92dbc67..49c80d2c 100644
--- a/fdp.c
+++ b/fdp.c
@@ -45,7 +45,7 @@ static int init_ruh_info(struct thread_data *td, struct fio_file *f)
 	struct fio_ruhs_info *ruhs, *tmp;
 	int i, ret;
 
-	ruhs = scalloc(1, sizeof(*ruhs) + 128 * sizeof(*ruhs->plis));
+	ruhs = scalloc(1, sizeof(*ruhs) + FDP_MAX_RUHS * sizeof(*ruhs->plis));
 	if (!ruhs)
 		return -ENOMEM;
 
@@ -56,8 +56,8 @@ static int init_ruh_info(struct thread_data *td, struct fio_file *f)
 		goto out;
 	}
 
-	if (ruhs->nr_ruhs > 128)
-		ruhs->nr_ruhs = 128;
+	if (ruhs->nr_ruhs > FDP_MAX_RUHS)
+		ruhs->nr_ruhs = FDP_MAX_RUHS;
 
 	if (td->o.fdp_nrpli == 0) {
 		f->ruhs_info = ruhs;
@@ -65,7 +65,7 @@ static int init_ruh_info(struct thread_data *td, struct fio_file *f)
 	}
 
 	for (i = 0; i < td->o.fdp_nrpli; i++) {
-		if (td->o.fdp_plis[i] > ruhs->nr_ruhs) {
+		if (td->o.fdp_plis[i] >= ruhs->nr_ruhs) {
 			ret = -EINVAL;
 			goto out;
 		}
@@ -119,10 +119,16 @@ void fdp_fill_dspec_data(struct thread_data *td, struct io_u *io_u)
 		return;
 	}
 
-	if (ruhs->pli_loc >= ruhs->nr_ruhs)
-		ruhs->pli_loc = 0;
+	if (td->o.fdp_pli_select == FIO_FDP_RR) {
+		if (ruhs->pli_loc >= ruhs->nr_ruhs)
+			ruhs->pli_loc = 0;
 
-	dspec = ruhs->plis[ruhs->pli_loc++];
-	io_u->dtype = 2;
+		dspec = ruhs->plis[ruhs->pli_loc++];
+	} else {
+		ruhs->pli_loc = rand_between(&td->fdp_state, 0, ruhs->nr_ruhs - 1);
+		dspec = ruhs->plis[ruhs->pli_loc];
+	}
+
+	io_u->dtype = FDP_DIR_DTYPE;
 	io_u->dspec = dspec;
 }
diff --git a/fdp.h b/fdp.h
index 81691f62..accbac38 100644
--- a/fdp.h
+++ b/fdp.h
@@ -3,6 +3,19 @@
 
 #include "io_u.h"
 
+#define FDP_DIR_DTYPE	2
+#define FDP_MAX_RUHS	128
+
+/*
+ * How fio chooses what placement identifier to use next. Choice of
+ * uniformly random, or roundrobin.
+ */
+
+enum {
+	FIO_FDP_RANDOM	= 0x1,
+	FIO_FDP_RR	= 0x2,
+};
+
 struct fio_ruhs_info {
 	uint32_t nr_ruhs;
 	uint32_t pli_loc;
diff --git a/fio.1 b/fio.1
index 73b7e8c9..da875276 100644
--- a/fio.1
+++ b/fio.1
@@ -2192,10 +2192,26 @@ cached data. Currently the RWF_NOWAIT flag does not supported for cached write.
 For direct I/O, requests will only succeed if cache invalidation isn't required,
 file blocks are fully allocated and the disk request could be issued immediately.
 .TP
-.BI (io_uring_cmd)fdp \fR=\fPbool
+.BI (io_uring_cmd,xnvme)fdp \fR=\fPbool
 Enable Flexible Data Placement mode for write commands.
 .TP
-.BI (io_uring_cmd)fdp_pli \fR=\fPstr
+.BI (io_uring_cmd,xnvme)fdp_pli_select \fR=\fPstr
+Defines how fio decides which placement ID to use next. The following types
+are defined:
+.RS
+.RS
+.TP
+.B random
+Choose a placement ID at random (uniform).
+.TP
+.B roundrobin
+Round robin over available placement IDs. This is the default.
+.RE
+.P
+The available placement ID index/indices is defined by \fBfdp_pli\fR option.
+.RE
+.TP
+.BI (io_uring_cmd,xnvme)fdp_pli \fR=\fPstr
 Select which Placement ID Index/Indicies this job is allowed to use for writes.
 By default, the job will cycle through all available Placement IDs, so use this
 to isolate these identifiers to specific jobs. If you want fio to use placement
@@ -4168,7 +4184,7 @@ They will look like this:
 .P
 .nf
 		  Disk stats (read/write):
-		    sda: ios=16398/16511, merge=30/162, ticks=6853/819634, in_queue=826487, util=100.00%
+		    sda: ios=16398/16511, sectors=32321/65472, merge=30/162, ticks=6853/819634, in_queue=826487, util=100.00%
 .fi
 .P
 Each value is printed for both reads and writes, with reads first. The
diff --git a/fio.h b/fio.h
index c5453d13..a54f57c9 100644
--- a/fio.h
+++ b/fio.h
@@ -144,6 +144,7 @@ enum {
 	FIO_RAND_POISSON3_OFF,
 	FIO_RAND_PRIO_CMDS,
 	FIO_RAND_DEDUPE_WORKING_SET_IX,
+	FIO_RAND_FDP_OFF,
 	FIO_RAND_NR_OFFS,
 };
 
@@ -262,6 +263,7 @@ struct thread_data {
 	struct frand_state verify_state_last_do_io;
 	struct frand_state trim_state;
 	struct frand_state delay_state;
+	struct frand_state fdp_state;
 
 	struct frand_state buf_state;
 	struct frand_state buf_state_prev;
diff --git a/init.c b/init.c
index 10e63cca..105339fa 100644
--- a/init.c
+++ b/init.c
@@ -1082,6 +1082,8 @@ void td_fill_rand_seeds(struct thread_data *td)
 
 	init_rand_seed(&td->buf_state, td->rand_seeds[FIO_RAND_BUF_OFF], use64);
 	frand_copy(&td->buf_state_prev, &td->buf_state);
+
+	init_rand_seed(&td->fdp_state, td->rand_seeds[FIO_RAND_FDP_OFF], use64);
 }
 
 static int setup_random_seeds(struct thread_data *td)
diff --git a/options.c b/options.c
index a7c4ef6e..0f739317 100644
--- a/options.c
+++ b/options.c
@@ -3679,6 +3679,26 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.category = FIO_OPT_C_IO,
 		.group  = FIO_OPT_G_INVALID,
 	},
+	{
+		.name	= "fdp_pli_select",
+		.lname	= "FDP Placement ID select",
+		.type	= FIO_OPT_STR,
+		.off1	= offsetof(struct thread_options, fdp_pli_select),
+		.help	= "Select which FDP placement ID to use next",
+		.def	= "roundrobin",
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_INVALID,
+		.posval	= {
+			  { .ival = "random",
+			    .oval = FIO_FDP_RANDOM,
+			    .help = "Choose a Placement ID at random (uniform)",
+			  },
+			  { .ival = "roundrobin",
+			    .oval = FIO_FDP_RR,
+			    .help = "Round robin select Placement IDs",
+			  },
+		},
+	},
 	{
 		.name	= "fdp_pli",
 		.lname	= "FDP Placement ID indicies",
diff --git a/stat.c b/stat.c
index 015b8e28..ced73645 100644
--- a/stat.c
+++ b/stat.c
@@ -1030,11 +1030,14 @@ void print_disk_util(struct disk_util_stat *dus, struct disk_util_agg *agg,
 		if (agg->slavecount)
 			log_buf(out, "  ");
 
-		log_buf(out, "  %s: ios=%llu/%llu, merge=%llu/%llu, "
-			 "ticks=%llu/%llu, in_queue=%llu, util=%3.2f%%",
+		log_buf(out, "  %s: ios=%llu/%llu, sectors=%llu/%llu, "
+			"merge=%llu/%llu, ticks=%llu/%llu, in_queue=%llu, "
+			"util=%3.2f%%",
 				dus->name,
 				(unsigned long long) dus->s.ios[0],
 				(unsigned long long) dus->s.ios[1],
+				(unsigned long long) dus->s.sectors[0],
+				(unsigned long long) dus->s.sectors[1],
 				(unsigned long long) dus->s.merges[0],
 				(unsigned long long) dus->s.merges[1],
 				(unsigned long long) dus->s.ticks[0],
diff --git a/thread_options.h b/thread_options.h
index a24ebee6..1715b36c 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -388,6 +388,7 @@ struct thread_options {
 
 #define FIO_MAX_PLIS 16
 	unsigned int fdp;
+	unsigned int fdp_pli_select;
 	unsigned int fdp_plis[FIO_MAX_PLIS];
 	unsigned int fdp_nrpli;
 
@@ -703,6 +704,7 @@ struct thread_options_pack {
 	uint32_t log_prio;
 
 	uint32_t fdp;
+	uint32_t fdp_pli_select;
 	uint32_t fdp_plis[FIO_MAX_PLIS];
 	uint32_t fdp_nrpli;
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-06 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-06 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 41508de67c06661ff1d473d108a8a01912ade114:

  fio/server: fix confusing sk_out check (2023-07-03 09:16:45 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 8e2b81b854286f32eae7951a434dddebd968f9d5:

  zbd: Support finishing zones on Android (2023-07-05 15:48:11 -0600)

----------------------------------------------------------------
Bart Van Assche (1):
      zbd: Support finishing zones on Android

Jens Axboe (1):
      Merge branch 'makefile-hardening-cpp-flags' of https://github.com/proact-de/fio

Martin Steigerwald (1):
      Keep C pre processor hardening build flags.

Vincent Fu (4):
      engines/io_uring_cmd: make trims async
      engines/io_uring: remove dead code related to trim
      t/nvmept: add check for iodepth
      t/nvmept: add trim test with ioengine options enabled

 Makefile               |  2 +-
 engines/io_uring.c     | 49 ++++++++++----------------
 engines/nvme.c         | 96 ++++++++++++++++++++++++--------------------------
 engines/nvme.h         |  5 +--
 oslib/linux-blkzoned.c | 24 ++++++-------
 t/nvmept.py            | 21 +++++++++++
 6 files changed, 100 insertions(+), 97 deletions(-)

---

Diff of recent changes:

diff --git a/Makefile b/Makefile
index 6d7fd4e2..cc8164b2 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ include config-host.mak
 endif
 
 DEBUGFLAGS = -DFIO_INC_DEBUG
-CPPFLAGS= -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DFIO_INTERNAL $(DEBUGFLAGS)
+CPPFLAGS+= -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DFIO_INTERNAL $(DEBUGFLAGS)
 OPTFLAGS= -g -ffast-math
 FIO_CFLAGS= -std=gnu99 -Wwrite-strings -Wall -Wdeclaration-after-statement $(OPTFLAGS) $(EXTFLAGS) $(BUILD_CFLAGS) -I. -I$(SRCDIR)
 LIBS	+= -lm $(EXTLIBS)
diff --git a/engines/io_uring.c b/engines/io_uring.c
index 73e4a27a..5021239e 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -78,6 +78,8 @@ struct ioring_data {
 	struct ioring_mmap mmap[3];
 
 	struct cmdprio cmdprio;
+
+	struct nvme_dsm_range *dsm;
 };
 
 struct ioring_options {
@@ -410,7 +412,7 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
 	if (o->cmd_type != FIO_URING_CMD_NVME)
 		return -EINVAL;
 
-	if (io_u->ddir == DDIR_TRIM)
+	if (io_u->ddir == DDIR_TRIM && td->io_ops->flags & FIO_ASYNCIO_SYNC_TRIM)
 		return 0;
 
 	sqe = &ld->sqes[(io_u->index) << 1];
@@ -444,7 +446,8 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
 
 	cmd = (struct nvme_uring_cmd *)sqe->cmd;
 	return fio_nvme_uring_cmd_prep(cmd, io_u,
-			o->nonvectored ? NULL : &ld->iovecs[io_u->index]);
+			o->nonvectored ? NULL : &ld->iovecs[io_u->index],
+			&ld->dsm[io_u->index]);
 }
 
 static struct io_u *fio_ioring_event(struct thread_data *td, int event)
@@ -561,27 +564,6 @@ static inline void fio_ioring_cmdprio_prep(struct thread_data *td,
 		ld->sqes[io_u->index].ioprio = io_u->ioprio;
 }
 
-static int fio_ioring_cmd_io_u_trim(struct thread_data *td,
-				    struct io_u *io_u)
-{
-	struct fio_file *f = io_u->file;
-	int ret;
-
-	if (td->o.zone_mode == ZONE_MODE_ZBD) {
-		ret = zbd_do_io_u_trim(td, io_u);
-		if (ret == io_u_completed)
-			return io_u->xfer_buflen;
-		if (ret)
-			goto err;
-	}
-
-	return fio_nvme_trim(td, f, io_u->offset, io_u->xfer_buflen);
-
-err:
-	io_u->error = ret;
-	return 0;
-}
-
 static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 					  struct io_u *io_u)
 {
@@ -594,14 +576,11 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 	if (ld->queued == ld->iodepth)
 		return FIO_Q_BUSY;
 
-	if (io_u->ddir == DDIR_TRIM) {
+	if (io_u->ddir == DDIR_TRIM && td->io_ops->flags & FIO_ASYNCIO_SYNC_TRIM) {
 		if (ld->queued)
 			return FIO_Q_BUSY;
 
-		if (!strcmp(td->io_ops->name, "io_uring_cmd"))
-			fio_ioring_cmd_io_u_trim(td, io_u);
-		else
-			do_io_u_trim(td, io_u);
+		do_io_u_trim(td, io_u);
 
 		io_u_mark_submit(td, 1);
 		io_u_mark_complete(td, 1);
@@ -734,6 +713,7 @@ static void fio_ioring_cleanup(struct thread_data *td)
 		free(ld->io_u_index);
 		free(ld->iovecs);
 		free(ld->fds);
+		free(ld->dsm);
 		free(ld);
 	}
 }
@@ -1146,6 +1126,16 @@ static int fio_ioring_init(struct thread_data *td)
 		return 1;
 	}
 
+	/*
+	 * For io_uring_cmd, trims are async operations unless we are operating
+	 * in zbd mode where trim means zone reset.
+	 */
+	if (!strcmp(td->io_ops->name, "io_uring_cmd") && td_trim(td) &&
+	    td->o.zone_mode == ZONE_MODE_ZBD)
+		td->io_ops->flags |= FIO_ASYNCIO_SYNC_TRIM;
+	else
+		ld->dsm = calloc(ld->iodepth, sizeof(*ld->dsm));
+
 	return 0;
 }
 
@@ -1361,8 +1351,7 @@ static struct ioengine_ops ioengine_uring = {
 static struct ioengine_ops ioengine_uring_cmd = {
 	.name			= "io_uring_cmd",
 	.version		= FIO_IOOPS_VERSION,
-	.flags			= FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD |
-					FIO_MEMALIGN | FIO_RAWIO |
+	.flags			= FIO_NO_OFFLOAD | FIO_MEMALIGN | FIO_RAWIO |
 					FIO_ASYNCIO_SETS_ISSUE_TIME,
 	.init			= fio_ioring_init,
 	.post_init		= fio_ioring_cmd_post_init,
diff --git a/engines/nvme.c b/engines/nvme.c
index 1047ade2..b18ad4c2 100644
--- a/engines/nvme.c
+++ b/engines/nvme.c
@@ -5,8 +5,41 @@
 
 #include "nvme.h"
 
+static inline __u64 get_slba(struct nvme_data *data, struct io_u *io_u)
+{
+	if (data->lba_ext)
+		return io_u->offset / data->lba_ext;
+	else
+		return io_u->offset >> data->lba_shift;
+}
+
+static inline __u32 get_nlb(struct nvme_data *data, struct io_u *io_u)
+{
+	if (data->lba_ext)
+		return io_u->xfer_buflen / data->lba_ext - 1;
+	else
+		return (io_u->xfer_buflen >> data->lba_shift) - 1;
+}
+
+void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
+				  struct nvme_dsm_range *dsm)
+{
+	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
+
+	cmd->opcode = nvme_cmd_dsm;
+	cmd->nsid = data->nsid;
+	cmd->cdw10 = 0;
+	cmd->cdw11 = NVME_ATTRIBUTE_DEALLOCATE;
+	cmd->addr = (__u64) (uintptr_t) dsm;
+	cmd->data_len = sizeof(*dsm);
+
+	dsm->slba = get_slba(data, io_u);
+	/* nlb is a 1-based value for deallocate */
+	dsm->nlb = get_nlb(data, io_u) + 1;
+}
+
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
-			    struct iovec *iov)
+			    struct iovec *iov, struct nvme_dsm_range *dsm)
 {
 	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
 	__u64 slba;
@@ -14,21 +47,23 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 
 	memset(cmd, 0, sizeof(struct nvme_uring_cmd));
 
-	if (io_u->ddir == DDIR_READ)
+	switch (io_u->ddir) {
+	case DDIR_READ:
 		cmd->opcode = nvme_cmd_read;
-	else if (io_u->ddir == DDIR_WRITE)
+		break;
+	case DDIR_WRITE:
 		cmd->opcode = nvme_cmd_write;
-	else
+		break;
+	case DDIR_TRIM:
+		fio_nvme_uring_cmd_trim_prep(cmd, io_u, dsm);
+		return 0;
+	default:
 		return -ENOTSUP;
-
-	if (data->lba_ext) {
-		slba = io_u->offset / data->lba_ext;
-		nlb = (io_u->xfer_buflen / data->lba_ext) - 1;
-	} else {
-		slba = io_u->offset >> data->lba_shift;
-		nlb = (io_u->xfer_buflen >> data->lba_shift) - 1;
 	}
 
+	slba = get_slba(data, io_u);
+	nlb = get_nlb(data, io_u);
+
 	/* cdw10 and cdw11 represent starting lba */
 	cmd->cdw10 = slba & 0xffffffff;
 	cmd->cdw11 = slba >> 32;
@@ -48,45 +83,6 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 	return 0;
 }
 
-static int nvme_trim(int fd, __u32 nsid, __u32 nr_range, __u32 data_len,
-		     void *data)
-{
-	struct nvme_passthru_cmd cmd = {
-		.opcode		= nvme_cmd_dsm,
-		.nsid		= nsid,
-		.addr		= (__u64)(uintptr_t)data,
-		.data_len 	= data_len,
-		.cdw10		= nr_range - 1,
-		.cdw11		= NVME_ATTRIBUTE_DEALLOCATE,
-	};
-
-	return ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
-}
-
-int fio_nvme_trim(const struct thread_data *td, struct fio_file *f,
-		  unsigned long long offset, unsigned long long len)
-{
-	struct nvme_data *data = FILE_ENG_DATA(f);
-	struct nvme_dsm_range dsm;
-	int ret;
-
-	if (data->lba_ext) {
-		dsm.nlb = len / data->lba_ext;
-		dsm.slba = offset / data->lba_ext;
-	} else {
-		dsm.nlb = len >> data->lba_shift;
-		dsm.slba = offset >> data->lba_shift;
-	}
-
-	ret = nvme_trim(f->fd, data->nsid, 1, sizeof(struct nvme_dsm_range),
-			&dsm);
-	if (ret)
-		log_err("%s: nvme_trim failed for offset %llu and len %llu, err=%d\n",
-			f->file_name, offset, len, ret);
-
-	return ret;
-}
-
 static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
 			 enum nvme_csi csi, void *data)
 {
diff --git a/engines/nvme.h b/engines/nvme.h
index f7cb820d..238471dd 100644
--- a/engines/nvme.h
+++ b/engines/nvme.h
@@ -216,9 +216,6 @@ struct nvme_dsm_range {
 	__le64	slba;
 };
 
-int fio_nvme_trim(const struct thread_data *td, struct fio_file *f,
-		  unsigned long long offset, unsigned long long len);
-
 int fio_nvme_iomgmt_ruhs(struct thread_data *td, struct fio_file *f,
 			 struct nvme_fdp_ruh_status *ruhs, __u32 bytes);
 
@@ -226,7 +223,7 @@ int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
 		      __u32 *ms, __u64 *nlba);
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
-			    struct iovec *iov);
+			    struct iovec *iov, struct nvme_dsm_range *dsm);
 
 int fio_nvme_get_zoned_model(struct thread_data *td, struct fio_file *f,
 			     enum zbd_zoned_model *model);
diff --git a/oslib/linux-blkzoned.c b/oslib/linux-blkzoned.c
index c3130d0e..722e0992 100644
--- a/oslib/linux-blkzoned.c
+++ b/oslib/linux-blkzoned.c
@@ -22,6 +22,9 @@
 #include "zbd_types.h"
 
 #include <linux/blkzoned.h>
+#ifndef BLKFINISHZONE
+#define BLKFINISHZONE _IOW(0x12, 136, struct blk_zone_range)
+#endif
 
 /*
  * If the uapi headers installed on the system lacks zone capacity support,
@@ -312,7 +315,6 @@ int blkzoned_reset_wp(struct thread_data *td, struct fio_file *f,
 int blkzoned_finish_zone(struct thread_data *td, struct fio_file *f,
 			 uint64_t offset, uint64_t length)
 {
-#ifdef BLKFINISHZONE
 	struct blk_zone_range zr = {
 		.sector         = offset >> 9,
 		.nr_sectors     = length >> 9,
@@ -327,21 +329,19 @@ int blkzoned_finish_zone(struct thread_data *td, struct fio_file *f,
 			return -errno;
 	}
 
-	if (ioctl(fd, BLKFINISHZONE, &zr) < 0)
+	if (ioctl(fd, BLKFINISHZONE, &zr) < 0) {
 		ret = -errno;
+		/*
+		 * Kernel versions older than 5.5 do not support BLKFINISHZONE
+		 * and return the ENOTTY error code. These old kernels only
+		 * support block devices that close zones automatically.
+		 */
+		if (ret == ENOTTY)
+			ret = 0;
+	}
 
 	if (f->fd < 0)
 		close(fd);
 
 	return ret;
-#else
-	/*
-	 * Kernel versions older than 5.5 does not support BLKFINISHZONE. These
-	 * old kernels assumed zones are closed automatically at max_open_zones
-	 * limit. Also they did not support max_active_zones limit. Then there
-	 * was no need to finish zones to avoid errors caused by max_open_zones
-	 * or max_active_zones. For those old versions, just do nothing.
-	 */
-	return 0;
-#endif
 }
diff --git a/t/nvmept.py b/t/nvmept.py
index e235d160..cc26d152 100755
--- a/t/nvmept.py
+++ b/t/nvmept.py
@@ -80,6 +80,10 @@ class PassThruTest(FioJobCmdTest):
             print(f"Unhandled rw value {self.fio_opts['rw']}")
             self.passed = False
 
+        if job['iodepth_level']['8'] < 95:
+            print("Did not achieve requested iodepth")
+            self.passed = False
+
 
 TEST_LIST = [
     {
@@ -232,6 +236,23 @@ TEST_LIST = [
             },
         "test_class": PassThruTest,
     },
+    {
+        # We can't enable fixedbufs because for trim-only
+        # workloads fio actually does not allocate any buffers
+        "test_id": 15,
+        "fio_opts": {
+            "rw": 'randtrim',
+            "timebased": 1,
+            "runtime": 3,
+            "fixedbufs": 0,
+            "nonvectored": 1,
+            "force_async": 1,
+            "registerfiles": 1,
+            "sqthread_poll": 1,
+            "output-format": "json",
+            },
+        "test_class": PassThruTest,
+    },
 ]
 
 def parse_args():

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-07-04 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-07-04 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 5087502fb05b2b4d756045c594a2e09c2ffc97dc:

  init: don't adjust time units again for subjobs (2023-06-20 14:11:36 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 41508de67c06661ff1d473d108a8a01912ade114:

  fio/server: fix confusing sk_out check (2023-07-03 09:16:45 -0600)

----------------------------------------------------------------
Denis Pronin (3):
      fixed compiler warnings if NDEBUG enabled in core code
      fixed compiler warnings if NDEBUG enabled in test code
      use 'min' macro to find out next value of actual_min in libaio

Jens Axboe (3):
      Merge branch 'libaio/actual_min_algo_update' of https://github.com/dpronin/fio
      Merge branch 'improvement/fix-warnings-if-NDEBUG-enabled' of https://github.com/dpronin/fio
      fio/server: fix confusing sk_out check

 backend.c              | 18 ++++++++++++++----
 engines/libaio.c       |  2 +-
 helper_thread.c        |  8 +++++++-
 io_u.c                 |  7 ++++---
 ioengines.c            | 10 ++++++++--
 rate-submit.c          | 18 +++++++++++++++---
 server.c               |  7 ++++++-
 t/read-to-pipe-async.c | 30 +++++++++++++++++++++++-------
 zbd.c                  | 18 ++++++++----------
 9 files changed, 86 insertions(+), 32 deletions(-)

---

Diff of recent changes:

diff --git a/backend.c b/backend.c
index f541676c..d67a4a07 100644
--- a/backend.c
+++ b/backend.c
@@ -1633,7 +1633,7 @@ static void *thread_main(void *data)
 	uint64_t bytes_done[DDIR_RWDIR_CNT];
 	int deadlock_loop_cnt;
 	bool clear_state;
-	int res, ret;
+	int ret;
 
 	sk_out_assign(sk_out);
 	free(fd);
@@ -1974,13 +1974,23 @@ static void *thread_main(void *data)
 	 * another thread is checking its io_u's for overlap
 	 */
 	if (td_offload_overlap(td)) {
-		int res = pthread_mutex_lock(&overlap_check);
-		assert(res == 0);
+		int res;
+
+		res = pthread_mutex_lock(&overlap_check);
+		if (res) {
+			td->error = errno;
+			goto err;
+		}
 	}
 	td_set_runstate(td, TD_FINISHING);
 	if (td_offload_overlap(td)) {
+		int res;
+
 		res = pthread_mutex_unlock(&overlap_check);
-		assert(res == 0);
+		if (res) {
+			td->error = errno;
+			goto err;
+		}
 	}
 
 	update_rusage_stat(td);
diff --git a/engines/libaio.c b/engines/libaio.c
index 1b82c90b..6a0745aa 100644
--- a/engines/libaio.c
+++ b/engines/libaio.c
@@ -296,7 +296,7 @@ static int fio_libaio_getevents(struct thread_data *td, unsigned int min,
 		}
 		if (r > 0) {
 			events += r;
-			actual_min = actual_min > events ? actual_min - events : 0;
+			actual_min -= min((unsigned int)events, actual_min);
 		}
 		else if ((min && r == 0) || r == -EAGAIN) {
 			fio_libaio_commit(td);
diff --git a/helper_thread.c b/helper_thread.c
index 77016638..53dea44b 100644
--- a/helper_thread.c
+++ b/helper_thread.c
@@ -1,4 +1,7 @@
+#include <errno.h>
 #include <signal.h>
+#include <stdio.h>
+#include <string.h>
 #include <unistd.h>
 #ifdef CONFIG_HAVE_TIMERFD_CREATE
 #include <sys/timerfd.h>
@@ -122,7 +125,10 @@ static void submit_action(enum action a)
 		return;
 
 	ret = write_to_pipe(helper_data->pipe[1], &data, sizeof(data));
-	assert(ret == 1);
+	if (ret != 1) {
+		log_err("failed to write action into pipe, err %i:%s", errno, strerror(errno));
+		assert(0);
+	}
 }
 
 void helper_reset(void)
diff --git a/io_u.c b/io_u.c
index faf512e5..27b6c92a 100644
--- a/io_u.c
+++ b/io_u.c
@@ -1613,7 +1613,6 @@ struct io_u *__get_io_u(struct thread_data *td)
 {
 	const bool needs_lock = td_async_processing(td);
 	struct io_u *io_u = NULL;
-	int ret;
 
 	if (td->stop_io)
 		return NULL;
@@ -1647,14 +1646,16 @@ again:
 		io_u_set(td, io_u, IO_U_F_IN_CUR_DEPTH);
 		io_u->ipo = NULL;
 	} else if (td_async_processing(td)) {
+		int ret;
 		/*
 		 * We ran out, wait for async verify threads to finish and
 		 * return one
 		 */
 		assert(!(td->flags & TD_F_CHILD));
 		ret = pthread_cond_wait(&td->free_cond, &td->io_u_lock);
-		assert(ret == 0);
-		if (!td->error)
+		if (fio_unlikely(ret != 0)) {
+			td->error = errno;
+		} else if (!td->error)
 			goto again;
 	}
 
diff --git a/ioengines.c b/ioengines.c
index 742f97dd..36172725 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -17,6 +17,7 @@
 #include <assert.h>
 #include <sys/types.h>
 #include <dirent.h>
+#include <errno.h>
 
 #include "fio.h"
 #include "diskutil.h"
@@ -342,8 +343,13 @@ enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u)
 	 * flag is now set
 	 */
 	if (td_offload_overlap(td)) {
-		int res = pthread_mutex_unlock(&overlap_check);
-		assert(res == 0);
+		int res;
+
+		res = pthread_mutex_unlock(&overlap_check);
+		if (fio_unlikely(res != 0)) {
+			log_err("failed to unlock overlap check mutex, err: %i:%s", errno, strerror(errno));
+			abort();
+		}
 	}
 
 	assert(fio_file_open(io_u->file));
diff --git a/rate-submit.c b/rate-submit.c
index 103a80aa..6f6d15bd 100644
--- a/rate-submit.c
+++ b/rate-submit.c
@@ -5,6 +5,9 @@
  *
  */
 #include <assert.h>
+#include <errno.h>
+#include <pthread.h>
+
 #include "fio.h"
 #include "ioengines.h"
 #include "lib/getrusage.h"
@@ -27,7 +30,10 @@ static void check_overlap(struct io_u *io_u)
 	 * threads as they assess overlap.
 	 */
 	res = pthread_mutex_lock(&overlap_check);
-	assert(res == 0);
+	if (fio_unlikely(res != 0)) {
+		log_err("failed to lock overlap check mutex, err: %i:%s", errno, strerror(errno));
+		abort();
+	}
 
 retry:
 	for_each_td(td) {
@@ -41,9 +47,15 @@ retry:
 			continue;
 
 		res = pthread_mutex_unlock(&overlap_check);
-		assert(res == 0);
+		if (fio_unlikely(res != 0)) {
+			log_err("failed to unlock overlap check mutex, err: %i:%s", errno, strerror(errno));
+			abort();
+		}
 		res = pthread_mutex_lock(&overlap_check);
-		assert(res == 0);
+		if (fio_unlikely(res != 0)) {
+			log_err("failed to lock overlap check mutex, err: %i:%s", errno, strerror(errno));
+			abort();
+		}
 		goto retry;
 	} end_for_each();
 }
diff --git a/server.c b/server.c
index a6347efd..bb423702 100644
--- a/server.c
+++ b/server.c
@@ -1,5 +1,6 @@
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 #include <errno.h>
 #include <poll.h>
@@ -2343,7 +2344,11 @@ void fio_server_send_start(struct thread_data *td)
 {
 	struct sk_out *sk_out = pthread_getspecific(sk_out_key);
 
-	assert(sk_out->sk != -1);
+	if (sk_out->sk == -1) {
+		log_err("pthread getting specific for key failed, sk_out %p, sk %i, err: %i:%s",
+			sk_out, sk_out->sk, errno, strerror(errno));
+		abort();
+	}
 
 	fio_net_queue_cmd(FIO_NET_CMD_SERVER_START, NULL, 0, NULL, SK_F_SIMPLE);
 }
diff --git a/t/read-to-pipe-async.c b/t/read-to-pipe-async.c
index 569fc62a..de98d032 100644
--- a/t/read-to-pipe-async.c
+++ b/t/read-to-pipe-async.c
@@ -36,6 +36,8 @@
 
 #include "../flist.h"
 
+#include "compiler/compiler.h"
+
 static int bs = 4096;
 static int max_us = 10000;
 static char *file;
@@ -47,6 +49,18 @@ static int separate_writer = 1;
 #define PLAT_NR		(PLAT_GROUP_NR * PLAT_VAL)
 #define PLAT_LIST_MAX	20
 
+#ifndef NDEBUG
+#define CHECK_ZERO_OR_ABORT(code) assert(code)
+#else
+#define CHECK_ZERO_OR_ABORT(code) 										\
+	do { 																\
+		if (fio_unlikely((code) != 0)) { 								\
+			log_err("failed checking code %i != 0", (code)); 	\
+			abort();													\
+		} 																\
+	} while (0)
+#endif
+
 struct stats {
 	unsigned int plat[PLAT_NR];
 	unsigned int nr_samples;
@@ -121,7 +135,7 @@ uint64_t utime_since(const struct timespec *s, const struct timespec *e)
 	return ret;
 }
 
-static struct work_item *find_seq(struct writer_thread *w, unsigned int seq)
+static struct work_item *find_seq(struct writer_thread *w, int seq)
 {
 	struct work_item *work;
 	struct flist_head *entry;
@@ -224,6 +238,8 @@ static int write_work(struct work_item *work)
 
 	clock_gettime(CLOCK_MONOTONIC, &s);
 	ret = write(STDOUT_FILENO, work->buf, work->buf_size);
+	if (ret < 0)
+		return (int)ret;
 	clock_gettime(CLOCK_MONOTONIC, &e);
 	assert(ret == work->buf_size);
 
@@ -241,10 +257,10 @@ static void *writer_fn(void *data)
 {
 	struct writer_thread *wt = data;
 	struct work_item *work;
-	unsigned int seq = 1;
+	int seq = 1;
 
 	work = NULL;
-	while (!wt->thread.exit || !flist_empty(&wt->list)) {
+	while (!(seq < 0) && (!wt->thread.exit || !flist_empty(&wt->list))) {
 		pthread_mutex_lock(&wt->thread.lock);
 
 		if (work)
@@ -467,10 +483,10 @@ static void init_thread(struct thread_data *thread)
 	int ret;
 
 	ret = pthread_condattr_init(&cattr);
-	assert(ret == 0);
+	CHECK_ZERO_OR_ABORT(ret);
 #ifdef CONFIG_PTHREAD_CONDATTR_SETCLOCK
 	ret = pthread_condattr_setclock(&cattr, CLOCK_MONOTONIC);
-	assert(ret == 0);
+	CHECK_ZERO_OR_ABORT(ret);
 #endif
 	pthread_cond_init(&thread->cond, &cattr);
 	pthread_cond_init(&thread->done_cond, &cattr);
@@ -624,10 +640,10 @@ int main(int argc, char *argv[])
 	bytes = 0;
 
 	ret = pthread_condattr_init(&cattr);
-	assert(ret == 0);
+	CHECK_ZERO_OR_ABORT(ret);
 #ifdef CONFIG_PTHREAD_CONDATTR_SETCLOCK
 	ret = pthread_condattr_setclock(&cattr, CLOCK_MONOTONIC);
-	assert(ret == 0);
+	CHECK_ZERO_OR_ABORT(ret);
 #endif
 
 	clock_gettime(CLOCK_MONOTONIC, &s);
diff --git a/zbd.c b/zbd.c
index 7fcf1ec4..d4565215 100644
--- a/zbd.c
+++ b/zbd.c
@@ -11,6 +11,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "compiler/compiler.h"
 #include "os/os.h"
 #include "file.h"
 #include "fio.h"
@@ -102,13 +103,13 @@ static bool zbd_zone_full(const struct fio_file *f, struct fio_zone_info *z,
 static void zone_lock(struct thread_data *td, const struct fio_file *f,
 		      struct fio_zone_info *z)
 {
+#ifndef NDEBUG
 	struct zoned_block_device_info *zbd = f->zbd_info;
-	uint32_t nz = z - zbd->zone_info;
-
+	uint32_t const nz = z - zbd->zone_info;
 	/* A thread should never lock zones outside its working area. */
 	assert(f->min_zone <= nz && nz < f->max_zone);
-
 	assert(z->has_wp);
+#endif
 
 	/*
 	 * Lock the io_u target zone. The zone will be unlocked if io_u offset
@@ -128,11 +129,8 @@ static void zone_lock(struct thread_data *td, const struct fio_file *f,
 
 static inline void zone_unlock(struct fio_zone_info *z)
 {
-	int ret;
-
 	assert(z->has_wp);
-	ret = pthread_mutex_unlock(&z->mutex);
-	assert(!ret);
+	pthread_mutex_unlock(&z->mutex);
 }
 
 static inline struct fio_zone_info *zbd_get_zone(const struct fio_file *f,
@@ -420,7 +418,8 @@ static int zbd_reset_zones(struct thread_data *td, struct fio_file *f,
 	const uint64_t min_bs = td->o.min_bs[DDIR_WRITE];
 	int res = 0;
 
-	assert(min_bs);
+	if (fio_unlikely(0 == min_bs))
+		return 1;
 
 	dprint(FD_ZBD, "%s: examining zones %u .. %u\n",
 	       f->file_name, zbd_zone_idx(f, zb), zbd_zone_idx(f, ze));
@@ -1714,10 +1713,9 @@ unlock:
 static void zbd_put_io(struct thread_data *td, const struct io_u *io_u)
 {
 	const struct fio_file *f = io_u->file;
-	struct zoned_block_device_info *zbd_info = f->zbd_info;
 	struct fio_zone_info *z;
 
-	assert(zbd_info);
+	assert(f->zbd_info);
 
 	z = zbd_offset_to_zone(f, io_u->offset);
 	assert(z->has_wp);

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-06-22 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-06-22 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 8ce9c4003aeaafa91c3278c1c7de4a32fadc5ea0:

  docs: clarify opendir description (2023-06-16 10:41:25 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 5087502fb05b2b4d756045c594a2e09c2ffc97dc:

  init: don't adjust time units again for subjobs (2023-06-20 14:11:36 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      init: don't adjust time units again for subjobs

 init.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

---

Diff of recent changes:

diff --git a/init.c b/init.c
index 437406ec..10e63cca 100644
--- a/init.c
+++ b/init.c
@@ -951,13 +951,16 @@ static int fixup_options(struct thread_data *td)
 	if (o->disable_slat)
 		o->slat_percentiles = 0;
 
-	/*
-	 * Fix these up to be nsec internally
-	 */
-	for_each_rw_ddir(ddir)
-		o->max_latency[ddir] *= 1000ULL;
+	/* Do this only for the parent job */
+	if (!td->subjob_number) {
+		/*
+		 * Fix these up to be nsec internally
+		 */
+		for_each_rw_ddir(ddir)
+			o->max_latency[ddir] *= 1000ULL;
 
-	o->latency_target *= 1000ULL;
+		o->latency_target *= 1000ULL;
+	}
 
 	/*
 	 * Dedupe working set verifications

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-06-17 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-06-17 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 62ac66490f5077e5fca1bd5b49165147cafc5a0d:

  zbd: avoid Coverity defect report (2023-06-09 18:04:45 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 8ce9c4003aeaafa91c3278c1c7de4a32fadc5ea0:

  docs: clarify opendir description (2023-06-16 10:41:25 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      docs: clarify opendir description

 HOWTO.rst | 4 +++-
 fio.1     | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 32fff5ec..2e1e55c2 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -843,7 +843,9 @@ Target file/device
 
 .. option:: opendir=str
 
-	Recursively open any files below directory `str`.
+        Recursively open any files below directory `str`. This accepts only a
+        single directory and unlike related options, colons appearing in the
+        path must not be escaped.
 
 .. option:: lockfile=str
 
diff --git a/fio.1 b/fio.1
index 80bf3371..73b7e8c9 100644
--- a/fio.1
+++ b/fio.1
@@ -627,7 +627,9 @@ generated filenames (with a directory specified) with the source of the
 client connecting. To disable this behavior, set this option to 0.
 .TP
 .BI opendir \fR=\fPstr
-Recursively open any files below directory \fIstr\fR.
+Recursively open any files below directory \fIstr\fR. This accepts only a
+single directory and unlike related options, colons appearing in the path must
+not be escaped.
 .TP
 .BI lockfile \fR=\fPstr
 Fio defaults to not locking any files before it does I/O to them. If a file

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-06-10 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-06-10 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit edaee5b96fd87c3c5fe7f64ec917a175cd9237fc:

  t/zbd: test write zone accounting of trim workload (2023-06-08 14:39:07 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 62ac66490f5077e5fca1bd5b49165147cafc5a0d:

  zbd: avoid Coverity defect report (2023-06-09 18:04:45 -0600)

----------------------------------------------------------------
Shin'ichiro Kawasaki (1):
      zbd: avoid Coverity defect report

 zbd.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

---

Diff of recent changes:

diff --git a/zbd.c b/zbd.c
index 9455140a..7fcf1ec4 100644
--- a/zbd.c
+++ b/zbd.c
@@ -1547,11 +1547,11 @@ retry:
 		dprint(FD_ZBD,
 		       "%s(%s): wait zone write and retry write target zone selection\n",
 		       __func__, f->file_name);
+		should_retry = in_flight;
 		pthread_mutex_unlock(&zbdi->mutex);
 		zone_unlock(z);
 		io_u_quiesce(td);
 		zone_lock(td, f, z);
-		should_retry = in_flight;
 		goto retry;
 	}
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-06-09 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-06-09 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 1b4ba547cf45377fffc7a1e60728369997cc7a9b:

  t/run-fio-tests: address issues identified by pylint (2023-06-01 14:12:41 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to edaee5b96fd87c3c5fe7f64ec917a175cd9237fc:

  t/zbd: test write zone accounting of trim workload (2023-06-08 14:39:07 -0400)

----------------------------------------------------------------
Shin'ichiro Kawasaki (7):
      zbd: rename 'open zones' to 'write zones'
      zbd: do not reset extra zones in open conditions
      zbd: fix write zone accounting of almost full zones
      zbd: fix write zone accounting of trim workload
      t/zbd: reset zones before tests with max_open_zones option
      t/zbd: test write zone accounting of almost full zones
      t/zbd: test write zone accounting of trim workload

Vincent Fu (17):
      t/run-fio-tests: split source file
      t/run-fio-tests: rename FioJobTest to FioJobFileTest
      t/run-fio-tests: move get_file outside of FioJobFileTest
      t/fiotestlib: use dictionaries for filenames and paths
      t/fiotestlib: use 'with' for opening files
      t/fiotestlib: use f-string for formatting
      t/fiotestlib: rearrange constructor and setup steps
      t/fiotestlib: record test command in more useful format
      t/fiotestlib: add class for command-line fio job
      t/random_seed: use logging module for debug prints
      t/random_seed: use methods provided in fiotestlib to run tests
      t/random_seed: fixes from pylint
      t/readonly: adapt to use fiotestlib
      t/nvmept: adapt to use fiotestlib
      t/fiotestlib: add ability to ingest iops logs
      t/strided: adapt to use fiotestlib
      t/strided: increase minumum recommended size to 64MiB

 engines/io_uring.c     |   2 +-
 fio.h                  |   2 +-
 io_u.c                 |   2 +-
 io_u.h                 |   2 +-
 options.c              |   4 +-
 t/fiotestcommon.py     | 176 +++++++++++++
 t/fiotestlib.py        | 485 ++++++++++++++++++++++++++++++++++
 t/nvmept.py            | 447 ++++++++++++--------------------
 t/random_seed.py       | 300 +++++++++------------
 t/readonly.py          | 220 +++++++++-------
 t/run-fio-tests.py     | 644 +++++----------------------------------------
 t/strided.py           | 691 ++++++++++++++++++++++++++++---------------------
 t/zbd/test-zbd-support |  64 ++++-
 zbd.c                  | 292 ++++++++++++---------
 zbd.h                  |  25 +-
 zbd_types.h            |   2 +-
 16 files changed, 1771 insertions(+), 1587 deletions(-)
 create mode 100644 t/fiotestcommon.py
 create mode 100755 t/fiotestlib.py

---

Diff of recent changes:

diff --git a/engines/io_uring.c b/engines/io_uring.c
index ff64fc9f..73e4a27a 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -561,7 +561,7 @@ static inline void fio_ioring_cmdprio_prep(struct thread_data *td,
 		ld->sqes[io_u->index].ioprio = io_u->ioprio;
 }
 
-static int fio_ioring_cmd_io_u_trim(const struct thread_data *td,
+static int fio_ioring_cmd_io_u_trim(struct thread_data *td,
 				    struct io_u *io_u)
 {
 	struct fio_file *f = io_u->file;
diff --git a/fio.h b/fio.h
index 6fc7fb9c..c5453d13 100644
--- a/fio.h
+++ b/fio.h
@@ -275,7 +275,7 @@ struct thread_data {
 	unsigned long long num_unique_pages;
 
 	struct zone_split_index **zone_state_index;
-	unsigned int num_open_zones;
+	unsigned int num_write_zones;
 
 	unsigned int verify_batch;
 	unsigned int trim_batch;
diff --git a/io_u.c b/io_u.c
index 6f5fc94d..faf512e5 100644
--- a/io_u.c
+++ b/io_u.c
@@ -2379,7 +2379,7 @@ int do_io_u_sync(const struct thread_data *td, struct io_u *io_u)
 	return ret;
 }
 
-int do_io_u_trim(const struct thread_data *td, struct io_u *io_u)
+int do_io_u_trim(struct thread_data *td, struct io_u *io_u)
 {
 #ifndef FIO_HAVE_TRIM
 	io_u->error = EINVAL;
diff --git a/io_u.h b/io_u.h
index 55b4d083..b432a540 100644
--- a/io_u.h
+++ b/io_u.h
@@ -162,7 +162,7 @@ void io_u_mark_submit(struct thread_data *, unsigned int);
 bool queue_full(const struct thread_data *);
 
 int do_io_u_sync(const struct thread_data *, struct io_u *);
-int do_io_u_trim(const struct thread_data *, struct io_u *);
+int do_io_u_trim(struct thread_data *, struct io_u *);
 
 #ifdef FIO_INC_DEBUG
 static inline void dprint_io_u(struct io_u *io_u, const char *p)
diff --git a/options.c b/options.c
index 8193fb29..a7c4ef6e 100644
--- a/options.c
+++ b/options.c
@@ -3618,7 +3618,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.lname	= "Per device/file maximum number of open zones",
 		.type	= FIO_OPT_INT,
 		.off1	= offsetof(struct thread_options, max_open_zones),
-		.maxval	= ZBD_MAX_OPEN_ZONES,
+		.maxval	= ZBD_MAX_WRITE_ZONES,
 		.help	= "Limit on the number of simultaneously opened sequential write zones with zonemode=zbd",
 		.def	= "0",
 		.category = FIO_OPT_C_IO,
@@ -3629,7 +3629,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.lname	= "Job maximum number of open zones",
 		.type	= FIO_OPT_INT,
 		.off1	= offsetof(struct thread_options, job_max_open_zones),
-		.maxval	= ZBD_MAX_OPEN_ZONES,
+		.maxval	= ZBD_MAX_WRITE_ZONES,
 		.help	= "Limit on the number of simultaneously opened sequential write zones with zonemode=zbd by one thread/process",
 		.def	= "0",
 		.category = FIO_OPT_C_IO,
diff --git a/t/fiotestcommon.py b/t/fiotestcommon.py
new file mode 100644
index 00000000..f5012c82
--- /dev/null
+++ b/t/fiotestcommon.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""
+fiotestcommon.py
+
+This contains constant definitions, helpers, and a Requirements class that can
+be used to help with running fio tests.
+"""
+
+import os
+import locale
+import logging
+import platform
+import subprocess
+import multiprocessing
+
+
+SUCCESS_DEFAULT = {
+    'zero_return': True,
+    'stderr_empty': True,
+    'timeout': 600,
+    }
+SUCCESS_NONZERO = {
+    'zero_return': False,
+    'stderr_empty': False,
+    'timeout': 600,
+    }
+SUCCESS_STDERR = {
+    'zero_return': True,
+    'stderr_empty': False,
+    'timeout': 600,
+    }
+
+
+def get_file(filename):
+    """Safely read a file."""
+    file_data = ''
+    success = True
+
+    try:
+        with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
+            file_data = output_file.read()
+    except OSError:
+        success = False
+
+    return file_data, success
+
+
+class Requirements():
+    """Requirements consists of multiple run environment characteristics.
+    These are to determine if a particular test can be run"""
+
+    _linux = False
+    _libaio = False
+    _io_uring = False
+    _zbd = False
+    _root = False
+    _zoned_nullb = False
+    _not_macos = False
+    _not_windows = False
+    _unittests = False
+    _cpucount4 = False
+    _nvmecdev = False
+
+    def __init__(self, fio_root, args):
+        Requirements._not_macos = platform.system() != "Darwin"
+        Requirements._not_windows = platform.system() != "Windows"
+        Requirements._linux = platform.system() == "Linux"
+
+        if Requirements._linux:
+            config_file = os.path.join(fio_root, "config-host.h")
+            contents, success = get_file(config_file)
+            if not success:
+                print(f"Unable to open {config_file} to check requirements")
+                Requirements._zbd = True
+            else:
+                Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
+                Requirements._libaio = "CONFIG_LIBAIO" in contents
+
+            contents, success = get_file("/proc/kallsyms")
+            if not success:
+                print("Unable to open '/proc/kallsyms' to probe for io_uring support")
+            else:
+                Requirements._io_uring = "io_uring_setup" in contents
+
+            Requirements._root = os.geteuid() == 0
+            if Requirements._zbd and Requirements._root:
+                try:
+                    subprocess.run(["modprobe", "null_blk"],
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+                    if os.path.exists("/sys/module/null_blk/parameters/zoned"):
+                        Requirements._zoned_nullb = True
+                except Exception:
+                    pass
+
+        if platform.system() == "Windows":
+            utest_exe = "unittest.exe"
+        else:
+            utest_exe = "unittest"
+        unittest_path = os.path.join(fio_root, "unittests", utest_exe)
+        Requirements._unittests = os.path.exists(unittest_path)
+
+        Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
+        Requirements._nvmecdev = args.nvmecdev
+
+        req_list = [
+                Requirements.linux,
+                Requirements.libaio,
+                Requirements.io_uring,
+                Requirements.zbd,
+                Requirements.root,
+                Requirements.zoned_nullb,
+                Requirements.not_macos,
+                Requirements.not_windows,
+                Requirements.unittests,
+                Requirements.cpucount4,
+                Requirements.nvmecdev,
+                    ]
+        for req in req_list:
+            value, desc = req()
+            logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
+
+    @classmethod
+    def linux(cls):
+        """Are we running on Linux?"""
+        return Requirements._linux, "Linux required"
+
+    @classmethod
+    def libaio(cls):
+        """Is libaio available?"""
+        return Requirements._libaio, "libaio required"
+
+    @classmethod
+    def io_uring(cls):
+        """Is io_uring available?"""
+        return Requirements._io_uring, "io_uring required"
+
+    @classmethod
+    def zbd(cls):
+        """Is ZBD support available?"""
+        return Requirements._zbd, "Zoned block device support required"
+
+    @classmethod
+    def root(cls):
+        """Are we running as root?"""
+        return Requirements._root, "root required"
+
+    @classmethod
+    def zoned_nullb(cls):
+        """Are zoned null block devices available?"""
+        return Requirements._zoned_nullb, "Zoned null block device support required"
+
+    @classmethod
+    def not_macos(cls):
+        """Are we running on a platform other than macOS?"""
+        return Requirements._not_macos, "platform other than macOS required"
+
+    @classmethod
+    def not_windows(cls):
+        """Are we running on a platform other than Windws?"""
+        return Requirements._not_windows, "platform other than Windows required"
+
+    @classmethod
+    def unittests(cls):
+        """Were unittests built?"""
+        return Requirements._unittests, "Unittests support required"
+
+    @classmethod
+    def cpucount4(cls):
+        """Do we have at least 4 CPUs?"""
+        return Requirements._cpucount4, "4+ CPUs required"
+
+    @classmethod
+    def nvmecdev(cls):
+        """Do we have an NVMe character device to test?"""
+        return Requirements._nvmecdev, "NVMe character device test target required"
diff --git a/t/fiotestlib.py b/t/fiotestlib.py
new file mode 100755
index 00000000..0fe17b74
--- /dev/null
+++ b/t/fiotestlib.py
@@ -0,0 +1,485 @@
+#!/usr/bin/env python3
+"""
+fiotestlib.py
+
+This library contains FioTest objects that provide convenient means to run
+different sorts of fio tests.
+
+It also contains a test runner that runs an array of dictionary objects
+describing fio tests.
+"""
+
+import os
+import sys
+import json
+import locale
+import logging
+import platform
+import traceback
+import subprocess
+from pathlib import Path
+from fiotestcommon import get_file, SUCCESS_DEFAULT
+
+
+class FioTest():
+    """Base for all fio tests."""
+
+    def __init__(self, exe_path, success, testnum, artifact_root):
+        self.success = success
+        self.testnum = testnum
+        self.output = {}
+        self.passed = True
+        self.failure_reason = ''
+        self.parameters = None
+        self.paths = {
+                        'exe': exe_path,
+                        'artifacts': artifact_root,
+                        'test_dir': os.path.join(artifact_root, \
+                                f"{testnum:04d}"),
+                        }
+        self.filenames = {
+                            'cmd': os.path.join(self.paths['test_dir'], \
+                                    f"{os.path.basename(self.paths['exe'])}.command"),
+                            'stdout': os.path.join(self.paths['test_dir'], \
+                                    f"{os.path.basename(self.paths['exe'])}.stdout"),
+                            'stderr': os.path.join(self.paths['test_dir'], \
+                                    f"{os.path.basename(self.paths['exe'])}.stderr"),
+                            'exitcode': os.path.join(self.paths['test_dir'], \
+                                    f"{os.path.basename(self.paths['exe'])}.exitcode"),
+                            }
+
+    def setup(self, parameters):
+        """Setup instance variables for test."""
+
+        self.parameters = parameters
+        if not os.path.exists(self.paths['test_dir']):
+            os.mkdir(self.paths['test_dir'])
+
+    def run(self):
+        """Run the test."""
+
+        raise NotImplementedError()
+
+    def check_result(self):
+        """Check test results."""
+
+        raise NotImplementedError()
+
+
+class FioExeTest(FioTest):
+    """Test consists of an executable binary or script"""
+
+    def run(self):
+        """Execute the binary or script described by this instance."""
+
+        command = [self.paths['exe']] + self.parameters
+        with open(self.filenames['cmd'], "w+",
+                  encoding=locale.getpreferredencoding()) as command_file:
+            command_file.write(" ".join(command))
+
+        try:
+            with open(self.filenames['stdout'], "w+",
+                      encoding=locale.getpreferredencoding()) as stdout_file, \
+                open(self.filenames['stderr'], "w+",
+                     encoding=locale.getpreferredencoding()) as stderr_file, \
+                open(self.filenames['exitcode'], "w+",
+                     encoding=locale.getpreferredencoding()) as exitcode_file:
+                proc = None
+                # Avoid using subprocess.run() here because when a timeout occurs,
+                # fio will be stopped with SIGKILL. This does not give fio a
+                # chance to clean up and means that child processes may continue
+                # running and submitting IO.
+                proc = subprocess.Popen(command,
+                                        stdout=stdout_file,
+                                        stderr=stderr_file,
+                                        cwd=self.paths['test_dir'],
+                                        universal_newlines=True)
+                proc.communicate(timeout=self.success['timeout'])
+                exitcode_file.write(f'{proc.returncode}\n')
+                logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
+                self.output['proc'] = proc
+        except subprocess.TimeoutExpired:
+            proc.terminate()
+            proc.communicate()
+            assert proc.poll()
+            self.output['failure'] = 'timeout'
+        except Exception:
+            if proc:
+                if not proc.poll():
+                    proc.terminate()
+                    proc.communicate()
+            self.output['failure'] = 'exception'
+            self.output['exc_info'] = sys.exc_info()
+
+    def check_result(self):
+        """Check results of test run."""
+
+        if 'proc' not in self.output:
+            if self.output['failure'] == 'timeout':
+                self.failure_reason = f"{self.failure_reason} timeout,"
+            else:
+                assert self.output['failure'] == 'exception'
+                self.failure_reason = f'{self.failure_reason} exception: ' + \
+                f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
+
+            self.passed = False
+            return
+
+        if 'zero_return' in self.success:
+            if self.success['zero_return']:
+                if self.output['proc'].returncode != 0:
+                    self.passed = False
+                    self.failure_reason = f"{self.failure_reason} non-zero return code,"
+            else:
+                if self.output['proc'].returncode == 0:
+                    self.failure_reason = f"{self.failure_reason} zero return code,"
+                    self.passed = False
+
+        stderr_size = os.path.getsize(self.filenames['stderr'])
+        if 'stderr_empty' in self.success:
+            if self.success['stderr_empty']:
+                if stderr_size != 0:
+                    self.failure_reason = f"{self.failure_reason} stderr not empty,"
+                    self.passed = False
+            else:
+                if stderr_size == 0:
+                    self.failure_reason = f"{self.failure_reason} stderr empty,"
+                    self.passed = False
+
+
+class FioJobFileTest(FioExeTest):
+    """Test consists of a fio job with options in a job file."""
+
+    def __init__(self, fio_path, fio_job, success, testnum, artifact_root,
+                 fio_pre_job=None, fio_pre_success=None,
+                 output_format="normal"):
+        """Construct a FioJobFileTest which is a FioExeTest consisting of a
+        single fio job file with an optional setup step.
+
+        fio_path:           location of fio executable
+        fio_job:            location of fio job file
+        success:            Definition of test success
+        testnum:            test ID
+        artifact_root:      root directory for artifacts
+        fio_pre_job:        fio job for preconditioning
+        fio_pre_success:    Definition of test success for fio precon job
+        output_format:      normal (default), json, jsonplus, or terse
+        """
+
+        self.fio_job = fio_job
+        self.fio_pre_job = fio_pre_job
+        self.fio_pre_success = fio_pre_success if fio_pre_success else success
+        self.output_format = output_format
+        self.precon_failed = False
+        self.json_data = None
+
+        super().__init__(fio_path, success, testnum, artifact_root)
+
+    def setup(self, parameters=None):
+        """Setup instance variables for fio job test."""
+
+        self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
+        fio_args = [
+            "--max-jobs=16",
+            f"--output-format={self.output_format}",
+            f"--output={self.filenames['fio_output']}",
+            self.fio_job,
+            ]
+
+        super().setup(fio_args)
+
+        # Update the filenames from the default
+        self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
+                                             f"{os.path.basename(self.fio_job)}.command")
+        self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
+                                                f"{os.path.basename(self.fio_job)}.stdout")
+        self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
+                                                f"{os.path.basename(self.fio_job)}.stderr")
+        self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
+                                                  f"{os.path.basename(self.fio_job)}.exitcode")
+
+    def run_pre_job(self):
+        """Run fio job precondition step."""
+
+        precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
+                            self.fio_pre_success,
+                            self.testnum,
+                            self.paths['artifacts'],
+                            output_format=self.output_format)
+        precon.setup()
+        precon.run()
+        precon.check_result()
+        self.precon_failed = not precon.passed
+        self.failure_reason = precon.failure_reason
+
+    def run(self):
+        """Run fio job test."""
+
+        if self.fio_pre_job:
+            self.run_pre_job()
+
+        if not self.precon_failed:
+            super().run()
+        else:
+            logging.debug("Test %d: precondition step failed", self.testnum)
+
+    def get_file_fail(self, filename):
+        """Safely read a file and fail the test upon error."""
+        file_data = None
+
+        try:
+            with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
+                file_data = output_file.read()
+        except OSError:
+            self.failure_reason += f" unable to read file {filename}"
+            self.passed = False
+
+        return file_data
+
+    def check_result(self):
+        """Check fio job results."""
+
+        if self.precon_failed:
+            self.passed = False
+            self.failure_reason = f"{self.failure_reason} precondition step failed,"
+            return
+
+        super().check_result()
+
+        if not self.passed:
+            return
+
+        if 'json' not in self.output_format:
+            return
+
+        file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
+                                                    self.filenames['fio_output']))
+        if not file_data:
+            return
+
+        #
+        # Sometimes fio informational messages are included at the top of the
+        # JSON output, especially under Windows. Try to decode output as JSON
+        # data, skipping everything until the first {
+        #
+        lines = file_data.splitlines()
+        file_data = '\n'.join(lines[lines.index("{"):])
+        try:
+            self.json_data = json.loads(file_data)
+        except json.JSONDecodeError:
+            self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
+            self.passed = False
+
+
+class FioJobCmdTest(FioExeTest):
+    """This runs a fio job with options specified on the command line."""
+
+    def __init__(self, fio_path, success, testnum, artifact_root, fio_opts, basename=None):
+
+        self.basename = basename if basename else os.path.basename(fio_path)
+        self.fio_opts = fio_opts
+        self.json_data = None
+        self.iops_log_lines = None
+
+        super().__init__(fio_path, success, testnum, artifact_root)
+
+        filename_stub = os.path.join(self.paths['test_dir'], f"{self.basename}{self.testnum:03d}")
+        self.filenames['cmd'] = f"{filename_stub}.command"
+        self.filenames['stdout'] = f"{filename_stub}.stdout"
+        self.filenames['stderr'] = f"{filename_stub}.stderr"
+        self.filenames['output'] = os.path.abspath(f"{filename_stub}.output")
+        self.filenames['exitcode'] = f"{filename_stub}.exitcode"
+        self.filenames['iopslog'] = os.path.abspath(f"{filename_stub}")
+
+    def run(self):
+        super().run()
+
+        if 'output-format' in self.fio_opts and 'json' in \
+                self.fio_opts['output-format']:
+            if not self.get_json():
+                print('Unable to decode JSON data')
+                self.passed = False
+
+        if any('--write_iops_log=' in param for param in self.parameters):
+            self.get_iops_log()
+
+    def get_iops_log(self):
+        """Read IOPS log from the first job."""
+
+        log_filename = self.filenames['iopslog'] + "_iops.1.log"
+        with open(log_filename, 'r', encoding=locale.getpreferredencoding()) as iops_file:
+            self.iops_log_lines = iops_file.read()
+
+    def get_json(self):
+        """Convert fio JSON output into a python JSON object"""
+
+        filename = self.filenames['output']
+        with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
+            file_data = file.read()
+
+        #
+        # Sometimes fio informational messages are included at the top of the
+        # JSON output, especially under Windows. Try to decode output as JSON
+        # data, lopping off up to the first four lines
+        #
+        lines = file_data.splitlines()
+        for i in range(5):
+            file_data = '\n'.join(lines[i:])
+            try:
+                self.json_data = json.loads(file_data)
+            except json.JSONDecodeError:
+                continue
+            else:
+                return True
+
+        return False
+
+    @staticmethod
+    def check_empty(job):
+        """
+        Make sure JSON data is empty.
+
+        Some data structures should be empty. This function makes sure that they are.
+
+        job         JSON object that we need to check for emptiness
+        """
+
+        return job['total_ios'] == 0 and \
+                job['slat_ns']['N'] == 0 and \
+                job['clat_ns']['N'] == 0 and \
+                job['lat_ns']['N'] == 0
+
+    def check_all_ddirs(self, ddir_nonzero, job):
+        """
+        Iterate over the data directions and check whether each is
+        appropriately empty or not.
+        """
+
+        retval = True
+        ddirlist = ['read', 'write', 'trim']
+
+        for ddir in ddirlist:
+            if ddir in ddir_nonzero:
+                if self.check_empty(job[ddir]):
+                    print(f"Unexpected zero {ddir} data found in output")
+                    retval = False
+            else:
+                if not self.check_empty(job[ddir]):
+                    print(f"Unexpected {ddir} data found in output")
+                    retval = False
+
+        return retval
+
+
+def run_fio_tests(test_list, test_env, args):
+    """
+    Run tests as specified in test_list.
+    """
+
+    passed = 0
+    failed = 0
+    skipped = 0
+
+    for config in test_list:
+        if (args.skip and config['test_id'] in args.skip) or \
+           (args.run_only and config['test_id'] not in args.run_only):
+            skipped = skipped + 1
+            print(f"Test {config['test_id']} SKIPPED (User request)")
+            continue
+
+        if issubclass(config['test_class'], FioJobFileTest):
+            if config['pre_job']:
+                fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
+                                           config['pre_job'])
+            else:
+                fio_pre_job = None
+            if config['pre_success']:
+                fio_pre_success = config['pre_success']
+            else:
+                fio_pre_success = None
+            if 'output_format' in config:
+                output_format = config['output_format']
+            else:
+                output_format = 'normal'
+            test = config['test_class'](
+                test_env['fio_path'],
+                os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
+                config['success'],
+                config['test_id'],
+                test_env['artifact_root'],
+                fio_pre_job=fio_pre_job,
+                fio_pre_success=fio_pre_success,
+                output_format=output_format)
+            desc = config['job']
+            parameters = []
+        elif issubclass(config['test_class'], FioJobCmdTest):
+            if not 'success' in config:
+                config['success'] = SUCCESS_DEFAULT
+            test = config['test_class'](test_env['fio_path'],
+                                        config['success'],
+                                        config['test_id'],
+                                        test_env['artifact_root'],
+                                        config['fio_opts'],
+                                        test_env['basename'])
+            desc = config['test_id']
+            parameters = config
+        elif issubclass(config['test_class'], FioExeTest):
+            exe_path = os.path.join(test_env['fio_root'], config['exe'])
+            parameters = []
+            if config['parameters']:
+                parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
+                              for p in config['parameters']]
+            if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
+                parameters.insert(0, exe_path)
+                exe_path = "python.exe"
+            if config['test_id'] in test_env['pass_through']:
+                parameters += test_env['pass_through'][config['test_id']].split()
+            test = config['test_class'](
+                    exe_path,
+                    config['success'],
+                    config['test_id'],
+                    test_env['artifact_root'])
+            desc = config['exe']
+        else:
+            print(f"Test {config['test_id']} FAILED: unable to process test config")
+            failed = failed + 1
+            continue
+
+        if 'requirements' in config and not args.skip_req:
+            reqs_met = True
+            for req in config['requirements']:
+                reqs_met, reason = req()
+                logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
+                              reqs_met)
+                if not reqs_met:
+                    break
+            if not reqs_met:
+                print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
+                skipped = skipped + 1
+                continue
+
+        try:
+            test.setup(parameters)
+            test.run()
+            test.check_result()
+        except KeyboardInterrupt:
+            break
+        except Exception as e:
+            test.passed = False
+            test.failure_reason += str(e)
+            logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
+        if test.passed:
+            result = "PASSED"
+            passed = passed + 1
+        else:
+            result = f"FAILED: {test.failure_reason}"
+            failed = failed + 1
+            contents, _ = get_file(test.filenames['stderr'])
+            logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
+            contents, _ = get_file(test.filenames['stdout'])
+            logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
+        print(f"Test {config['test_id']} {result} {desc}")
+
+    print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
+
+    return passed, failed, skipped
diff --git a/t/nvmept.py b/t/nvmept.py
index a25192f2..e235d160 100755
--- a/t/nvmept.py
+++ b/t/nvmept.py
@@ -17,42 +17,20 @@
 """
 import os
 import sys
-import json
 import time
-import locale
 import argparse
-import subprocess
 from pathlib import Path
+from fiotestlib import FioJobCmdTest, run_fio_tests
 
-class FioTest():
-    """fio test."""
 
-    def __init__(self, artifact_root, test_opts, debug):
-        """
-        artifact_root   root directory for artifacts (subdirectory will be created under here)
-        test            test specification
-        """
-        self.artifact_root = artifact_root
-        self.test_opts = test_opts
-        self.debug = debug
-        self.filename_stub = None
-        self.filenames = {}
-        self.json_data = None
-
-        self.test_dir = os.path.abspath(os.path.join(self.artifact_root,
-                                     f"{self.test_opts['test_id']:03d}"))
-        if not os.path.exists(self.test_dir):
-            os.mkdir(self.test_dir)
-
-        self.filename_stub = f"pt{self.test_opts['test_id']:03d}"
-        self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command")
-        self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout")
-        self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr")
-        self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode")
-        self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output")
+class PassThruTest(FioJobCmdTest):
+    """
+    NVMe pass-through test class. Check to make sure output for selected data
+    direction(s) is non-zero and that zero data appears for other directions.
+    """
 
-    def run_fio(self, fio_path):
-        """Run a test."""
+    def setup(self, parameters):
+        """Setup a test."""
 
         fio_args = [
             "--name=nvmept",
@@ -61,300 +39,172 @@ class FioTest():
             "--iodepth=8",
             "--iodepth_batch=4",
             "--iodepth_batch_complete=4",
-            f"--filename={self.test_opts['filename']}",
-            f"--rw={self.test_opts['rw']}",
+            f"--filename={self.fio_opts['filename']}",
+            f"--rw={self.fio_opts['rw']}",
             f"--output={self.filenames['output']}",
-            f"--output-format={self.test_opts['output-format']}",
+            f"--output-format={self.fio_opts['output-format']}",
         ]
         for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles',
                     'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait',
                     'time_based', 'runtime', 'verify', 'io_size']:
-            if opt in self.test_opts:
-                option = f"--{opt}={self.test_opts[opt]}"
+            if opt in self.fio_opts:
+                option = f"--{opt}={self.fio_opts[opt]}"
                 fio_args.append(option)
 
-        command = [fio_path] + fio_args
-        with open(self.filenames['command'], "w+",
-                  encoding=locale.getpreferredencoding()) as command_file:
-            command_file.write(" ".join(command))
-
-        passed = True
-
-        try:
-            with open(self.filenames['stdout'], "w+",
-                      encoding=locale.getpreferredencoding()) as stdout_file, \
-                open(self.filenames['stderr'], "w+",
-                     encoding=locale.getpreferredencoding()) as stderr_file, \
-                open(self.filenames['exitcode'], "w+",
-                     encoding=locale.getpreferredencoding()) as exitcode_file:
-                proc = None
-                # Avoid using subprocess.run() here because when a timeout occurs,
-                # fio will be stopped with SIGKILL. This does not give fio a
-                # chance to clean up and means that child processes may continue
-                # running and submitting IO.
-                proc = subprocess.Popen(command,
-                                        stdout=stdout_file,
-                                        stderr=stderr_file,
-                                        cwd=self.test_dir,
-                                        universal_newlines=True)
-                proc.communicate(timeout=300)
-                exitcode_file.write(f'{proc.returncode}\n')
-                passed &= (proc.returncode == 0)
-        except subprocess.TimeoutExpired:
-            proc.terminate()
-            proc.communicate()
-            assert proc.poll()
-            print("Timeout expired")
-            passed = False
-        except Exception:
-            if proc:
-                if not proc.poll():
-                    proc.terminate()
-                    proc.communicate()
-            print(f"Exception: {sys.exc_info()}")
-            passed = False
-
-        if passed:
-            if 'output-format' in self.test_opts and 'json' in \
-                    self.test_opts['output-format']:
-                if not self.get_json():
-                    print('Unable to decode JSON data')
-                    passed = False
-
-        return passed
-
-    def get_json(self):
-        """Convert fio JSON output into a python JSON object"""
-
-        filename = self.filenames['output']
-        with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
-            file_data = file.read()
-
-        #
-        # Sometimes fio informational messages are included at the top of the
-        # JSON output, especially under Windows. Try to decode output as JSON
-        # data, lopping off up to the first four lines
-        #
-        lines = file_data.splitlines()
-        for i in range(5):
-            file_data = '\n'.join(lines[i:])
-            try:
-                self.json_data = json.loads(file_data)
-            except json.JSONDecodeError:
-                continue
-            else:
-                return True
-
-        return False
-
-    @staticmethod
-    def check_empty(job):
-        """
-        Make sure JSON data is empty.
-
-        Some data structures should be empty. This function makes sure that they are.
-
-        job         JSON object that we need to check for emptiness
-        """
-
-        return job['total_ios'] == 0 and \
-                job['slat_ns']['N'] == 0 and \
-                job['clat_ns']['N'] == 0 and \
-                job['lat_ns']['N'] == 0
-
-    def check_all_ddirs(self, ddir_nonzero, job):
-        """
-        Iterate over the data directions and check whether each is
-        appropriately empty or not.
-        """
-
-        retval = True
-        ddirlist = ['read', 'write', 'trim']
-
-        for ddir in ddirlist:
-            if ddir in ddir_nonzero:
-                if self.check_empty(job[ddir]):
-                    print(f"Unexpected zero {ddir} data found in output")
-                    retval = False
-            else:
-                if not self.check_empty(job[ddir]):
-                    print(f"Unexpected {ddir} data found in output")
-                    retval = False
-
-        return retval
-
-    def check(self):
-        """Check test output."""
-
-        raise NotImplementedError()
+        super().setup(fio_args)
 
 
-class PTTest(FioTest):
-    """
-    NVMe pass-through test class. Check to make sure output for selected data
-    direction(s) is non-zero and that zero data appears for other directions.
-    """
+    def check_result(self):
+        if 'rw' not in self.fio_opts:
+            return
 
-    def check(self):
-        if 'rw' not in self.test_opts:
-            return True
+        if not self.passed:
+            return
 
         job = self.json_data['jobs'][0]
-        retval = True
 
-        if self.test_opts['rw'] in ['read', 'randread']:
-            retval = self.check_all_ddirs(['read'], job)
-        elif self.test_opts['rw'] in ['write', 'randwrite']:
-            if 'verify' not in self.test_opts:
-                retval = self.check_all_ddirs(['write'], job)
+        if self.fio_opts['rw'] in ['read', 'randread']:
+            self.passed = self.check_all_ddirs(['read'], job)
+        elif self.fio_opts['rw'] in ['write', 'randwrite']:
+            if 'verify' not in self.fio_opts:
+                self.passed = self.check_all_ddirs(['write'], job)
             else:
-                retval = self.check_all_ddirs(['read', 'write'], job)
-        elif self.test_opts['rw'] in ['trim', 'randtrim']:
-            retval = self.check_all_ddirs(['trim'], job)
-        elif self.test_opts['rw'] in ['readwrite', 'randrw']:
-            retval = self.check_all_ddirs(['read', 'write'], job)
-        elif self.test_opts['rw'] in ['trimwrite', 'randtrimwrite']:
-            retval = self.check_all_ddirs(['trim', 'write'], job)
+                self.passed = self.check_all_ddirs(['read', 'write'], job)
+        elif self.fio_opts['rw'] in ['trim', 'randtrim']:
+            self.passed = self.check_all_ddirs(['trim'], job)
+        elif self.fio_opts['rw'] in ['readwrite', 'randrw']:
+            self.passed = self.check_all_ddirs(['read', 'write'], job)
+        elif self.fio_opts['rw'] in ['trimwrite', 'randtrimwrite']:
+            self.passed = self.check_all_ddirs(['trim', 'write'], job)
         else:
-            print(f"Unhandled rw value {self.test_opts['rw']}")
-            retval = False
-
-        return retval
-
+            print(f"Unhandled rw value {self.fio_opts['rw']}")
+            self.passed = False
 
-def parse_args():
-    """Parse command-line arguments."""
 
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
-    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
-    parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
-    parser.add_argument('-s', '--skip', nargs='+', type=int,
-                        help='list of test(s) to skip')
-    parser.add_argument('-o', '--run-only', nargs='+', type=int,
-                        help='list of test(s) to run, skipping all others')
-    parser.add_argument('--dut', help='target NVMe character device to test '
-                        '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
-    args = parser.parse_args()
-
-    return args
-
-
-def main():
-    """Run tests using fio's io_uring_cmd ioengine to send NVMe pass through commands."""
-
-    args = parse_args()
-
-    artifact_root = args.artifact_root if args.artifact_root else \
-        f"nvmept-test-{time.strftime('%Y%m%d-%H%M%S')}"
-    os.mkdir(artifact_root)
-    print(f"Artifact directory is {artifact_root}")
-
-    if args.fio:
-        fio = str(Path(args.fio).absolute())
-    else:
-        fio = 'fio'
-    print(f"fio path is {fio}")
-
-    test_list = [
-        {
-            "test_id": 1,
+TEST_LIST = [
+    {
+        "test_id": 1,
+        "fio_opts": {
             "rw": 'read',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 2,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 2,
+        "fio_opts": {
             "rw": 'randread',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 3,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 3,
+        "fio_opts": {
             "rw": 'write',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 4,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 4,
+        "fio_opts": {
             "rw": 'randwrite',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 5,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 5,
+        "fio_opts": {
             "rw": 'trim',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 6,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 6,
+        "fio_opts": {
             "rw": 'randtrim',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 7,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 7,
+        "fio_opts": {
             "rw": 'write',
             "io_size": 1024*1024,
             "verify": "crc32c",
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 8,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 8,
+        "fio_opts": {
             "rw": 'randwrite',
             "io_size": 1024*1024,
             "verify": "crc32c",
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 9,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 9,
+        "fio_opts": {
             "rw": 'readwrite',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 10,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 10,
+        "fio_opts": {
             "rw": 'randrw',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 11,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 11,
+        "fio_opts": {
             "rw": 'trimwrite',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 12,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 12,
+        "fio_opts": {
             "rw": 'randtrimwrite',
             "timebased": 1,
             "runtime": 3,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 13,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 13,
+        "fio_opts": {
             "rw": 'randread',
             "timebased": 1,
             "runtime": 3,
@@ -364,10 +214,12 @@ def main():
             "registerfiles": 1,
             "sqthread_poll": 1,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-        {
-            "test_id": 14,
+            },
+        "test_class": PassThruTest,
+    },
+    {
+        "test_id": 14,
+        "fio_opts": {
             "rw": 'randwrite',
             "timebased": 1,
             "runtime": 3,
@@ -377,36 +229,55 @@ def main():
             "registerfiles": 1,
             "sqthread_poll": 1,
             "output-format": "json",
-            "test_obj": PTTest,
-        },
-    ]
+            },
+        "test_class": PassThruTest,
+    },
+]
 
-    passed = 0
-    failed = 0
-    skipped = 0
+def parse_args():
+    """Parse command-line arguments."""
 
-    for test in test_list:
-        if (args.skip and test['test_id'] in args.skip) or \
-           (args.run_only and test['test_id'] not in args.run_only):
-            skipped = skipped + 1
-            outcome = 'SKIPPED (User request)'
-        else:
-            test['filename'] = args.dut
-            test_obj = test['test_obj'](artifact_root, test, args.debug)
-            status = test_obj.run_fio(fio)
-            if status:
-                status = test_obj.check()
-            if status:
-                passed = passed + 1
-                outcome = 'PASSED'
-            else:
-                failed = failed + 1
-                outcome = 'FAILED'
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
+    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
+    parser.add_argument('-s', '--skip', nargs='+', type=int,
+                        help='list of test(s) to skip')
+    parser.add_argument('-o', '--run-only', nargs='+', type=int,
+                        help='list of test(s) to run, skipping all others')
+    parser.add_argument('--dut', help='target NVMe character device to test '
+                        '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
+    args = parser.parse_args()
+
+    return args
+
+
+def main():
+    """Run tests using fio's io_uring_cmd ioengine to send NVMe pass through commands."""
+
+    args = parse_args()
+
+    artifact_root = args.artifact_root if args.artifact_root else \
+        f"nvmept-test-{time.strftime('%Y%m%d-%H%M%S')}"
+    os.mkdir(artifact_root)
+    print(f"Artifact directory is {artifact_root}")
+
+    if args.fio:
+        fio_path = str(Path(args.fio).absolute())
+    else:
+        fio_path = 'fio'
+    print(f"fio path is {fio_path}")
 
-        print(f"**********Test {test['test_id']} {outcome}**********")
+    for test in TEST_LIST:
+        test['fio_opts']['filename'] = args.dut
 
-    print(f"{passed} tests passed, {failed} failed, {skipped} skipped")
+    test_env = {
+              'fio_path': fio_path,
+              'fio_root': str(Path(__file__).absolute().parent.parent),
+              'artifact_root': artifact_root,
+              'basename': 'readonly',
+              }
 
+    _, failed, _ = run_fio_tests(TEST_LIST, test_env, args)
     sys.exit(failed)
 
 
diff --git a/t/random_seed.py b/t/random_seed.py
index 86f2eb21..02187046 100755
--- a/t/random_seed.py
+++ b/t/random_seed.py
@@ -23,38 +23,16 @@ import os
 import sys
 import time
 import locale
+import logging
 import argparse
-import subprocess
 from pathlib import Path
+from fiotestlib import FioJobCmdTest, run_fio_tests
 
-class FioRandTest():
+class FioRandTest(FioJobCmdTest):
     """fio random seed test."""
 
-    def __init__(self, artifact_root, test_options, debug):
-        """
-        artifact_root   root directory for artifacts (subdirectory will be created under here)
-        test            test specification
-        """
-        self.artifact_root = artifact_root
-        self.test_options = test_options
-        self.debug = debug
-        self.filename_stub = None
-        self.filenames = {}
-
-        self.test_dir = os.path.abspath(os.path.join(self.artifact_root,
-                                     f"{self.test_options['test_id']:03d}"))
-        if not os.path.exists(self.test_dir):
-            os.mkdir(self.test_dir)
-
-        self.filename_stub = f"random{self.test_options['test_id']:03d}"
-        self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command")
-        self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout")
-        self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr")
-        self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode")
-        self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output")
-
-    def run_fio(self, fio_path):
-        """Run a test."""
+    def setup(self, parameters):
+        """Setup the test."""
 
         fio_args = [
             "--debug=random",
@@ -65,52 +43,16 @@ class FioRandTest():
             f"--output={self.filenames['output']}",
         ]
         for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
-            if opt in self.test_options:
-                option = f"--{opt}={self.test_options[opt]}"
+            if opt in self.fio_opts:
+                option = f"--{opt}={self.fio_opts[opt]}"
                 fio_args.append(option)
 
-        command = [fio_path] + fio_args
-        with open(self.filenames['command'], "w+", encoding=locale.getpreferredencoding()) as command_file:
-            command_file.write(" ".join(command))
-
-        passed = True
-
-        try:
-            with open(self.filenames['stdout'], "w+", encoding=locale.getpreferredencoding()) as stdout_file, \
-                open(self.filenames['stderr'], "w+", encoding=locale.getpreferredencoding()) as stderr_file, \
-                open(self.filenames['exitcode'], "w+", encoding=locale.getpreferredencoding()) as exitcode_file:
-                proc = None
-                # Avoid using subprocess.run() here because when a timeout occurs,
-                # fio will be stopped with SIGKILL. This does not give fio a
-                # chance to clean up and means that child processes may continue
-                # running and submitting IO.
-                proc = subprocess.Popen(command,
-                                        stdout=stdout_file,
-                                        stderr=stderr_file,
-                                        cwd=self.test_dir,
-                                        universal_newlines=True)
-                proc.communicate(timeout=300)
-                exitcode_file.write(f'{proc.returncode}\n')
-                passed &= (proc.returncode == 0)
-        except subprocess.TimeoutExpired:
-            proc.terminate()
-            proc.communicate()
-            assert proc.poll()
-            print("Timeout expired")
-            passed = False
-        except Exception:
-            if proc:
-                if not proc.poll():
-                    proc.terminate()
-                    proc.communicate()
-            print(f"Exception: {sys.exc_info()}")
-            passed = False
-
-        return passed
+        super().setup(fio_args)
 
     def get_rand_seeds(self):
         """Collect random seeds from --debug=random output."""
-        with open(self.filenames['output'], "r", encoding=locale.getpreferredencoding()) as out_file:
+        with open(self.filenames['output'], "r",
+                  encoding=locale.getpreferredencoding()) as out_file:
             file_data = out_file.read()
 
             offsets = 0
@@ -136,11 +78,6 @@ class FioRandTest():
 
             return seed_list
 
-    def check(self):
-        """Check test output."""
-
-        raise NotImplementedError()
-
 
 class TestRR(FioRandTest):
     """
@@ -151,41 +88,35 @@ class TestRR(FioRandTest):
     # one set of seeds is for randrepeat=0 and the other is for randrepeat=1
     seeds = { 0: None, 1: None }
 
-    def check(self):
+    def check_result(self):
         """Check output for allrandrepeat=1."""
 
-        retval = True
-        opt = 'randrepeat' if 'randrepeat' in self.test_options else 'allrandrepeat'
-        rr = self.test_options[opt]
+        opt = 'randrepeat' if 'randrepeat' in self.fio_opts else 'allrandrepeat'
+        rr = self.fio_opts[opt]
         rand_seeds = self.get_rand_seeds()
 
         if not TestRR.seeds[rr]:
             TestRR.seeds[rr] = rand_seeds
-            if self.debug:
-                print(f"TestRR: saving rand_seeds for [a]rr={rr}")
+            logging.debug("TestRR: saving rand_seeds for [a]rr=%d", rr)
         else:
             if rr:
                 if TestRR.seeds[1] != rand_seeds:
-                    retval = False
+                    self.passed = False
                     print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
                 else:
-                    if self.debug:
-                        print(f"TestRR: seeds correctly match for [a]rr={rr}")
+                    logging.debug("TestRR: seeds correctly match for [a]rr=%d", rr)
                 if TestRR.seeds[0] == rand_seeds:
-                    retval = False
+                    self.passed = False
                     print("TestRR: seeds unexpectedly match those from system RNG")
             else:
                 if TestRR.seeds[0] == rand_seeds:
-                    retval = False
+                    self.passed = False
                     print(f"TestRR: unexpected seed match for [a]rr={rr}")
                 else:
-                    if self.debug:
-                        print(f"TestRR: seeds correctly don't match for [a]rr={rr}")
+                    logging.debug("TestRR: seeds correctly don't match for [a]rr=%d", rr)
                 if TestRR.seeds[1] == rand_seeds:
-                    retval = False
-                    print(f"TestRR: random seeds unexpectedly match those from [a]rr=1")
-
-        return retval
+                    self.passed = False
+                    print("TestRR: random seeds unexpectedly match those from [a]rr=1")
 
 
 class TestRS(FioRandTest):
@@ -197,40 +128,33 @@ class TestRS(FioRandTest):
     """
     seeds = {}
 
-    def check(self):
+    def check_result(self):
         """Check output for randseed=something."""
 
-        retval = True
         rand_seeds = self.get_rand_seeds()
-        randseed = self.test_options['randseed']
+        randseed = self.fio_opts['randseed']
 
-        if self.debug:
-            print("randseed = ", randseed)
+        logging.debug("randseed = %s", randseed)
 
         if randseed not in TestRS.seeds:
             TestRS.seeds[randseed] = rand_seeds
-            if self.debug:
-                print("TestRS: saving rand_seeds")
+            logging.debug("TestRS: saving rand_seeds")
         else:
             if TestRS.seeds[randseed] != rand_seeds:
-                retval = False
+                self.passed = False
                 print("TestRS: seeds don't match when they should")
             else:
-                if self.debug:
-                    print("TestRS: seeds correctly match")
+                logging.debug("TestRS: seeds correctly match")
 
         # Now try to find seeds generated using a different randseed and make
         # sure they *don't* match
-        for key in TestRS.seeds:
+        for key, value in TestRS.seeds.items():
             if key != randseed:
-                if TestRS.seeds[key] == rand_seeds:
-                    retval = False
+                if value == rand_seeds:
+                    self.passed = False
                     print("TestRS: randseeds differ but generated seeds match.")
                 else:
-                    if self.debug:
-                        print("TestRS: randseeds differ and generated seeds also differ.")
-
-        return retval
+                    logging.debug("TestRS: randseeds differ and generated seeds also differ.")
 
 
 def parse_args():
@@ -254,139 +178,161 @@ def main():
 
     args = parse_args()
 
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
     artifact_root = args.artifact_root if args.artifact_root else \
         f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
     os.mkdir(artifact_root)
     print(f"Artifact directory is {artifact_root}")
 
     if args.fio:
-        fio = str(Path(args.fio).absolute())
+        fio_path = str(Path(args.fio).absolute())
     else:
-        fio = 'fio'
-    print(f"fio path is {fio}")
+        fio_path = 'fio'
+    print(f"fio path is {fio_path}")
 
     test_list = [
         {
             "test_id": 1,
-            "randrepeat": 0,
-            "test_obj": TestRR,
+            "fio_opts": {
+                "randrepeat": 0,
+                },
+            "test_class": TestRR,
         },
         {
             "test_id": 2,
-            "randrepeat": 0,
-            "test_obj": TestRR,
+            "fio_opts": {
+                "randrepeat": 0,
+                },
+            "test_class": TestRR,
         },
         {
             "test_id": 3,
-            "randrepeat": 1,
-            "test_obj": TestRR,
+            "fio_opts": {
+                "randrepeat": 1,
+                },
+            "test_class": TestRR,
         },
         {
             "test_id": 4,
-            "randrepeat": 1,
-            "test_obj": TestRR,
+            "fio_opts": {
+                "randrepeat": 1,
+                },
+            "test_class": TestRR,
         },
         {
             "test_id": 5,
-            "allrandrepeat": 0,
-            "test_obj": TestRR,
+            "fio_opts": {
+                "allrandrepeat": 0,
+                },
+            "test_class": TestRR,
         },
         {
             "test_id": 6,
-            "allrandrepeat": 0,
-            "test_obj": TestRR,
+            "fio_opts": {
+                "allrandrepeat": 0,
+                },
+            "test_class": TestRR,
         },
         {
             "test_id": 7,
-            "allrandrepeat": 1,
-            "test_obj": TestRR,
+            "fio_opts": {
+                "allrandrepeat": 1,
+                },
+            "test_class": TestRR,
         },
         {
             "test_id": 8,
-            "allrandrepeat": 1,
-            "test_obj": TestRR,
+            "fio_opts": {
+                "allrandrepeat": 1,
+                },
+            "test_class": TestRR,
         },
         {
             "test_id": 9,
-            "randrepeat": 0,
-            "randseed": "12345",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "randrepeat": 0,
+                "randseed": "12345",
+                },
+            "test_class": TestRS,
         },
         {
             "test_id": 10,
-            "randrepeat": 0,
-            "randseed": "12345",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "randrepeat": 0,
+                "randseed": "12345",
+                },
+            "test_class": TestRS,
         },
         {
             "test_id": 11,
-            "randrepeat": 1,
-            "randseed": "12345",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "randrepeat": 1,
+                "randseed": "12345",
+                },
+            "test_class": TestRS,
         },
         {
             "test_id": 12,
-            "allrandrepeat": 0,
-            "randseed": "12345",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "allrandrepeat": 0,
+                "randseed": "12345",
+                },
+            "test_class": TestRS,
         },
         {
             "test_id": 13,
-            "allrandrepeat": 1,
-            "randseed": "12345",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "allrandrepeat": 1,
+                "randseed": "12345",
+                },
+            "test_class": TestRS,
         },
         {
             "test_id": 14,
-            "randrepeat": 0,
-            "randseed": "67890",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "randrepeat": 0,
+                "randseed": "67890",
+                },
+            "test_class": TestRS,
         },
         {
             "test_id": 15,
-            "randrepeat": 1,
-            "randseed": "67890",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "randrepeat": 1,
+                "randseed": "67890",
+                },
+            "test_class": TestRS,
         },
         {
             "test_id": 16,
-            "allrandrepeat": 0,
-            "randseed": "67890",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "allrandrepeat": 0,
+                "randseed": "67890",
+                },
+            "test_class": TestRS,
         },
         {
             "test_id": 17,
-            "allrandrepeat": 1,
-            "randseed": "67890",
-            "test_obj": TestRS,
+            "fio_opts": {
+                "allrandrepeat": 1,
+                "randseed": "67890",
+                },
+            "test_class": TestRS,
         },
     ]
 
-    passed = 0
-    failed = 0
-    skipped = 0
-
-    for test in test_list:
-        if (args.skip and test['test_id'] in args.skip) or \
-           (args.run_only and test['test_id'] not in args.run_only):
-            skipped = skipped + 1
-            outcome = 'SKIPPED (User request)'
-        else:
-            test_obj = test['test_obj'](artifact_root, test, args.debug)
-            status = test_obj.run_fio(fio)
-            if status:
-                status = test_obj.check()
-            if status:
-                passed = passed + 1
-                outcome = 'PASSED'
-            else:
-                failed = failed + 1
-                outcome = 'FAILED'
-
-        print(f"**********Test {test['test_id']} {outcome}**********")
-
-    print(f"{passed} tests passed, {failed} failed, {skipped} skipped")
+    test_env = {
+              'fio_path': fio_path,
+              'fio_root': str(Path(__file__).absolute().parent.parent),
+              'artifact_root': artifact_root,
+              'basename': 'random',
+              }
 
+    _, failed, _ = run_fio_tests(test_list, test_env, args)
     sys.exit(failed)
 
 
diff --git a/t/readonly.py b/t/readonly.py
index 80fac639..d36faafa 100755
--- a/t/readonly.py
+++ b/t/readonly.py
@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
-#
-#
+
+"""
 # readonly.py
 #
 # Do some basic tests of the --readonly parameter
@@ -18,122 +18,144 @@
 # REQUIREMENTS
 # Python 3.5+
 #
-#
+"""
 
+import os
 import sys
+import time
 import argparse
-import subprocess
+from pathlib import Path
+from fiotestlib import FioJobCmdTest, run_fio_tests
+from fiotestcommon import SUCCESS_DEFAULT, SUCCESS_NONZERO
+
+
+class FioReadOnlyTest(FioJobCmdTest):
+    """fio read only test."""
+
+    def setup(self, parameters):
+        """Setup the test."""
+
+        fio_args = [
+                    "--name=readonly",
+                    "--ioengine=null",
+                    "--time_based",
+                    "--runtime=1s",
+                    "--size=1M",
+                    f"--rw={self.fio_opts['rw']}",
+                   ]
+        if 'readonly-pre' in parameters:
+            fio_args.insert(0, "--readonly")
+        if 'readonly-post' in parameters:
+            fio_args.append("--readonly")
+
+        super().setup(fio_args)
+
+
+TEST_LIST = [
+            {
+                "test_id": 1,
+                "fio_opts": { "rw": "randread", },
+                "readonly-pre": 1,
+                "success": SUCCESS_DEFAULT,
+                "test_class": FioReadOnlyTest,
+            },
+            {
+                "test_id": 2,
+                "fio_opts": { "rw": "randwrite", },
+                "readonly-pre": 1,
+                "success": SUCCESS_NONZERO,
+                "test_class": FioReadOnlyTest,
+            },
+            {
+                "test_id": 3,
+                "fio_opts": { "rw": "randtrim", },
+                "readonly-pre": 1,
+                "success": SUCCESS_NONZERO,
+                "test_class": FioReadOnlyTest,
+            },
+            {
+                "test_id": 4,
+                "fio_opts": { "rw": "randread", },
+                "readonly-post": 1,
+                "success": SUCCESS_DEFAULT,
+                "test_class": FioReadOnlyTest,
+            },
+            {
+                "test_id": 5,
+                "fio_opts": { "rw": "randwrite", },
+                "readonly-post": 1,
+                "success": SUCCESS_NONZERO,
+                "test_class": FioReadOnlyTest,
+            },
+            {
+                "test_id": 6,
+                "fio_opts": { "rw": "randtrim", },
+                "readonly-post": 1,
+                "success": SUCCESS_NONZERO,
+                "test_class": FioReadOnlyTest,
+            },
+            {
+                "test_id": 7,
+                "fio_opts": { "rw": "randread", },
+                "success": SUCCESS_DEFAULT,
+                "test_class": FioReadOnlyTest,
+            },
+            {
+                "test_id": 8,
+                "fio_opts": { "rw": "randwrite", },
+                "success": SUCCESS_DEFAULT,
+                "test_class": FioReadOnlyTest,
+            },
+            {
+                "test_id": 9,
+                "fio_opts": { "rw": "randtrim", },
+                "success": SUCCESS_DEFAULT,
+                "test_class": FioReadOnlyTest,
+            },
+        ]
 
 
 def parse_args():
+    """Parse command-line arguments."""
+
     parser = argparse.ArgumentParser()
-    parser.add_argument('-f', '--fio',
-                        help='path to fio executable (e.g., ./fio)')
+    parser.add_argument('-f', '--fio', help='path to fio executable (e.g., ./fio)')
+    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
+    parser.add_argument('-s', '--skip', nargs='+', type=int,
+                        help='list of test(s) to skip')
+    parser.add_argument('-o', '--run-only', nargs='+', type=int,
+                        help='list of test(s) to run, skipping all others')
     args = parser.parse_args()
 
     return args
 
 
-def run_fio(fio, test, index):
-    fio_args = [
-                "--max-jobs=16",
-                "--name=readonly",
-                "--ioengine=null",
-                "--time_based",
-                "--runtime=1s",
-                "--size=1M",
-                "--rw={rw}".format(**test),
-               ]
-    if 'readonly-pre' in test:
-        fio_args.insert(0, "--readonly")
-    if 'readonly-post' in test:
-        fio_args.append("--readonly")
-
-    output = subprocess.run([fio] + fio_args, stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE)
-
-    return output
-
-
-def check_output(output, test):
-    expect_error = False
-    if 'readonly-pre' in test or 'readonly-post' in test:
-        if 'write' in test['rw'] or 'trim' in test['rw']:
-            expect_error = True
-
-#    print(output.stdout)
-#    print(output.stderr)
-
-    if output.returncode == 0:
-        if expect_error:
-            return False
-        else:
-            return True
-    else:
-        if expect_error:
-            return True
-        else:
-            return False
-
+def main():
+    """Run readonly tests."""
 
-if __name__ == '__main__':
     args = parse_args()
 
-    tests = [
-                {
-                    "rw": "randread",
-                    "readonly-pre": 1,
-                },
-                {
-                    "rw": "randwrite",
-                    "readonly-pre": 1,
-                },
-                {
-                    "rw": "randtrim",
-                    "readonly-pre": 1,
-                },
-                {
-                    "rw": "randread",
-                    "readonly-post": 1,
-                },
-                {
-                    "rw": "randwrite",
-                    "readonly-post": 1,
-                },
-                {
-                    "rw": "randtrim",
-                    "readonly-post": 1,
-                },
-                {
-                    "rw": "randread",
-                },
-                {
-                    "rw": "randwrite",
-                },
-                {
-                    "rw": "randtrim",
-                },
-            ]
-
-    index = 1
-    passed = 0
-    failed = 0
-
     if args.fio:
-        fio_path = args.fio
+        fio_path = str(Path(args.fio).absolute())
     else:
         fio_path = 'fio'
+    print(f"fio path is {fio_path}")
 
-    for test in tests:
-        output = run_fio(fio_path, test, index)
-        status = check_output(output, test)
-        print("Test {0} {1}".format(index, ("PASSED" if status else "FAILED")))
-        if status:
-            passed = passed + 1
-        else:
-            failed = failed + 1
-        index = index + 1
+    artifact_root = args.artifact_root if args.artifact_root else \
+        f"readonly-test-{time.strftime('%Y%m%d-%H%M%S')}"
+    os.mkdir(artifact_root)
+    print(f"Artifact directory is {artifact_root}")
 
-    print("{0} tests passed, {1} failed".format(passed, failed))
+    test_env = {
+              'fio_path': fio_path,
+              'fio_root': str(Path(__file__).absolute().parent.parent),
+              'artifact_root': artifact_root,
+              'basename': 'readonly',
+              }
 
+    _, failed, _ = run_fio_tests(TEST_LIST, test_env, args)
     sys.exit(failed)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index c91deed4..1448f7cb 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -43,298 +43,17 @@
 
 import os
 import sys
-import json
 import time
 import shutil
 import logging
 import argparse
-import platform
-import traceback
-import subprocess
-import multiprocessing
 from pathlib import Path
 from statsmodels.sandbox.stats.runs import runstest_1samp
+from fiotestlib import FioExeTest, FioJobFileTest, run_fio_tests
+from fiotestcommon import *
 
 
-class FioTest():
-    """Base for all fio tests."""
-
-    def __init__(self, exe_path, parameters, success):
-        self.exe_path = exe_path
-        self.parameters = parameters
-        self.success = success
-        self.output = {}
-        self.artifact_root = None
-        self.testnum = None
-        self.test_dir = None
-        self.passed = True
-        self.failure_reason = ''
-        self.command_file = None
-        self.stdout_file = None
-        self.stderr_file = None
-        self.exitcode_file = None
-
-    def setup(self, artifact_root, testnum):
-        """Setup instance variables for test."""
-
-        self.artifact_root = artifact_root
-        self.testnum = testnum
-        self.test_dir = os.path.join(artifact_root, f"{testnum:04d}")
-        if not os.path.exists(self.test_dir):
-            os.mkdir(self.test_dir)
-
-        self.command_file = os.path.join(
-            self.test_dir,
-            f"{os.path.basename(self.exe_path)}.command")
-        self.stdout_file = os.path.join(
-            self.test_dir,
-            f"{os.path.basename(self.exe_path)}.stdout")
-        self.stderr_file = os.path.join(
-            self.test_dir,
-            f"{os.path.basename(self.exe_path)}.stderr")
-        self.exitcode_file = os.path.join(
-            self.test_dir,
-            f"{os.path.basename(self.exe_path)}.exitcode")
-
-    def run(self):
-        """Run the test."""
-
-        raise NotImplementedError()
-
-    def check_result(self):
-        """Check test results."""
-
-        raise NotImplementedError()
-
-
-class FioExeTest(FioTest):
-    """Test consists of an executable binary or script"""
-
-    def __init__(self, exe_path, parameters, success):
-        """Construct a FioExeTest which is a FioTest consisting of an
-        executable binary or script.
-
-        exe_path:       location of executable binary or script
-        parameters:     list of parameters for executable
-        success:        Definition of test success
-        """
-
-        FioTest.__init__(self, exe_path, parameters, success)
-
-    def run(self):
-        """Execute the binary or script described by this instance."""
-
-        command = [self.exe_path] + self.parameters
-        command_file = open(self.command_file, "w+")
-        command_file.write(f"{command}\n")
-        command_file.close()
-
-        stdout_file = open(self.stdout_file, "w+")
-        stderr_file = open(self.stderr_file, "w+")
-        exitcode_file = open(self.exitcode_file, "w+")
-        try:
-            proc = None
-            # Avoid using subprocess.run() here because when a timeout occurs,
-            # fio will be stopped with SIGKILL. This does not give fio a
-            # chance to clean up and means that child processes may continue
-            # running and submitting IO.
-            proc = subprocess.Popen(command,
-                                    stdout=stdout_file,
-                                    stderr=stderr_file,
-                                    cwd=self.test_dir,
-                                    universal_newlines=True)
-            proc.communicate(timeout=self.success['timeout'])
-            exitcode_file.write(f'{proc.returncode}\n')
-            logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
-            self.output['proc'] = proc
-        except subprocess.TimeoutExpired:
-            proc.terminate()
-            proc.communicate()
-            assert proc.poll()
-            self.output['failure'] = 'timeout'
-        except Exception:
-            if proc:
-                if not proc.poll():
-                    proc.terminate()
-                    proc.communicate()
-            self.output['failure'] = 'exception'
-            self.output['exc_info'] = sys.exc_info()
-        finally:
-            stdout_file.close()
-            stderr_file.close()
-            exitcode_file.close()
-
-    def check_result(self):
-        """Check results of test run."""
-
-        if 'proc' not in self.output:
-            if self.output['failure'] == 'timeout':
-                self.failure_reason = f"{self.failure_reason} timeout,"
-            else:
-                assert self.output['failure'] == 'exception'
-                self.failure_reason = '{0} exception: {1}, {2}'.format(
-                    self.failure_reason, self.output['exc_info'][0],
-                    self.output['exc_info'][1])
-
-            self.passed = False
-            return
-
-        if 'zero_return' in self.success:
-            if self.success['zero_return']:
-                if self.output['proc'].returncode != 0:
-                    self.passed = False
-                    self.failure_reason = f"{self.failure_reason} non-zero return code,"
-            else:
-                if self.output['proc'].returncode == 0:
-                    self.failure_reason = f"{self.failure_reason} zero return code,"
-                    self.passed = False
-
-        stderr_size = os.path.getsize(self.stderr_file)
-        if 'stderr_empty' in self.success:
-            if self.success['stderr_empty']:
-                if stderr_size != 0:
-                    self.failure_reason = f"{self.failure_reason} stderr not empty,"
-                    self.passed = False
-            else:
-                if stderr_size == 0:
-                    self.failure_reason = f"{self.failure_reason} stderr empty,"
-                    self.passed = False
-
-
-class FioJobTest(FioExeTest):
-    """Test consists of a fio job"""
-
-    def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
-                 fio_pre_success=None, output_format="normal"):
-        """Construct a FioJobTest which is a FioExeTest consisting of a
-        single fio job file with an optional setup step.
-
-        fio_path:           location of fio executable
-        fio_job:            location of fio job file
-        success:            Definition of test success
-        fio_pre_job:        fio job for preconditioning
-        fio_pre_success:    Definition of test success for fio precon job
-        output_format:      normal (default), json, jsonplus, or terse
-        """
-
-        self.fio_job = fio_job
-        self.fio_pre_job = fio_pre_job
-        self.fio_pre_success = fio_pre_success if fio_pre_success else success
-        self.output_format = output_format
-        self.precon_failed = False
-        self.json_data = None
-        self.fio_output = f"{os.path.basename(self.fio_job)}.output"
-        self.fio_args = [
-            "--max-jobs=16",
-            f"--output-format={self.output_format}",
-            f"--output={self.fio_output}",
-            self.fio_job,
-            ]
-        FioExeTest.__init__(self, fio_path, self.fio_args, success)
-
-    def setup(self, artifact_root, testnum):
-        """Setup instance variables for fio job test."""
-
-        super().setup(artifact_root, testnum)
-
-        self.command_file = os.path.join(
-            self.test_dir,
-            f"{os.path.basename(self.fio_job)}.command")
-        self.stdout_file = os.path.join(
-            self.test_dir,
-            f"{os.path.basename(self.fio_job)}.stdout")
-        self.stderr_file = os.path.join(
-            self.test_dir,
-            f"{os.path.basename(self.fio_job)}.stderr")
-        self.exitcode_file = os.path.join(
-            self.test_dir,
-            f"{os.path.basename(self.fio_job)}.exitcode")
-
-    def run_pre_job(self):
-        """Run fio job precondition step."""
-
-        precon = FioJobTest(self.exe_path, self.fio_pre_job,
-                            self.fio_pre_success,
-                            output_format=self.output_format)
-        precon.setup(self.artifact_root, self.testnum)
-        precon.run()
-        precon.check_result()
-        self.precon_failed = not precon.passed
-        self.failure_reason = precon.failure_reason
-
-    def run(self):
-        """Run fio job test."""
-
-        if self.fio_pre_job:
-            self.run_pre_job()
-
-        if not self.precon_failed:
-            super().run()
-        else:
-            logging.debug("Test %d: precondition step failed", self.testnum)
-
-    @classmethod
-    def get_file(cls, filename):
-        """Safely read a file."""
-        file_data = ''
-        success = True
-
-        try:
-            with open(filename, "r") as output_file:
-                file_data = output_file.read()
-        except OSError:
-            success = False
-
-        return file_data, success
-
-    def get_file_fail(self, filename):
-        """Safely read a file and fail the test upon error."""
-        file_data = None
-
-        try:
-            with open(filename, "r") as output_file:
-                file_data = output_file.read()
-        except OSError:
-            self.failure_reason += f" unable to read file {filename}"
-            self.passed = False
-
-        return file_data
-
-    def check_result(self):
-        """Check fio job results."""
-
-        if self.precon_failed:
-            self.passed = False
-            self.failure_reason = f"{self.failure_reason} precondition step failed,"
-            return
-
-        super().check_result()
-
-        if not self.passed:
-            return
-
-        if 'json' not in self.output_format:
-            return
-
-        file_data = self.get_file_fail(os.path.join(self.test_dir, self.fio_output))
-        if not file_data:
-            return
-
-        #
-        # Sometimes fio informational messages are included at the top of the
-        # JSON output, especially under Windows. Try to decode output as JSON
-        # data, skipping everything until the first {
-        #
-        lines = file_data.splitlines()
-        file_data = '\n'.join(lines[lines.index("{"):])
-        try:
-            self.json_data = json.loads(file_data)
-        except json.JSONDecodeError:
-            self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
-            self.passed = False
-
-
-class FioJobTest_t0005(FioJobTest):
+class FioJobFileTest_t0005(FioJobFileTest):
     """Test consists of fio test job t0005
     Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
 
@@ -352,7 +71,7 @@ class FioJobTest_t0005(FioJobTest):
             self.passed = False
 
 
-class FioJobTest_t0006(FioJobTest):
+class FioJobFileTest_t0006(FioJobFileTest):
     """Test consists of fio test job t0006
     Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
 
@@ -370,7 +89,7 @@ class FioJobTest_t0006(FioJobTest):
             self.passed = False
 
 
-class FioJobTest_t0007(FioJobTest):
+class FioJobFileTest_t0007(FioJobFileTest):
     """Test consists of fio test job t0007
     Confirm that read['io_kbytes'] = 87040"""
 
@@ -385,7 +104,7 @@ class FioJobTest_t0007(FioJobTest):
             self.passed = False
 
 
-class FioJobTest_t0008(FioJobTest):
+class FioJobFileTest_t0008(FioJobFileTest):
     """Test consists of fio test job t0008
     Confirm that read['io_kbytes'] = 32768 and that
                 write['io_kbytes'] ~ 16384
@@ -413,7 +132,7 @@ class FioJobTest_t0008(FioJobTest):
             self.passed = False
 
 
-class FioJobTest_t0009(FioJobTest):
+class FioJobFileTest_t0009(FioJobFileTest):
     """Test consists of fio test job t0009
     Confirm that runtime >= 60s"""
 
@@ -430,7 +149,7 @@ class FioJobTest_t0009(FioJobTest):
             self.passed = False
 
 
-class FioJobTest_t0012(FioJobTest):
+class FioJobFileTest_t0012(FioJobFileTest):
     """Test consists of fio test job t0012
     Confirm ratios of job iops are 1:5:10
     job1,job2,job3 respectively"""
@@ -443,7 +162,7 @@ class FioJobTest_t0012(FioJobTest):
 
         iops_files = []
         for i in range(1, 4):
-            filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(
+            filename = os.path.join(self.paths['test_dir'], "{0}_iops.{1}.log".format(os.path.basename(
                 self.fio_job), i))
             file_data = self.get_file_fail(filename)
             if not file_data:
@@ -475,7 +194,7 @@ class FioJobTest_t0012(FioJobTest):
             return
 
 
-class FioJobTest_t0014(FioJobTest):
+class FioJobFileTest_t0014(FioJobFileTest):
     """Test consists of fio test job t0014
 	Confirm that job1_iops / job2_iops ~ 1:2 for entire duration
 	and that job1_iops / job3_iops ~ 1:3 for first half of duration.
@@ -491,7 +210,7 @@ class FioJobTest_t0014(FioJobTest):
 
         iops_files = []
         for i in range(1, 4):
-            filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(
+            filename = os.path.join(self.paths['test_dir'], "{0}_iops.{1}.log".format(os.path.basename(
                 self.fio_job), i))
             file_data = self.get_file_fail(filename)
             if not file_data:
@@ -534,7 +253,7 @@ class FioJobTest_t0014(FioJobTest):
             return
 
 
-class FioJobTest_t0015(FioJobTest):
+class FioJobFileTest_t0015(FioJobFileTest):
     """Test consists of fio test jobs t0015 and t0016
     Confirm that mean(slat) + mean(clat) = mean(tlat)"""
 
@@ -555,14 +274,14 @@ class FioJobTest_t0015(FioJobTest):
             self.passed = False
 
 
-class FioJobTest_t0019(FioJobTest):
+class FioJobFileTest_t0019(FioJobFileTest):
     """Test consists of fio test job t0019
     Confirm that all offsets were touched sequentially"""
 
     def check_result(self):
         super().check_result()
 
-        bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
+        bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log")
         file_data = self.get_file_fail(bw_log_filename)
         if not file_data:
             return
@@ -585,14 +304,14 @@ class FioJobTest_t0019(FioJobTest):
             self.failure_reason = f"unexpected last offset {cur}"
 
 
-class FioJobTest_t0020(FioJobTest):
+class FioJobFileTest_t0020(FioJobFileTest):
     """Test consists of fio test jobs t0020 and t0021
     Confirm that almost all offsets were touched non-sequentially"""
 
     def check_result(self):
         super().check_result()
 
-        bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
+        bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log")
         file_data = self.get_file_fail(bw_log_filename)
         if not file_data:
             return
@@ -624,13 +343,13 @@ class FioJobTest_t0020(FioJobTest):
             self.failure_reason += f" runs test failed with p = {p}"
 
 
-class FioJobTest_t0022(FioJobTest):
+class FioJobFileTest_t0022(FioJobFileTest):
     """Test consists of fio test job t0022"""
 
     def check_result(self):
         super().check_result()
 
-        bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
+        bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log")
         file_data = self.get_file_fail(bw_log_filename)
         if not file_data:
             return
@@ -662,13 +381,13 @@ class FioJobTest_t0022(FioJobTest):
             self.failure_reason += " no duplicate offsets found with norandommap=1"
 
 
-class FioJobTest_t0023(FioJobTest):
+class FioJobFileTest_t0023(FioJobFileTest):
     """Test consists of fio test job t0023 randtrimwrite test."""
 
     def check_trimwrite(self, filename):
         """Make sure that trims are followed by writes of the same size at the same offset."""
 
-        bw_log_filename = os.path.join(self.test_dir, filename)
+        bw_log_filename = os.path.join(self.paths['test_dir'], filename)
         file_data = self.get_file_fail(bw_log_filename)
         if not file_data:
             return
@@ -716,7 +435,7 @@ class FioJobTest_t0023(FioJobTest):
     def check_all_offsets(self, filename, sectorsize, filesize):
         """Make sure all offsets were touched."""
 
-        file_data = self.get_file_fail(os.path.join(self.test_dir, filename))
+        file_data = self.get_file_fail(os.path.join(self.paths['test_dir'], filename))
         if not file_data:
             return
 
@@ -771,12 +490,12 @@ class FioJobTest_t0023(FioJobTest):
         self.check_all_offsets("bssplit_bw.log", 512, filesize)
 
 
-class FioJobTest_t0024(FioJobTest_t0023):
+class FioJobFileTest_t0024(FioJobFileTest_t0023):
     """Test consists of fio test job t0024 trimwrite test."""
 
     def check_result(self):
-        # call FioJobTest_t0023's parent to skip checks done by t0023
-        super(FioJobTest_t0023, self).check_result()
+        # call FioJobFileTest_t0023's parent to skip checks done by t0023
+        super(FioJobFileTest_t0023, self).check_result()
 
         filesize = 1024*1024
 
@@ -791,7 +510,7 @@ class FioJobTest_t0024(FioJobTest_t0023):
         self.check_all_offsets("bssplit_bw.log", 512, filesize)
 
 
-class FioJobTest_t0025(FioJobTest):
+class FioJobFileTest_t0025(FioJobFileTest):
     """Test experimental verify read backs written data pattern."""
     def check_result(self):
         super().check_result()
@@ -802,11 +521,11 @@ class FioJobTest_t0025(FioJobTest):
         if self.json_data['jobs'][0]['read']['io_kbytes'] != 128:
             self.passed = False
 
-class FioJobTest_t0027(FioJobTest):
+class FioJobFileTest_t0027(FioJobFileTest):
     def setup(self, *args, **kws):
         super().setup(*args, **kws)
-        self.pattern_file = os.path.join(self.test_dir, "t0027.pattern")
-        self.output_file = os.path.join(self.test_dir, "t0027file")
+        self.pattern_file = os.path.join(self.paths['test_dir'], "t0027.pattern")
+        self.output_file = os.path.join(self.paths['test_dir'], "t0027file")
         self.pattern = os.urandom(16 << 10)
         with open(self.pattern_file, "wb") as f:
             f.write(self.pattern)
@@ -823,7 +542,7 @@ class FioJobTest_t0027(FioJobTest):
         if data != self.pattern:
             self.passed = False
 
-class FioJobTest_iops_rate(FioJobTest):
+class FioJobFileTest_iops_rate(FioJobFileTest):
     """Test consists of fio test job t0011
     Confirm that job0 iops == 1000
     and that job1_iops / job0_iops ~ 8
@@ -851,156 +570,10 @@ class FioJobTest_iops_rate(FioJobTest):
             self.passed = False
 
 
-class Requirements():
-    """Requirements consists of multiple run environment characteristics.
-    These are to determine if a particular test can be run"""
-
-    _linux = False
-    _libaio = False
-    _io_uring = False
-    _zbd = False
-    _root = False
-    _zoned_nullb = False
-    _not_macos = False
-    _not_windows = False
-    _unittests = False
-    _cpucount4 = False
-    _nvmecdev = False
-
-    def __init__(self, fio_root, args):
-        Requirements._not_macos = platform.system() != "Darwin"
-        Requirements._not_windows = platform.system() != "Windows"
-        Requirements._linux = platform.system() == "Linux"
-
-        if Requirements._linux:
-            config_file = os.path.join(fio_root, "config-host.h")
-            contents, success = FioJobTest.get_file(config_file)
-            if not success:
-                print(f"Unable to open {config_file} to check requirements")
-                Requirements._zbd = True
-            else:
-                Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
-                Requirements._libaio = "CONFIG_LIBAIO" in contents
-
-            contents, success = FioJobTest.get_file("/proc/kallsyms")
-            if not success:
-                print("Unable to open '/proc/kallsyms' to probe for io_uring support")
-            else:
-                Requirements._io_uring = "io_uring_setup" in contents
-
-            Requirements._root = os.geteuid() == 0
-            if Requirements._zbd and Requirements._root:
-                try:
-                    subprocess.run(["modprobe", "null_blk"],
-                                   stdout=subprocess.PIPE,
-                                   stderr=subprocess.PIPE)
-                    if os.path.exists("/sys/module/null_blk/parameters/zoned"):
-                        Requirements._zoned_nullb = True
-                except Exception:
-                    pass
-
-        if platform.system() == "Windows":
-            utest_exe = "unittest.exe"
-        else:
-            utest_exe = "unittest"
-        unittest_path = os.path.join(fio_root, "unittests", utest_exe)
-        Requirements._unittests = os.path.exists(unittest_path)
-
-        Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
-        Requirements._nvmecdev = args.nvmecdev
-
-        req_list = [
-                Requirements.linux,
-                Requirements.libaio,
-                Requirements.io_uring,
-                Requirements.zbd,
-                Requirements.root,
-                Requirements.zoned_nullb,
-                Requirements.not_macos,
-                Requirements.not_windows,
-                Requirements.unittests,
-                Requirements.cpucount4,
-                Requirements.nvmecdev,
-                    ]
-        for req in req_list:
-            value, desc = req()
-            logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
-
-    @classmethod
-    def linux(cls):
-        """Are we running on Linux?"""
-        return Requirements._linux, "Linux required"
-
-    @classmethod
-    def libaio(cls):
-        """Is libaio available?"""
-        return Requirements._libaio, "libaio required"
-
-    @classmethod
-    def io_uring(cls):
-        """Is io_uring available?"""
-        return Requirements._io_uring, "io_uring required"
-
-    @classmethod
-    def zbd(cls):
-        """Is ZBD support available?"""
-        return Requirements._zbd, "Zoned block device support required"
-
-    @classmethod
-    def root(cls):
-        """Are we running as root?"""
-        return Requirements._root, "root required"
-
-    @classmethod
-    def zoned_nullb(cls):
-        """Are zoned null block devices available?"""
-        return Requirements._zoned_nullb, "Zoned null block device support required"
-
-    @classmethod
-    def not_macos(cls):
-        """Are we running on a platform other than macOS?"""
-        return Requirements._not_macos, "platform other than macOS required"
-
-    @classmethod
-    def not_windows(cls):
-        """Are we running on a platform other than Windws?"""
-        return Requirements._not_windows, "platform other than Windows required"
-
-    @classmethod
-    def unittests(cls):
-        """Were unittests built?"""
-        return Requirements._unittests, "Unittests support required"
-
-    @classmethod
-    def cpucount4(cls):
-        """Do we have at least 4 CPUs?"""
-        return Requirements._cpucount4, "4+ CPUs required"
-
-    @classmethod
-    def nvmecdev(cls):
-        """Do we have an NVMe character device to test?"""
-        return Requirements._nvmecdev, "NVMe character device test target required"
-
-
-SUCCESS_DEFAULT = {
-    'zero_return': True,
-    'stderr_empty': True,
-    'timeout': 600,
-    }
-SUCCESS_NONZERO = {
-    'zero_return': False,
-    'stderr_empty': False,
-    'timeout': 600,
-    }
-SUCCESS_STDERR = {
-    'zero_return': True,
-    'stderr_empty': False,
-    'timeout': 600,
-    }
 TEST_LIST = [
     {
         'test_id':          1,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0001-52c58027.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1009,7 +582,7 @@ TEST_LIST = [
     },
     {
         'test_id':          2,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0002-13af05ae-post.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          't0002-13af05ae-pre.fio',
@@ -1018,7 +591,7 @@ TEST_LIST = [
     },
     {
         'test_id':          3,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0003-0ae2c6e1-post.fio',
         'success':          SUCCESS_NONZERO,
         'pre_job':          't0003-0ae2c6e1-pre.fio',
@@ -1027,7 +600,7 @@ TEST_LIST = [
     },
     {
         'test_id':          4,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0004-8a99fdf6.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1036,7 +609,7 @@ TEST_LIST = [
     },
     {
         'test_id':          5,
-        'test_class':       FioJobTest_t0005,
+        'test_class':       FioJobFileTest_t0005,
         'job':              't0005-f7078f7b.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1046,7 +619,7 @@ TEST_LIST = [
     },
     {
         'test_id':          6,
-        'test_class':       FioJobTest_t0006,
+        'test_class':       FioJobFileTest_t0006,
         'job':              't0006-82af2a7c.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1056,7 +629,7 @@ TEST_LIST = [
     },
     {
         'test_id':          7,
-        'test_class':       FioJobTest_t0007,
+        'test_class':       FioJobFileTest_t0007,
         'job':              't0007-37cf9e3c.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1066,7 +639,7 @@ TEST_LIST = [
     },
     {
         'test_id':          8,
-        'test_class':       FioJobTest_t0008,
+        'test_class':       FioJobFileTest_t0008,
         'job':              't0008-ae2fafc8.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1076,7 +649,7 @@ TEST_LIST = [
     },
     {
         'test_id':          9,
-        'test_class':       FioJobTest_t0009,
+        'test_class':       FioJobFileTest_t0009,
         'job':              't0009-f8b0bd10.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1088,7 +661,7 @@ TEST_LIST = [
     },
     {
         'test_id':          10,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0010-b7aae4ba.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1097,7 +670,7 @@ TEST_LIST = [
     },
     {
         'test_id':          11,
-        'test_class':       FioJobTest_iops_rate,
+        'test_class':       FioJobFileTest_iops_rate,
         'job':              't0011-5d2788d5.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1107,7 +680,7 @@ TEST_LIST = [
     },
     {
         'test_id':          12,
-        'test_class':       FioJobTest_t0012,
+        'test_class':       FioJobFileTest_t0012,
         'job':              't0012.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1117,7 +690,7 @@ TEST_LIST = [
     },
     {
         'test_id':          13,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0013.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1127,7 +700,7 @@ TEST_LIST = [
     },
     {
         'test_id':          14,
-        'test_class':       FioJobTest_t0014,
+        'test_class':       FioJobFileTest_t0014,
         'job':              't0014.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1137,7 +710,7 @@ TEST_LIST = [
     },
     {
         'test_id':          15,
-        'test_class':       FioJobTest_t0015,
+        'test_class':       FioJobFileTest_t0015,
         'job':              't0015-e78980ff.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1147,7 +720,7 @@ TEST_LIST = [
     },
     {
         'test_id':          16,
-        'test_class':       FioJobTest_t0015,
+        'test_class':       FioJobFileTest_t0015,
         'job':              't0016-d54ae22.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1157,7 +730,7 @@ TEST_LIST = [
     },
     {
         'test_id':          17,
-        'test_class':       FioJobTest_t0015,
+        'test_class':       FioJobFileTest_t0015,
         'job':              't0017.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1167,7 +740,7 @@ TEST_LIST = [
     },
     {
         'test_id':          18,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0018.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1176,7 +749,7 @@ TEST_LIST = [
     },
     {
         'test_id':          19,
-        'test_class':       FioJobTest_t0019,
+        'test_class':       FioJobFileTest_t0019,
         'job':              't0019.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1185,7 +758,7 @@ TEST_LIST = [
     },
     {
         'test_id':          20,
-        'test_class':       FioJobTest_t0020,
+        'test_class':       FioJobFileTest_t0020,
         'job':              't0020.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1194,7 +767,7 @@ TEST_LIST = [
     },
     {
         'test_id':          21,
-        'test_class':       FioJobTest_t0020,
+        'test_class':       FioJobFileTest_t0020,
         'job':              't0021.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1203,7 +776,7 @@ TEST_LIST = [
     },
     {
         'test_id':          22,
-        'test_class':       FioJobTest_t0022,
+        'test_class':       FioJobFileTest_t0022,
         'job':              't0022.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1212,7 +785,7 @@ TEST_LIST = [
     },
     {
         'test_id':          23,
-        'test_class':       FioJobTest_t0023,
+        'test_class':       FioJobFileTest_t0023,
         'job':              't0023.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1221,7 +794,7 @@ TEST_LIST = [
     },
     {
         'test_id':          24,
-        'test_class':       FioJobTest_t0024,
+        'test_class':       FioJobFileTest_t0024,
         'job':              't0024.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1230,7 +803,7 @@ TEST_LIST = [
     },
     {
         'test_id':          25,
-        'test_class':       FioJobTest_t0025,
+        'test_class':       FioJobFileTest_t0025,
         'job':              't0025.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1240,7 +813,7 @@ TEST_LIST = [
     },
     {
         'test_id':          26,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0026.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1249,7 +822,7 @@ TEST_LIST = [
     },
     {
         'test_id':          27,
-        'test_class':       FioJobTest_t0027,
+        'test_class':       FioJobFileTest_t0027,
         'job':              't0027.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1258,7 +831,7 @@ TEST_LIST = [
     },
     {
         'test_id':          28,
-        'test_class':       FioJobTest,
+        'test_class':       FioJobFileTest,
         'job':              't0028-c6cade16.fio',
         'success':          SUCCESS_DEFAULT,
         'pre_job':          None,
@@ -1317,7 +890,7 @@ TEST_LIST = [
         'test_id':          1006,
         'test_class':       FioExeTest,
         'exe':              't/strided.py',
-        'parameters':       ['{fio_path}'],
+        'parameters':       ['--fio', '{fio_path}'],
         'success':          SUCCESS_DEFAULT,
         'requirements':     [],
     },
@@ -1461,98 +1034,15 @@ def main():
     print(f"Artifact directory is {artifact_root}")
 
     if not args.skip_req:
-        req = Requirements(fio_root, args)
-
-    passed = 0
-    failed = 0
-    skipped = 0
-
-    for config in TEST_LIST:
-        if (args.skip and config['test_id'] in args.skip) or \
-           (args.run_only and config['test_id'] not in args.run_only):
-            skipped = skipped + 1
-            print(f"Test {config['test_id']} SKIPPED (User request)")
-            continue
-
-        if issubclass(config['test_class'], FioJobTest):
-            if config['pre_job']:
-                fio_pre_job = os.path.join(fio_root, 't', 'jobs',
-                                           config['pre_job'])
-            else:
-                fio_pre_job = None
-            if config['pre_success']:
-                fio_pre_success = config['pre_success']
-            else:
-                fio_pre_success = None
-            if 'output_format' in config:
-                output_format = config['output_format']
-            else:
-                output_format = 'normal'
-            test = config['test_class'](
-                fio_path,
-                os.path.join(fio_root, 't', 'jobs', config['job']),
-                config['success'],
-                fio_pre_job=fio_pre_job,
-                fio_pre_success=fio_pre_success,
-                output_format=output_format)
-            desc = config['job']
-        elif issubclass(config['test_class'], FioExeTest):
-            exe_path = os.path.join(fio_root, config['exe'])
-            if config['parameters']:
-                parameters = [p.format(fio_path=fio_path, nvmecdev=args.nvmecdev)
-                              for p in config['parameters']]
-            else:
-                parameters = []
-            if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
-                parameters.insert(0, exe_path)
-                exe_path = "python.exe"
-            if config['test_id'] in pass_through:
-                parameters += pass_through[config['test_id']].split()
-            test = config['test_class'](exe_path, parameters,
-                                        config['success'])
-            desc = config['exe']
-        else:
-            print(f"Test {config['test_id']} FAILED: unable to process test config")
-            failed = failed + 1
-            continue
-
-        if not args.skip_req:
-            reqs_met = True
-            for req in config['requirements']:
-                reqs_met, reason = req()
-                logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
-                              reqs_met)
-                if not reqs_met:
-                    break
-            if not reqs_met:
-                print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
-                skipped = skipped + 1
-                continue
-
-        try:
-            test.setup(artifact_root, config['test_id'])
-            test.run()
-            test.check_result()
-        except KeyboardInterrupt:
-            break
-        except Exception as e:
-            test.passed = False
-            test.failure_reason += str(e)
-            logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
-        if test.passed:
-            result = "PASSED"
-            passed = passed + 1
-        else:
-            result = f"FAILED: {test.failure_reason}"
-            failed = failed + 1
-            contents, _ = FioJobTest.get_file(test.stderr_file)
-            logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
-            contents, _ = FioJobTest.get_file(test.stdout_file)
-            logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
-        print(f"Test {config['test_id']} {result} {desc}")
-
-    print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
-
+        Requirements(fio_root, args)
+
+    test_env = {
+              'fio_path': fio_path,
+              'fio_root': fio_root,
+              'artifact_root': artifact_root,
+              'pass_through': pass_through,
+              }
+    _, failed, _ = run_fio_tests(TEST_LIST, test_env, args)
     sys.exit(failed)
 
 
diff --git a/t/strided.py b/t/strided.py
index 45e6f148..b7655e1e 100755
--- a/t/strided.py
+++ b/t/strided.py
@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
-#
+
+"""
 # strided.py
 #
 # Test zonemode=strided. This uses the null ioengine when no file is
 # specified. If a file is specified, use it for randdom read testing.
 # Some of the zoneranges in the tests are 16MiB. So when using a file
-# a minimum size of 32MiB is recommended.
+# a minimum size of 64MiB is recommended.
 #
 # USAGE
 # python strided.py fio-executable [-f file/device]
@@ -13,12 +14,9 @@
 # EXAMPLES
 # python t/strided.py ./fio
 # python t/strided.py ./fio -f /dev/sda
-# dd if=/dev/zero of=temp bs=1M count=32
+# dd if=/dev/zero of=temp bs=1M count=64
 # python t/strided.py ./fio -f temp
 #
-# REQUIREMENTS
-# Python 2.6+
-#
 # ===TEST MATRIX===
 #
 # --zonemode=strided, zoneskip unset
@@ -28,322 +26,417 @@
 #       zonesize<zonerange  all blocks inside zone
 #
 #   w/o randommap       all blocks inside zone
-#
+"""
 
-from __future__ import absolute_import
-from __future__ import print_function
 import os
 import sys
+import time
 import argparse
-import subprocess
+from pathlib import Path
+from fiotestlib import FioJobCmdTest, run_fio_tests
 
 
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('fio',
-                        help='path to fio executable (e.g., ./fio)')
-    parser.add_argument('-f', '--filename', help="file/device to test")
-    args = parser.parse_args()
+class StridedTest(FioJobCmdTest):
+    """Test zonemode=strided."""
 
-    return args
+    def setup(self, parameters):
+        fio_args = [
+                    "--name=strided",
+                    "--zonemode=strided",
+                    "--log_offset=1",
+                    "--randrepeat=0",
+                    "--rw=randread",
+                    f"--write_iops_log={self.filenames['iopslog']}",
+                    f"--output={self.filenames['output']}",
+                    f"--zonerange={self.fio_opts['zonerange']}",
+                    f"--zonesize={self.fio_opts['zonesize']}",
+                    f"--bs={self.fio_opts['bs']}",
+                   ]
 
+        for opt in ['norandommap', 'random_generator', 'offset']:
+            if opt in self.fio_opts:
+                option = f"--{opt}={self.fio_opts[opt]}"
+                fio_args.append(option)
 
-def run_fio(fio, test, index):
-    filename = "strided"
-    fio_args = [
-                "--max-jobs=16",
-                "--name=strided",
-                "--zonemode=strided",
-                "--log_offset=1",
-                "--randrepeat=0",
-                "--rw=randread",
-                "--write_iops_log={0}{1:03d}".format(filename, index),
-                "--output={0}{1:03d}.out".format(filename, index),
-                "--zonerange={zonerange}".format(**test),
-                "--zonesize={zonesize}".format(**test),
-                "--bs={bs}".format(**test),
-               ]
-    if 'norandommap' in test:
-        fio_args.append('--norandommap')
-    if 'random_generator' in test:
-        fio_args.append('--random_generator={random_generator}'.format(**test))
-    if 'offset' in test:
-        fio_args.append('--offset={offset}'.format(**test))
-    if 'filename' in test:
-        fio_args.append('--filename={filename}'.format(**test))
-        fio_args.append('--filesize={filesize})'.format(**test))
-    else:
-        fio_args.append('--ioengine=null')
-        fio_args.append('--size={size}'.format(**test))
-        fio_args.append('--io_size={io_size}'.format(**test))
-        fio_args.append('--filesize={size})'.format(**test))
-
-    output = subprocess.check_output([fio] + fio_args, universal_newlines=True)
-
-    f = open("{0}{1:03d}_iops.1.log".format(filename, index), "r")
-    log = f.read()
-    f.close()
-
-    return log
-
-
-def check_output(iops_log, test):
-    zonestart = 0 if 'offset' not in test else test['offset']
-    iospersize = test['zonesize'] / test['bs']
-    iosperrange = test['zonerange'] / test['bs']
-    iosperzone = 0
-    lines = iops_log.split('\n')
-    zoneset = set()
-
-    for line in lines:
-        if len(line) == 0:
-            continue
-
-        if iosperzone == iospersize:
-            # time to move to a new zone
-            iosperzone = 0
-            zoneset = set()
-            zonestart += test['zonerange']
-            if zonestart >= test['filesize']:
-                zonestart = 0 if 'offset' not in test else test['offset']
-
-        iosperzone = iosperzone + 1
-        tokens = line.split(',')
-        offset = int(tokens[4])
-        if offset < zonestart or offset >= zonestart + test['zonerange']:
-            print("Offset {0} outside of zone starting at {1}".format(
-                    offset, zonestart))
-            return False
-
-        # skip next section if norandommap is enabled with no
-        # random_generator or with a random_generator != lfsr
-        if 'norandommap' in test:
-            if 'random_generator' in test:
-                if test['random_generator'] != 'lfsr':
-                    continue
-            else:
+        if 'filename' in self.fio_opts:
+            for opt in ['filename', 'filesize']:
+                option = f"--{opt}={self.fio_opts[opt]}"
+                fio_args.append(option)
+        else:
+            fio_args.append('--ioengine=null')
+            for opt in ['size', 'io_size', 'filesize']:
+                option = f"--{opt}={self.fio_opts[opt]}"
+                fio_args.append(option)
+
+        super().setup(fio_args)
+
+    def check_result(self):
+        zonestart = 0 if 'offset' not in self.fio_opts else self.fio_opts['offset']
+        iospersize = self.fio_opts['zonesize'] / self.fio_opts['bs']
+        iosperrange = self.fio_opts['zonerange'] / self.fio_opts['bs']
+        iosperzone = 0
+        lines = self.iops_log_lines.split('\n')
+        zoneset = set()
+
+        for line in lines:
+            if len(line) == 0:
                 continue
 
-        # we either have a random map enabled or we
-        # are using an LFSR
-        # so all blocks should be unique and we should have
-        # covered the entire zone when iosperzone % iosperrange == 0
-        block = (offset - zonestart) / test['bs']
-        if block in zoneset:
-            print("Offset {0} in zone already touched".format(offset))
-            return False
-
-        zoneset.add(block)
-        if iosperzone % iosperrange == 0:
-            if len(zoneset) != iosperrange:
-                print("Expected {0} blocks in zone but only saw {1}".format(
-                        iosperrange, len(zoneset)))
+            if iosperzone == iospersize:
+                # time to move to a new zone
+                iosperzone = 0
+                zoneset = set()
+                zonestart += self.fio_opts['zonerange']
+                if zonestart >= self.fio_opts['filesize']:
+                    zonestart = 0 if 'offset' not in self.fio_opts else self.fio_opts['offset']
+
+            iosperzone = iosperzone + 1
+            tokens = line.split(',')
+            offset = int(tokens[4])
+            if offset < zonestart or offset >= zonestart + self.fio_opts['zonerange']:
+                print(f"Offset {offset} outside of zone starting at {zonestart}")
                 return False
-            zoneset = set()
 
-    return True
+            # skip next section if norandommap is enabled with no
+            # random_generator or with a random_generator != lfsr
+            if 'norandommap' in self.fio_opts:
+                if 'random_generator' in self.fio_opts:
+                    if self.fio_opts['random_generator'] != 'lfsr':
+                        continue
+                else:
+                    continue
 
+            # we either have a random map enabled or we
+            # are using an LFSR
+            # so all blocks should be unique and we should have
+            # covered the entire zone when iosperzone % iosperrange == 0
+            block = (offset - zonestart) / self.fio_opts['bs']
+            if block in zoneset:
+                print(f"Offset {offset} in zone already touched")
+                return False
+
+            zoneset.add(block)
+            if iosperzone % iosperrange == 0:
+                if len(zoneset) != iosperrange:
+                    print(f"Expected {iosperrange} blocks in zone but only saw {len(zoneset)}")
+                    return False
+                zoneset = set()
+
+        return True
+
+
+TEST_LIST = [   # randommap enabled
+    {
+        "test_id": 1,
+        "fio_opts": {
+            "zonerange": 4096,
+            "zonesize": 4096,
+            "bs": 4096,
+            "offset": 8*4096,
+            "size": 16*4096,
+            "io_size": 16*4096,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 2,
+        "fio_opts": {
+            "zonerange": 4096,
+            "zonesize": 4096,
+            "bs": 4096,
+            "size": 16*4096,
+            "io_size": 16*4096,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 3,
+        "fio_opts": {
+            "zonerange": 16*1024*1024,
+            "zonesize": 16*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*204,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 4,
+        "fio_opts": {
+            "zonerange": 4096,
+            "zonesize": 4*4096,
+            "bs": 4096,
+            "size": 16*4096,
+            "io_size": 16*4096,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 5,
+        "fio_opts": {
+            "zonerange": 16*1024*1024,
+            "zonesize": 32*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*204,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 6,
+        "fio_opts": {
+            "zonerange": 8192,
+            "zonesize": 4096,
+            "bs": 4096,
+            "size": 16*4096,
+            "io_size": 16*4096,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 7,
+        "fio_opts": {
+            "zonerange": 16*1024*1024,
+            "zonesize": 8*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*204,
+            },
+        "test_class": StridedTest,
+    },
+            # lfsr
+    {
+        "test_id": 8,
+        "fio_opts": {
+            "random_generator": "lfsr",
+            "zonerange": 4096*1024,
+            "zonesize": 4096*1024,
+            "bs": 4096,
+            "offset": 8*4096*1024,
+            "size": 16*4096*1024,
+            "io_size": 16*4096*1024,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 9,
+        "fio_opts": {
+            "random_generator": "lfsr",
+            "zonerange": 4096*1024,
+            "zonesize": 4096*1024,
+            "bs": 4096,
+            "size": 16*4096*1024,
+            "io_size": 16*4096*1024,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 10,
+        "fio_opts": {
+            "random_generator": "lfsr",
+            "zonerange": 16*1024*1024,
+            "zonesize": 16*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*204,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 11,
+        "fio_opts": {
+            "random_generator": "lfsr",
+            "zonerange": 4096*1024,
+            "zonesize": 4*4096*1024,
+            "bs": 4096,
+            "size": 16*4096*1024,
+            "io_size": 16*4096*1024,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 12,
+        "fio_opts": {
+            "random_generator": "lfsr",
+            "zonerange": 16*1024*1024,
+            "zonesize": 32*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*204,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 13,
+        "fio_opts": {
+            "random_generator": "lfsr",
+            "zonerange": 8192*1024,
+            "zonesize": 4096*1024,
+            "bs": 4096,
+            "size": 16*4096*1024,
+            "io_size": 16*4096*1024,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 14,
+        "fio_opts": {
+            "random_generator": "lfsr",
+            "zonerange": 16*1024*1024,
+            "zonesize": 8*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*204,
+            },
+        "test_class": StridedTest,
+    },
+    # norandommap
+    {
+        "test_id": 15,
+        "fio_opts": {
+            "norandommap": 1,
+            "zonerange": 4096,
+            "zonesize": 4096,
+            "bs": 4096,
+            "offset": 8*4096,
+            "size": 16*4096,
+            "io_size": 16*4096,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 16,
+        "fio_opts": {
+            "norandommap": 1,
+            "zonerange": 4096,
+            "zonesize": 4096,
+            "bs": 4096,
+            "size": 16*4096,
+            "io_size": 16*4096,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 17,
+        "fio_opts": {
+            "norandommap": 1,
+            "zonerange": 16*1024*1024,
+            "zonesize": 16*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*204,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 18,
+        "fio_opts": {
+            "norandommap": 1,
+            "zonerange": 4096,
+            "zonesize": 8192,
+            "bs": 4096,
+            "size": 16*4096,
+            "io_size": 16*4096,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 19,
+        "fio_opts": {
+            "norandommap": 1,
+            "zonerange": 16*1024*1024,
+            "zonesize": 32*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*204,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 20,
+        "fio_opts": {
+            "norandommap": 1,
+            "zonerange": 8192,
+            "zonesize": 4096,
+            "bs": 4096,
+            "size": 16*4096,
+            "io_size": 16*4096,
+            },
+        "test_class": StridedTest,
+    },
+    {
+        "test_id": 21,
+        "fio_opts": {
+            "norandommap": 1,
+            "zonerange": 16*1024*1024,
+            "zonesize": 8*1024*1024,
+            "bs": 4096,
+            "size": 256*1024*1024,
+            "io_size": 256*1024*1024,
+            },
+        "test_class": StridedTest,
+    },
+]
+
+
+def parse_args():
+    """Parse command-line arguments."""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
+    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
+    parser.add_argument('-s', '--skip', nargs='+', type=int,
+                        help='list of test(s) to skip')
+    parser.add_argument('-o', '--run-only', nargs='+', type=int,
+                        help='list of test(s) to run, skipping all others')
+    parser.add_argument('--dut',
+                        help='target file/device to test.')
+    args = parser.parse_args()
+
+    return args
+
+
+def main():
+    """Run zonemode=strided tests."""
 
-if __name__ == '__main__':
     args = parse_args()
 
-    tests = [   # randommap enabled
-                {
-                    "zonerange": 4096,
-                    "zonesize": 4096,
-                    "bs": 4096,
-                    "offset": 8*4096,
-                    "size": 16*4096,
-                    "io_size": 16*4096,
-                },
-                {
-                    "zonerange": 4096,
-                    "zonesize": 4096,
-                    "bs": 4096,
-                    "size": 16*4096,
-                    "io_size": 16*4096,
-                },
-                {
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 16*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*204,
-                },
-                {
-                    "zonerange": 4096,
-                    "zonesize": 4*4096,
-                    "bs": 4096,
-                    "size": 16*4096,
-                    "io_size": 16*4096,
-                },
-                {
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 32*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*204,
-                },
-                {
-                    "zonerange": 8192,
-                    "zonesize": 4096,
-                    "bs": 4096,
-                    "size": 16*4096,
-                    "io_size": 16*4096,
-                },
-                {
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 8*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*204,
-                },
-                # lfsr
-                {
-                    "random_generator": "lfsr",
-                    "zonerange": 4096*1024,
-                    "zonesize": 4096*1024,
-                    "bs": 4096,
-                    "offset": 8*4096*1024,
-                    "size": 16*4096*1024,
-                    "io_size": 16*4096*1024,
-                },
-                {
-                    "random_generator": "lfsr",
-                    "zonerange": 4096*1024,
-                    "zonesize": 4096*1024,
-                    "bs": 4096,
-                    "size": 16*4096*1024,
-                    "io_size": 16*4096*1024,
-                },
-                {
-                    "random_generator": "lfsr",
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 16*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*204,
-                },
-                {
-                    "random_generator": "lfsr",
-                    "zonerange": 4096*1024,
-                    "zonesize": 4*4096*1024,
-                    "bs": 4096,
-                    "size": 16*4096*1024,
-                    "io_size": 16*4096*1024,
-                },
-                {
-                    "random_generator": "lfsr",
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 32*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*204,
-                },
-                {
-                    "random_generator": "lfsr",
-                    "zonerange": 8192*1024,
-                    "zonesize": 4096*1024,
-                    "bs": 4096,
-                    "size": 16*4096*1024,
-                    "io_size": 16*4096*1024,
-                },
-                {
-                    "random_generator": "lfsr",
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 8*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*204,
-                },
-                # norandommap
-                {
-                    "norandommap": 1,
-                    "zonerange": 4096,
-                    "zonesize": 4096,
-                    "bs": 4096,
-                    "offset": 8*4096,
-                    "size": 16*4096,
-                    "io_size": 16*4096,
-                },
-                {
-                    "norandommap": 1,
-                    "zonerange": 4096,
-                    "zonesize": 4096,
-                    "bs": 4096,
-                    "size": 16*4096,
-                    "io_size": 16*4096,
-                },
-                {
-                    "norandommap": 1,
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 16*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*204,
-                },
-                {
-                    "norandommap": 1,
-                    "zonerange": 4096,
-                    "zonesize": 8192,
-                    "bs": 4096,
-                    "size": 16*4096,
-                    "io_size": 16*4096,
-                },
-                {
-                    "norandommap": 1,
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 32*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*204,
-                },
-                {
-                    "norandommap": 1,
-                    "zonerange": 8192,
-                    "zonesize": 4096,
-                    "bs": 4096,
-                    "size": 16*4096,
-                    "io_size": 16*4096,
-                },
-                {
-                    "norandommap": 1,
-                    "zonerange": 16*1024*1024,
-                    "zonesize": 8*1024*1024,
-                    "bs": 4096,
-                    "size": 256*1024*1024,
-                    "io_size": 256*1024*1024,
-                },
-
-            ]
-
-    index = 1
-    passed = 0
-    failed = 0
-
-    if args.filename:
-        statinfo = os.stat(args.filename)
+    artifact_root = args.artifact_root if args.artifact_root else \
+        f"strided-test-{time.strftime('%Y%m%d-%H%M%S')}"
+    os.mkdir(artifact_root)
+    print(f"Artifact directory is {artifact_root}")
+
+    if args.fio:
+        fio_path = str(Path(args.fio).absolute())
+    else:
+        fio_path = 'fio'
+    print(f"fio path is {fio_path}")
+
+    if args.dut:
+        statinfo = os.stat(args.dut)
         filesize = statinfo.st_size
         if filesize == 0:
-            f = os.open(args.filename, os.O_RDONLY)
+            f = os.open(args.dut, os.O_RDONLY)
             filesize = os.lseek(f, 0, os.SEEK_END)
             os.close(f)
 
-    for test in tests:
-        if args.filename:
-            test['filename'] = args.filename
-            test['filesize'] = filesize
+    for test in TEST_LIST:
+        if args.dut:
+            test['fio_opts']['filename'] = os.path.abspath(args.dut)
+            test['fio_opts']['filesize'] = filesize
         else:
-            test['filesize'] = test['size']
-        iops_log = run_fio(args.fio, test, index)
-        status = check_output(iops_log, test)
-        print("Test {0} {1}".format(index, ("PASSED" if status else "FAILED")))
-        if status:
-            passed = passed + 1
-        else:
-            failed = failed + 1
-        index = index + 1
+            test['fio_opts']['filesize'] = test['fio_opts']['size']
 
-    print("{0} tests passed, {1} failed".format(passed, failed))
+    test_env = {
+              'fio_path': fio_path,
+              'fio_root': str(Path(__file__).absolute().parent.parent),
+              'artifact_root': artifact_root,
+              'basename': 'strided',
+              }
 
+    _, failed, _ = run_fio_tests(TEST_LIST, test_env, args)
     sys.exit(failed)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/t/zbd/test-zbd-support b/t/zbd/test-zbd-support
index 996160e7..a3d37a7d 100755
--- a/t/zbd/test-zbd-support
+++ b/t/zbd/test-zbd-support
@@ -460,7 +460,8 @@ test11() {
 test12() {
     local size off capacity
 
-    prep_write
+    [ -n "$is_zbd" ] && reset_zone "$dev" -1
+
     size=$((8 * zone_size))
     off=$((first_sequential_zone_sector * 512))
     capacity=$(total_zone_capacity 8 $off $dev)
@@ -477,7 +478,8 @@ test13() {
 
     require_max_open_zones 4 || return $SKIP_TESTCASE
 
-    prep_write
+    [ -n "$is_zbd" ] && reset_zone "$dev" -1
+
     size=$((8 * zone_size))
     off=$((first_sequential_zone_sector * 512))
     capacity=$(total_zone_capacity 8 $off $dev)
@@ -726,7 +728,9 @@ test29() {
     require_seq_zones 80 || return $SKIP_TESTCASE
     off=$((first_sequential_zone_sector * 512 + 64 * zone_size))
     size=$((16*zone_size))
-    prep_write
+
+    [ -n "$is_zbd" ] && reset_zone "$dev" -1
+
     opts=("--debug=zbd")
     for ((i=0;i<jobs;i++)); do
 	opts+=("--name=job$i" "--filename=$dev" "--offset=$off" "--bs=16K")
@@ -796,7 +800,8 @@ test32() {
 
     require_zbd || return $SKIP_TESTCASE
 
-    prep_write
+    [ -n "$is_zbd" ] && reset_zone "$dev" -1
+
     off=$((first_sequential_zone_sector * 512))
     size=$((disk_size - off))
     opts+=("--name=$dev" "--filename=$dev" "--offset=$off" "--size=$size")
@@ -1024,7 +1029,9 @@ test48() {
 
     off=$((first_sequential_zone_sector * 512 + 64 * zone_size))
     size=$((16*zone_size))
-    prep_write
+
+    [ -n "$is_zbd" ] && reset_zone "$dev" -1
+
     opts=("--aux-path=/tmp" "--allow_file_create=0" "--significant_figures=10")
     opts+=("--debug=zbd")
     opts+=("$(ioengine "libaio")" "--rw=randwrite" "--direct=1")
@@ -1094,7 +1101,7 @@ test51() {
 	require_conv_zones 8 || return $SKIP_TESTCASE
 	require_seq_zones 8 || return $SKIP_TESTCASE
 
-	prep_write
+	reset_zone "$dev" -1
 
 	off=$((first_sequential_zone_sector * 512 - 8 * zone_size))
 	opts+=("--size=$((16 * zone_size))" "$(ioengine "libaio")")
@@ -1361,6 +1368,51 @@ test63() {
 	check_reset_count -eq 3 || return $?
 }
 
+# Test write zone accounting handles almost full zones correctly. Prepare an
+# almost full, but not full zone. Write to the zone with verify using larger
+# block size. Then confirm fio does not report write zone accounting failure.
+test64() {
+	local bs cap
+
+	[ -n "$is_zbd" ] && reset_zone "$dev" -1
+
+	bs=$((zone_size / 8))
+	cap=$(total_zone_capacity 1 $((first_sequential_zone_sector*512)) $dev)
+	run_fio_on_seq "$(ioengine "psync")" --rw=write --bs="$bs" \
+		       --size=$((zone_size)) \
+		       --io_size=$((cap - bs)) \
+		       >> "${logfile}.${test_number}" 2>&1 || return $?
+
+	bs=$((zone_size / 2))
+	run_fio_on_seq "$(ioengine "psync")" --rw=write --bs="$bs" \
+		       --size=$((zone_size)) --do_verify=1 --verify=md5 \
+		       >> "${logfile}.${test_number}" 2>&1 || return $?
+}
+
+# Test open zone accounting handles trim workload correctly. Prepare open zones
+# as many as max_open_zones=4. Trim one of the 4 zones. Then write to another
+# zone and check the write amount is expected size.
+test65() {
+	local off capacity
+
+	[ -n "$is_zbd" ] && reset_zone "$dev" -1
+
+	off=$((first_sequential_zone_sector * 512))
+	capacity=$(total_zone_capacity 1 $off "$dev")
+	run_fio --zonemode=zbd --direct=1 --zonesize="$zone_size" --thread=1 \
+		--filename="$dev" --group_reporting=1 --max_open_zones=4 \
+		"$(ioengine "psync")" \
+		--name="prep_open_zones" --rw=randwrite --offset="$off" \
+		--size="$((zone_size * 4))" --bs=4096 --io_size="$zone_size" \
+		--name=trimjob --wait_for="prep_open_zones" --rw=trim \
+		--bs="$zone_size" --offset="$off" --size="$zone_size" \
+		--name=write --wait_for="trimjob" --rw=write --bs=4096 \
+		--offset="$((off + zone_size * 4))" --size="$zone_size" \
+		>> "${logfile}.${test_number}" 2>&1
+
+	check_written $((zone_size + capacity))
+}
+
 SECONDS=0
 tests=()
 dynamic_analyzer=()
diff --git a/zbd.c b/zbd.c
index 5f1a7d7f..9455140a 100644
--- a/zbd.c
+++ b/zbd.c
@@ -254,7 +254,7 @@ static int zbd_reset_wp(struct thread_data *td, struct fio_file *f,
 }
 
 /**
- * zbd_reset_zone - reset the write pointer of a single zone
+ * __zbd_reset_zone - reset the write pointer of a single zone
  * @td: FIO thread data.
  * @f: FIO file associated with the disk for which to reset a write pointer.
  * @z: Zone to reset.
@@ -263,8 +263,8 @@ static int zbd_reset_wp(struct thread_data *td, struct fio_file *f,
  *
  * The caller must hold z->mutex.
  */
-static int zbd_reset_zone(struct thread_data *td, struct fio_file *f,
-			  struct fio_zone_info *z)
+static int __zbd_reset_zone(struct thread_data *td, struct fio_file *f,
+			    struct fio_zone_info *z)
 {
 	uint64_t offset = z->start;
 	uint64_t length = (z+1)->start - offset;
@@ -304,39 +304,65 @@ static int zbd_reset_zone(struct thread_data *td, struct fio_file *f,
 }
 
 /**
- * zbd_close_zone - Remove a zone from the open zones array.
+ * zbd_write_zone_put - Remove a zone from the write target zones array.
  * @td: FIO thread data.
- * @f: FIO file associated with the disk for which to reset a write pointer.
+ * @f: FIO file that has the write zones array to remove.
  * @zone_idx: Index of the zone to remove.
  *
  * The caller must hold f->zbd_info->mutex.
  */
-static void zbd_close_zone(struct thread_data *td, const struct fio_file *f,
-			   struct fio_zone_info *z)
+static void zbd_write_zone_put(struct thread_data *td, const struct fio_file *f,
+			       struct fio_zone_info *z)
 {
-	uint32_t ozi;
+	uint32_t zi;
 
-	if (!z->open)
+	if (!z->write)
 		return;
 
-	for (ozi = 0; ozi < f->zbd_info->num_open_zones; ozi++) {
-		if (zbd_get_zone(f, f->zbd_info->open_zones[ozi]) == z)
+	for (zi = 0; zi < f->zbd_info->num_write_zones; zi++) {
+		if (zbd_get_zone(f, f->zbd_info->write_zones[zi]) == z)
 			break;
 	}
-	if (ozi == f->zbd_info->num_open_zones)
+	if (zi == f->zbd_info->num_write_zones)
 		return;
 
-	dprint(FD_ZBD, "%s: closing zone %u\n",
+	dprint(FD_ZBD, "%s: removing zone %u from write zone array\n",
 	       f->file_name, zbd_zone_idx(f, z));
 
-	memmove(f->zbd_info->open_zones + ozi,
-		f->zbd_info->open_zones + ozi + 1,
-		(ZBD_MAX_OPEN_ZONES - (ozi + 1)) *
-		sizeof(f->zbd_info->open_zones[0]));
+	memmove(f->zbd_info->write_zones + zi,
+		f->zbd_info->write_zones + zi + 1,
+		(ZBD_MAX_WRITE_ZONES - (zi + 1)) *
+		sizeof(f->zbd_info->write_zones[0]));
+
+	f->zbd_info->num_write_zones--;
+	td->num_write_zones--;
+	z->write = 0;
+}
 
-	f->zbd_info->num_open_zones--;
-	td->num_open_zones--;
-	z->open = 0;
+/**
+ * zbd_reset_zone - reset the write pointer of a single zone and remove the zone
+ *                  from the array of write zones.
+ * @td: FIO thread data.
+ * @f: FIO file associated with the disk for which to reset a write pointer.
+ * @z: Zone to reset.
+ *
+ * Returns 0 upon success and a negative error code upon failure.
+ *
+ * The caller must hold z->mutex.
+ */
+static int zbd_reset_zone(struct thread_data *td, struct fio_file *f,
+			  struct fio_zone_info *z)
+{
+	int ret;
+
+	ret = __zbd_reset_zone(td, f, z);
+	if (ret)
+		return ret;
+
+	pthread_mutex_lock(&f->zbd_info->mutex);
+	zbd_write_zone_put(td, f, z);
+	pthread_mutex_unlock(&f->zbd_info->mutex);
+	return 0;
 }
 
 /**
@@ -404,9 +430,6 @@ static int zbd_reset_zones(struct thread_data *td, struct fio_file *f,
 			continue;
 
 		zone_lock(td, f, z);
-		pthread_mutex_lock(&f->zbd_info->mutex);
-		zbd_close_zone(td, f, z);
-		pthread_mutex_unlock(&f->zbd_info->mutex);
 
 		if (z->wp != z->start) {
 			dprint(FD_ZBD, "%s: resetting zone %u\n",
@@ -450,21 +473,19 @@ static int zbd_get_max_open_zones(struct thread_data *td, struct fio_file *f,
 }
 
 /**
- * zbd_open_zone - Add a zone to the array of open zones.
+ * __zbd_write_zone_get - Add a zone to the array of write zones.
  * @td: fio thread data.
- * @f: fio file that has the open zones to add.
+ * @f: fio file that has the write zones array to add.
  * @zone_idx: Index of the zone to add.
  *
- * Open a ZBD zone if it is not already open. Returns true if either the zone
- * was already open or if the zone was successfully added to the array of open
- * zones without exceeding the maximum number of open zones. Returns false if
- * the zone was not already open and opening the zone would cause the zone limit
- * to be exceeded.
+ * Do same operation as @zbd_write_zone_get, except it adds the zone at
+ * @zone_idx to write target zones array even when it does not have remainder
+ * space to write one block.
  */
-static bool zbd_open_zone(struct thread_data *td, const struct fio_file *f,
-			  struct fio_zone_info *z)
+static bool __zbd_write_zone_get(struct thread_data *td,
+				 const struct fio_file *f,
+				 struct fio_zone_info *z)
 {
-	const uint64_t min_bs = td->o.min_bs[DDIR_WRITE];
 	struct zoned_block_device_info *zbdi = f->zbd_info;
 	uint32_t zone_idx = zbd_zone_idx(f, z);
 	bool res = true;
@@ -476,24 +497,24 @@ static bool zbd_open_zone(struct thread_data *td, const struct fio_file *f,
 	 * Skip full zones with data verification enabled because resetting a
 	 * zone causes data loss and hence causes verification to fail.
 	 */
-	if (td->o.verify != VERIFY_NONE && zbd_zone_full(f, z, min_bs))
+	if (td->o.verify != VERIFY_NONE && zbd_zone_remainder(z) == 0)
 		return false;
 
 	/*
-	 * zbdi->max_open_zones == 0 means that there is no limit on the maximum
-	 * number of open zones. In this case, do no track open zones in
-	 * zbdi->open_zones array.
+	 * zbdi->max_write_zones == 0 means that there is no limit on the
+	 * maximum number of write target zones. In this case, do no track write
+	 * target zones in zbdi->write_zones array.
 	 */
-	if (!zbdi->max_open_zones)
+	if (!zbdi->max_write_zones)
 		return true;
 
 	pthread_mutex_lock(&zbdi->mutex);
 
-	if (z->open) {
+	if (z->write) {
 		/*
 		 * If the zone is going to be completely filled by writes
-		 * already in-flight, handle it as a full zone instead of an
-		 * open zone.
+		 * already in-flight, handle it as a full zone instead of a
+		 * write target zone.
 		 */
 		if (!zbd_zone_remainder(z))
 			res = false;
@@ -503,17 +524,17 @@ static bool zbd_open_zone(struct thread_data *td, const struct fio_file *f,
 	res = false;
 	/* Zero means no limit */
 	if (td->o.job_max_open_zones > 0 &&
-	    td->num_open_zones >= td->o.job_max_open_zones)
+	    td->num_write_zones >= td->o.job_max_open_zones)
 		goto out;
-	if (zbdi->num_open_zones >= zbdi->max_open_zones)
+	if (zbdi->num_write_zones >= zbdi->max_write_zones)
 		goto out;
 
-	dprint(FD_ZBD, "%s: opening zone %u\n",
+	dprint(FD_ZBD, "%s: adding zone %u to write zone array\n",
 	       f->file_name, zone_idx);
 
-	zbdi->open_zones[zbdi->num_open_zones++] = zone_idx;
-	td->num_open_zones++;
-	z->open = 1;
+	zbdi->write_zones[zbdi->num_write_zones++] = zone_idx;
+	td->num_write_zones++;
+	z->write = 1;
 	res = true;
 
 out:
@@ -521,6 +542,33 @@ out:
 	return res;
 }
 
+/**
+ * zbd_write_zone_get - Add a zone to the array of write zones.
+ * @td: fio thread data.
+ * @f: fio file that has the open zones to add.
+ * @zone_idx: Index of the zone to add.
+ *
+ * Add a ZBD zone to write target zones array, if it is not yet added. Returns
+ * true if either the zone was already added or if the zone was successfully
+ * added to the array without exceeding the maximum number of write zones.
+ * Returns false if the zone was not already added and addition of the zone
+ * would cause the zone limit to be exceeded.
+ */
+static bool zbd_write_zone_get(struct thread_data *td, const struct fio_file *f,
+			       struct fio_zone_info *z)
+{
+	const uint64_t min_bs = td->o.min_bs[DDIR_WRITE];
+
+	/*
+	 * Skip full zones with data verification enabled because resetting a
+	 * zone causes data loss and hence causes verification to fail.
+	 */
+	if (td->o.verify != VERIFY_NONE && zbd_zone_full(f, z, min_bs))
+		return false;
+
+	return __zbd_write_zone_get(td, f, z);
+}
+
 /* Verify whether direct I/O is used for all host-managed zoned block drives. */
 static bool zbd_using_direct_io(void)
 {
@@ -894,7 +942,7 @@ out:
 	return ret;
 }
 
-static int zbd_set_max_open_zones(struct thread_data *td, struct fio_file *f)
+static int zbd_set_max_write_zones(struct thread_data *td, struct fio_file *f)
 {
 	struct zoned_block_device_info *zbd = f->zbd_info;
 	unsigned int max_open_zones;
@@ -902,7 +950,7 @@ static int zbd_set_max_open_zones(struct thread_data *td, struct fio_file *f)
 
 	if (zbd->model != ZBD_HOST_MANAGED || td->o.ignore_zone_limits) {
 		/* Only host-managed devices have a max open limit */
-		zbd->max_open_zones = td->o.max_open_zones;
+		zbd->max_write_zones = td->o.max_open_zones;
 		goto out;
 	}
 
@@ -913,13 +961,13 @@ static int zbd_set_max_open_zones(struct thread_data *td, struct fio_file *f)
 
 	if (!max_open_zones) {
 		/* No device limit */
-		zbd->max_open_zones = td->o.max_open_zones;
+		zbd->max_write_zones = td->o.max_open_zones;
 	} else if (!td->o.max_open_zones) {
 		/* No user limit. Set limit to device limit */
-		zbd->max_open_zones = max_open_zones;
+		zbd->max_write_zones = max_open_zones;
 	} else if (td->o.max_open_zones <= max_open_zones) {
 		/* Both user limit and dev limit. User limit not too large */
-		zbd->max_open_zones = td->o.max_open_zones;
+		zbd->max_write_zones = td->o.max_open_zones;
 	} else {
 		/* Both user limit and dev limit. User limit too large */
 		td_verror(td, EINVAL,
@@ -931,15 +979,15 @@ static int zbd_set_max_open_zones(struct thread_data *td, struct fio_file *f)
 
 out:
 	/* Ensure that the limit is not larger than FIO's internal limit */
-	if (zbd->max_open_zones > ZBD_MAX_OPEN_ZONES) {
+	if (zbd->max_write_zones > ZBD_MAX_WRITE_ZONES) {
 		td_verror(td, EINVAL, "'max_open_zones' value is too large");
 		log_err("'max_open_zones' value is larger than %u\n",
-			ZBD_MAX_OPEN_ZONES);
+			ZBD_MAX_WRITE_ZONES);
 		return -EINVAL;
 	}
 
-	dprint(FD_ZBD, "%s: using max open zones limit: %"PRIu32"\n",
-	       f->file_name, zbd->max_open_zones);
+	dprint(FD_ZBD, "%s: using max write zones limit: %"PRIu32"\n",
+	       f->file_name, zbd->max_write_zones);
 
 	return 0;
 }
@@ -981,7 +1029,7 @@ static int zbd_create_zone_info(struct thread_data *td, struct fio_file *f)
 	assert(f->zbd_info);
 	f->zbd_info->model = zbd_model;
 
-	ret = zbd_set_max_open_zones(td, f);
+	ret = zbd_set_max_write_zones(td, f);
 	if (ret) {
 		zbd_free_zone_info(f);
 		return ret;
@@ -1174,7 +1222,7 @@ int zbd_setup_files(struct thread_data *td)
 			assert(f->min_zone < f->max_zone);
 
 		if (td->o.max_open_zones > 0 &&
-		    zbd->max_open_zones != td->o.max_open_zones) {
+		    zbd->max_write_zones != td->o.max_open_zones) {
 			log_err("Different 'max_open_zones' values\n");
 			return 1;
 		}
@@ -1184,34 +1232,32 @@ int zbd_setup_files(struct thread_data *td)
 		 * global max open zones limit. (As the tracking of open zones
 		 * is disabled when there is no global max open zones limit.)
 		 */
-		if (td->o.job_max_open_zones && !zbd->max_open_zones) {
+		if (td->o.job_max_open_zones && !zbd->max_write_zones) {
 			log_err("'job_max_open_zones' cannot be used without a global open zones limit\n");
 			return 1;
 		}
 
 		/*
-		 * zbd->max_open_zones is the global limit shared for all jobs
+		 * zbd->max_write_zones is the global limit shared for all jobs
 		 * that target the same zoned block device. Force sync the per
 		 * thread global limit with the actual global limit. (The real
 		 * per thread/job limit is stored in td->o.job_max_open_zones).
 		 */
-		td->o.max_open_zones = zbd->max_open_zones;
+		td->o.max_open_zones = zbd->max_write_zones;
 
 		for (zi = f->min_zone; zi < f->max_zone; zi++) {
 			z = &zbd->zone_info[zi];
 			if (z->cond != ZBD_ZONE_COND_IMP_OPEN &&
 			    z->cond != ZBD_ZONE_COND_EXP_OPEN)
 				continue;
-			if (zbd_open_zone(td, f, z))
+			if (__zbd_write_zone_get(td, f, z))
 				continue;
 			/*
 			 * If the number of open zones exceeds specified limits,
-			 * reset all extra open zones.
+			 * error out.
 			 */
-			if (zbd_reset_zone(td, f, z) < 0) {
-				log_err("Failed to reest zone %d\n", zi);
-				return 1;
-			}
+			log_err("Number of open zones exceeds max_open_zones limit\n");
+			return 1;
 		}
 	}
 
@@ -1284,12 +1330,12 @@ void zbd_file_reset(struct thread_data *td, struct fio_file *f)
 	zbd_reset_write_cnt(td, f);
 }
 
-/* Return random zone index for one of the open zones. */
+/* Return random zone index for one of the write target zones. */
 static uint32_t pick_random_zone_idx(const struct fio_file *f,
 				     const struct io_u *io_u)
 {
 	return (io_u->offset - f->file_offset) *
-		f->zbd_info->num_open_zones / f->io_size;
+		f->zbd_info->num_write_zones / f->io_size;
 }
 
 static bool any_io_in_flight(void)
@@ -1303,35 +1349,35 @@ static bool any_io_in_flight(void)
 }
 
 /*
- * Modify the offset of an I/O unit that does not refer to an open zone such
- * that it refers to an open zone. Close an open zone and open a new zone if
- * necessary. The open zone is searched across sequential zones.
+ * Modify the offset of an I/O unit that does not refer to a zone such that
+ * in write target zones array. Add a zone to or remove a zone from the lsit if
+ * necessary. The write target zone is searched across sequential zones.
  * This algorithm can only work correctly if all write pointers are
  * a multiple of the fio block size. The caller must neither hold z->mutex
  * nor f->zbd_info->mutex. Returns with z->mutex held upon success.
  */
-static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td,
-						      struct io_u *io_u)
+static struct fio_zone_info *zbd_convert_to_write_zone(struct thread_data *td,
+						       struct io_u *io_u)
 {
 	const uint64_t min_bs = td->o.min_bs[io_u->ddir];
 	struct fio_file *f = io_u->file;
 	struct zoned_block_device_info *zbdi = f->zbd_info;
 	struct fio_zone_info *z;
-	unsigned int open_zone_idx = -1;
+	unsigned int write_zone_idx = -1;
 	uint32_t zone_idx, new_zone_idx;
 	int i;
-	bool wait_zone_close;
+	bool wait_zone_write;
 	bool in_flight;
 	bool should_retry = true;
 
 	assert(is_valid_offset(f, io_u->offset));
 
-	if (zbdi->max_open_zones || td->o.job_max_open_zones) {
+	if (zbdi->max_write_zones || td->o.job_max_open_zones) {
 		/*
-		 * This statement accesses zbdi->open_zones[] on purpose
+		 * This statement accesses zbdi->write_zones[] on purpose
 		 * without locking.
 		 */
-		zone_idx = zbdi->open_zones[pick_random_zone_idx(f, io_u)];
+		zone_idx = zbdi->write_zones[pick_random_zone_idx(f, io_u)];
 	} else {
 		zone_idx = zbd_offset_to_zone_idx(f, io_u->offset);
 	}
@@ -1361,34 +1407,34 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td,
 
 		if (z->has_wp) {
 			if (z->cond != ZBD_ZONE_COND_OFFLINE &&
-			    zbdi->max_open_zones == 0 &&
+			    zbdi->max_write_zones == 0 &&
 			    td->o.job_max_open_zones == 0)
 				goto examine_zone;
-			if (zbdi->num_open_zones == 0) {
-				dprint(FD_ZBD, "%s(%s): no zones are open\n",
+			if (zbdi->num_write_zones == 0) {
+				dprint(FD_ZBD, "%s(%s): no zone is write target\n",
 				       __func__, f->file_name);
-				goto open_other_zone;
+				goto choose_other_zone;
 			}
 		}
 
 		/*
-		 * List of opened zones is per-device, shared across all
+		 * Array of write target zones is per-device, shared across all
 		 * threads. Start with quasi-random candidate zone. Ignore
 		 * zones which don't belong to thread's offset/size area.
 		 */
-		open_zone_idx = pick_random_zone_idx(f, io_u);
-		assert(!open_zone_idx ||
-		       open_zone_idx < zbdi->num_open_zones);
-		tmp_idx = open_zone_idx;
+		write_zone_idx = pick_random_zone_idx(f, io_u);
+		assert(!write_zone_idx ||
+		       write_zone_idx < zbdi->num_write_zones);
+		tmp_idx = write_zone_idx;
 
-		for (i = 0; i < zbdi->num_open_zones; i++) {
+		for (i = 0; i < zbdi->num_write_zones; i++) {
 			uint32_t tmpz;
 
-			if (tmp_idx >= zbdi->num_open_zones)
+			if (tmp_idx >= zbdi->num_write_zones)
 				tmp_idx = 0;
-			tmpz = zbdi->open_zones[tmp_idx];
+			tmpz = zbdi->write_zones[tmp_idx];
 			if (f->min_zone <= tmpz && tmpz < f->max_zone) {
-				open_zone_idx = tmp_idx;
+				write_zone_idx = tmp_idx;
 				goto found_candidate_zone;
 			}
 
@@ -1406,7 +1452,7 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td,
 		return NULL;
 
 found_candidate_zone:
-		new_zone_idx = zbdi->open_zones[open_zone_idx];
+		new_zone_idx = zbdi->write_zones[write_zone_idx];
 		if (new_zone_idx == zone_idx)
 			break;
 		zone_idx = new_zone_idx;
@@ -1425,32 +1471,32 @@ examine_zone:
 		goto out;
 	}
 
-open_other_zone:
-	/* Check if number of open zones reaches one of limits. */
-	wait_zone_close =
-		zbdi->num_open_zones == f->max_zone - f->min_zone ||
-		(zbdi->max_open_zones &&
-		 zbdi->num_open_zones == zbdi->max_open_zones) ||
+choose_other_zone:
+	/* Check if number of write target zones reaches one of limits. */
+	wait_zone_write =
+		zbdi->num_write_zones == f->max_zone - f->min_zone ||
+		(zbdi->max_write_zones &&
+		 zbdi->num_write_zones == zbdi->max_write_zones) ||
 		(td->o.job_max_open_zones &&
-		 td->num_open_zones == td->o.job_max_open_zones);
+		 td->num_write_zones == td->o.job_max_open_zones);
 
 	pthread_mutex_unlock(&zbdi->mutex);
 
 	/* Only z->mutex is held. */
 
 	/*
-	 * When number of open zones reaches to one of limits, wait for
-	 * zone close before opening a new zone.
+	 * When number of write target zones reaches to one of limits, wait for
+	 * zone write completion to one of them before trying a new zone.
 	 */
-	if (wait_zone_close) {
+	if (wait_zone_write) {
 		dprint(FD_ZBD,
-		       "%s(%s): quiesce to allow open zones to close\n",
+		       "%s(%s): quiesce to remove a zone from write target zones array\n",
 		       __func__, f->file_name);
 		io_u_quiesce(td);
 	}
 
 retry:
-	/* Zone 'z' is full, so try to open a new zone. */
+	/* Zone 'z' is full, so try to choose a new zone. */
 	for (i = f->io_size / zbdi->zone_size; i > 0; i--) {
 		zone_idx++;
 		if (z->has_wp)
@@ -1465,18 +1511,18 @@ retry:
 		if (!z->has_wp)
 			continue;
 		zone_lock(td, f, z);
-		if (z->open)
+		if (z->write)
 			continue;
-		if (zbd_open_zone(td, f, z))
+		if (zbd_write_zone_get(td, f, z))
 			goto out;
 	}
 
 	/* Only z->mutex is held. */
 
-	/* Check whether the write fits in any of the already opened zones. */
+	/* Check whether the write fits in any of the write target zones. */
 	pthread_mutex_lock(&zbdi->mutex);
-	for (i = 0; i < zbdi->num_open_zones; i++) {
-		zone_idx = zbdi->open_zones[i];
+	for (i = 0; i < zbdi->num_write_zones; i++) {
+		zone_idx = zbdi->write_zones[i];
 		if (zone_idx < f->min_zone || zone_idx >= f->max_zone)
 			continue;
 		pthread_mutex_unlock(&zbdi->mutex);
@@ -1492,13 +1538,14 @@ retry:
 
 	/*
 	 * When any I/O is in-flight or when all I/Os in-flight get completed,
-	 * the I/Os might have closed zones then retry the steps to open a zone.
-	 * Before retry, call io_u_quiesce() to complete in-flight writes.
+	 * the I/Os might have removed zones from the write target array then
+	 * retry the steps to choose a zone. Before retry, call io_u_quiesce()
+	 * to complete in-flight writes.
 	 */
 	in_flight = any_io_in_flight();
 	if (in_flight || should_retry) {
 		dprint(FD_ZBD,
-		       "%s(%s): wait zone close and retry open zones\n",
+		       "%s(%s): wait zone write and retry write target zone selection\n",
 		       __func__, f->file_name);
 		pthread_mutex_unlock(&zbdi->mutex);
 		zone_unlock(z);
@@ -1512,7 +1559,7 @@ retry:
 
 	zone_unlock(z);
 
-	dprint(FD_ZBD, "%s(%s): did not open another zone\n",
+	dprint(FD_ZBD, "%s(%s): did not choose another write zone\n",
 	       __func__, f->file_name);
 
 	return NULL;
@@ -1582,7 +1629,8 @@ zbd_find_zone(struct thread_data *td, struct io_u *io_u, uint64_t min_bytes,
  * @io_u: I/O unit
  * @z: zone info pointer
  *
- * If the write command made the zone full, close it.
+ * If the write command made the zone full, remove it from the write target
+ * zones array.
  *
  * The caller must hold z->mutex.
  */
@@ -1594,7 +1642,7 @@ static void zbd_end_zone_io(struct thread_data *td, const struct io_u *io_u,
 	if (io_u->ddir == DDIR_WRITE &&
 	    io_u->offset + io_u->buflen >= zbd_zone_capacity_end(z)) {
 		pthread_mutex_lock(&f->zbd_info->mutex);
-		zbd_close_zone(td, f, z);
+		zbd_write_zone_put(td, f, z);
 		pthread_mutex_unlock(&f->zbd_info->mutex);
 	}
 }
@@ -1954,7 +2002,7 @@ retry:
 		if (zbd_zone_remainder(zb) > 0 &&
 		    zbd_zone_remainder(zb) < min_bs) {
 			pthread_mutex_lock(&f->zbd_info->mutex);
-			zbd_close_zone(td, f, zb);
+			zbd_write_zone_put(td, f, zb);
 			pthread_mutex_unlock(&f->zbd_info->mutex);
 			dprint(FD_ZBD,
 			       "%s: finish zone %d\n",
@@ -1977,11 +2025,11 @@ retry:
 			zone_lock(td, f, zb);
 		}
 
-		if (!zbd_open_zone(td, f, zb)) {
+		if (!zbd_write_zone_get(td, f, zb)) {
 			zone_unlock(zb);
-			zb = zbd_convert_to_open_zone(td, io_u);
+			zb = zbd_convert_to_write_zone(td, io_u);
 			if (!zb) {
-				dprint(FD_IO, "%s: can't convert to open zone",
+				dprint(FD_IO, "%s: can't convert to write target zone",
 				       f->file_name);
 				goto eof;
 			}
@@ -2023,7 +2071,7 @@ retry:
 			 */
 			io_u_quiesce(td);
 			zb->reset_zone = 0;
-			if (zbd_reset_zone(td, f, zb) < 0)
+			if (__zbd_reset_zone(td, f, zb) < 0)
 				goto eof;
 
 			if (zb->capacity < min_bs) {
@@ -2142,7 +2190,7 @@ char *zbd_write_status(const struct thread_stat *ts)
  * Return io_u_completed when reset zone succeeds. Return 0 when the target zone
  * does not have write pointer. On error, return negative errno.
  */
-int zbd_do_io_u_trim(const struct thread_data *td, struct io_u *io_u)
+int zbd_do_io_u_trim(struct thread_data *td, struct io_u *io_u)
 {
 	struct fio_file *f = io_u->file;
 	struct fio_zone_info *z;
diff --git a/zbd.h b/zbd.h
index 05189555..f0ac9876 100644
--- a/zbd.h
+++ b/zbd.h
@@ -29,8 +29,8 @@ enum io_u_action {
  * @type: zone type (BLK_ZONE_TYPE_*)
  * @cond: zone state (BLK_ZONE_COND_*)
  * @has_wp: whether or not this zone can have a valid write pointer
- * @open: whether or not this zone is currently open. Only relevant if
- *		max_open_zones > 0.
+ * @write: whether or not this zone is the write target at this moment. Only
+ *              relevant if zbd->max_open_zones > 0.
  * @reset_zone: whether or not this zone should be reset before writing to it
  */
 struct fio_zone_info {
@@ -41,16 +41,17 @@ struct fio_zone_info {
 	enum zbd_zone_type	type:2;
 	enum zbd_zone_cond	cond:4;
 	unsigned int		has_wp:1;
-	unsigned int		open:1;
+	unsigned int		write:1;
 	unsigned int		reset_zone:1;
 };
 
 /**
  * zoned_block_device_info - zoned block device characteristics
  * @model: Device model.
- * @max_open_zones: global limit on the number of simultaneously opened
- *	sequential write zones. A zero value means unlimited open zones,
- *	and that open zones will not be tracked in the open_zones array.
+ * @max_write_zones: global limit on the number of sequential write zones which
+ *      are simultaneously written. A zero value means unlimited zones of
+ *      simultaneous writes and that write target zones will not be tracked in
+ *      the write_zones array.
  * @mutex: Protects the modifiable members in this structure (refcount and
  *		num_open_zones).
  * @zone_size: size of a single zone in bytes.
@@ -61,10 +62,10 @@ struct fio_zone_info {
  *		if the zone size is not a power of 2.
  * @nr_zones: number of zones
  * @refcount: number of fio files that share this structure
- * @num_open_zones: number of open zones
+ * @num_write_zones: number of write target zones
  * @write_cnt: Number of writes since the latest zone reset triggered by
  *	       the zone_reset_frequency fio job parameter.
- * @open_zones: zone numbers of open zones
+ * @write_zones: zone numbers of write target zones
  * @zone_info: description of the individual zones
  *
  * Only devices for which all zones have the same size are supported.
@@ -73,7 +74,7 @@ struct fio_zone_info {
  */
 struct zoned_block_device_info {
 	enum zbd_zoned_model	model;
-	uint32_t		max_open_zones;
+	uint32_t		max_write_zones;
 	pthread_mutex_t		mutex;
 	uint64_t		zone_size;
 	uint64_t		wp_valid_data_bytes;
@@ -82,9 +83,9 @@ struct zoned_block_device_info {
 	uint32_t		zone_size_log2;
 	uint32_t		nr_zones;
 	uint32_t		refcount;
-	uint32_t		num_open_zones;
+	uint32_t		num_write_zones;
 	uint32_t		write_cnt;
-	uint32_t		open_zones[ZBD_MAX_OPEN_ZONES];
+	uint32_t		write_zones[ZBD_MAX_WRITE_ZONES];
 	struct fio_zone_info	zone_info[0];
 };
 
@@ -99,7 +100,7 @@ enum fio_ddir zbd_adjust_ddir(struct thread_data *td, struct io_u *io_u,
 			      enum fio_ddir ddir);
 enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u);
 char *zbd_write_status(const struct thread_stat *ts);
-int zbd_do_io_u_trim(const struct thread_data *td, struct io_u *io_u);
+int zbd_do_io_u_trim(struct thread_data *td, struct io_u *io_u);
 
 static inline void zbd_close_file(struct fio_file *f)
 {
diff --git a/zbd_types.h b/zbd_types.h
index 0a8630cb..5f44f308 100644
--- a/zbd_types.h
+++ b/zbd_types.h
@@ -8,7 +8,7 @@
 
 #include <inttypes.h>
 
-#define ZBD_MAX_OPEN_ZONES	4096
+#define ZBD_MAX_WRITE_ZONES	4096
 
 /*
  * Zoned block device models.

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-06-02 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-06-02 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 4820d46cef75f806d8c95afaa77f86ded4e3603e:

  ci: disable tls for msys2 builds (2023-05-26 20:09:53 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 1b4ba547cf45377fffc7a1e60728369997cc7a9b:

  t/run-fio-tests: address issues identified by pylint (2023-06-01 14:12:41 -0400)

----------------------------------------------------------------
Vincent Fu (3):
      t/nvmept.py: test script for io_uring_cmd NVMe pass through
      t/run-fio-tests: integrate t/nvmept.py
      t/run-fio-tests: address issues identified by pylint

 t/nvmept.py        | 414 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 t/run-fio-tests.py | 191 +++++++++++++-----------
 2 files changed, 521 insertions(+), 84 deletions(-)
 create mode 100755 t/nvmept.py

---

Diff of recent changes:

diff --git a/t/nvmept.py b/t/nvmept.py
new file mode 100755
index 00000000..a25192f2
--- /dev/null
+++ b/t/nvmept.py
@@ -0,0 +1,414 @@
+#!/usr/bin/env python3
+"""
+# nvmept.py
+#
+# Test fio's io_uring_cmd ioengine with NVMe pass-through commands.
+#
+# USAGE
+# see python3 nvmept.py --help
+#
+# EXAMPLES
+# python3 t/nvmept.py --dut /dev/ng0n1
+# python3 t/nvmept.py --dut /dev/ng1n1 -f ./fio
+#
+# REQUIREMENTS
+# Python 3.6
+#
+"""
+import os
+import sys
+import json
+import time
+import locale
+import argparse
+import subprocess
+from pathlib import Path
+
+class FioTest():
+    """fio test."""
+
+    def __init__(self, artifact_root, test_opts, debug):
+        """
+        artifact_root   root directory for artifacts (subdirectory will be created under here)
+        test            test specification
+        """
+        self.artifact_root = artifact_root
+        self.test_opts = test_opts
+        self.debug = debug
+        self.filename_stub = None
+        self.filenames = {}
+        self.json_data = None
+
+        self.test_dir = os.path.abspath(os.path.join(self.artifact_root,
+                                     f"{self.test_opts['test_id']:03d}"))
+        if not os.path.exists(self.test_dir):
+            os.mkdir(self.test_dir)
+
+        self.filename_stub = f"pt{self.test_opts['test_id']:03d}"
+        self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command")
+        self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout")
+        self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr")
+        self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode")
+        self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output")
+
+    def run_fio(self, fio_path):
+        """Run a test."""
+
+        fio_args = [
+            "--name=nvmept",
+            "--ioengine=io_uring_cmd",
+            "--cmd_type=nvme",
+            "--iodepth=8",
+            "--iodepth_batch=4",
+            "--iodepth_batch_complete=4",
+            f"--filename={self.test_opts['filename']}",
+            f"--rw={self.test_opts['rw']}",
+            f"--output={self.filenames['output']}",
+            f"--output-format={self.test_opts['output-format']}",
+        ]
+        for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles',
+                    'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait',
+                    'time_based', 'runtime', 'verify', 'io_size']:
+            if opt in self.test_opts:
+                option = f"--{opt}={self.test_opts[opt]}"
+                fio_args.append(option)
+
+        command = [fio_path] + fio_args
+        with open(self.filenames['command'], "w+",
+                  encoding=locale.getpreferredencoding()) as command_file:
+            command_file.write(" ".join(command))
+
+        passed = True
+
+        try:
+            with open(self.filenames['stdout'], "w+",
+                      encoding=locale.getpreferredencoding()) as stdout_file, \
+                open(self.filenames['stderr'], "w+",
+                     encoding=locale.getpreferredencoding()) as stderr_file, \
+                open(self.filenames['exitcode'], "w+",
+                     encoding=locale.getpreferredencoding()) as exitcode_file:
+                proc = None
+                # Avoid using subprocess.run() here because when a timeout occurs,
+                # fio will be stopped with SIGKILL. This does not give fio a
+                # chance to clean up and means that child processes may continue
+                # running and submitting IO.
+                proc = subprocess.Popen(command,
+                                        stdout=stdout_file,
+                                        stderr=stderr_file,
+                                        cwd=self.test_dir,
+                                        universal_newlines=True)
+                proc.communicate(timeout=300)
+                exitcode_file.write(f'{proc.returncode}\n')
+                passed &= (proc.returncode == 0)
+        except subprocess.TimeoutExpired:
+            proc.terminate()
+            proc.communicate()
+            assert proc.poll()
+            print("Timeout expired")
+            passed = False
+        except Exception:
+            if proc:
+                if not proc.poll():
+                    proc.terminate()
+                    proc.communicate()
+            print(f"Exception: {sys.exc_info()}")
+            passed = False
+
+        if passed:
+            if 'output-format' in self.test_opts and 'json' in \
+                    self.test_opts['output-format']:
+                if not self.get_json():
+                    print('Unable to decode JSON data')
+                    passed = False
+
+        return passed
+
+    def get_json(self):
+        """Convert fio JSON output into a python JSON object"""
+
+        filename = self.filenames['output']
+        with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
+            file_data = file.read()
+
+        #
+        # Sometimes fio informational messages are included at the top of the
+        # JSON output, especially under Windows. Try to decode output as JSON
+        # data, lopping off up to the first four lines
+        #
+        lines = file_data.splitlines()
+        for i in range(5):
+            file_data = '\n'.join(lines[i:])
+            try:
+                self.json_data = json.loads(file_data)
+            except json.JSONDecodeError:
+                continue
+            else:
+                return True
+
+        return False
+
+    @staticmethod
+    def check_empty(job):
+        """
+        Make sure JSON data is empty.
+
+        Some data structures should be empty. This function makes sure that they are.
+
+        job         JSON object that we need to check for emptiness
+        """
+
+        return job['total_ios'] == 0 and \
+                job['slat_ns']['N'] == 0 and \
+                job['clat_ns']['N'] == 0 and \
+                job['lat_ns']['N'] == 0
+
+    def check_all_ddirs(self, ddir_nonzero, job):
+        """
+        Iterate over the data directions and check whether each is
+        appropriately empty or not.
+        """
+
+        retval = True
+        ddirlist = ['read', 'write', 'trim']
+
+        for ddir in ddirlist:
+            if ddir in ddir_nonzero:
+                if self.check_empty(job[ddir]):
+                    print(f"Unexpected zero {ddir} data found in output")
+                    retval = False
+            else:
+                if not self.check_empty(job[ddir]):
+                    print(f"Unexpected {ddir} data found in output")
+                    retval = False
+
+        return retval
+
+    def check(self):
+        """Check test output."""
+
+        raise NotImplementedError()
+
+
+class PTTest(FioTest):
+    """
+    NVMe pass-through test class. Check to make sure output for selected data
+    direction(s) is non-zero and that zero data appears for other directions.
+    """
+
+    def check(self):
+        if 'rw' not in self.test_opts:
+            return True
+
+        job = self.json_data['jobs'][0]
+        retval = True
+
+        if self.test_opts['rw'] in ['read', 'randread']:
+            retval = self.check_all_ddirs(['read'], job)
+        elif self.test_opts['rw'] in ['write', 'randwrite']:
+            if 'verify' not in self.test_opts:
+                retval = self.check_all_ddirs(['write'], job)
+            else:
+                retval = self.check_all_ddirs(['read', 'write'], job)
+        elif self.test_opts['rw'] in ['trim', 'randtrim']:
+            retval = self.check_all_ddirs(['trim'], job)
+        elif self.test_opts['rw'] in ['readwrite', 'randrw']:
+            retval = self.check_all_ddirs(['read', 'write'], job)
+        elif self.test_opts['rw'] in ['trimwrite', 'randtrimwrite']:
+            retval = self.check_all_ddirs(['trim', 'write'], job)
+        else:
+            print(f"Unhandled rw value {self.test_opts['rw']}")
+            retval = False
+
+        return retval
+
+
+def parse_args():
+    """Parse command-line arguments."""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
+    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
+    parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
+    parser.add_argument('-s', '--skip', nargs='+', type=int,
+                        help='list of test(s) to skip')
+    parser.add_argument('-o', '--run-only', nargs='+', type=int,
+                        help='list of test(s) to run, skipping all others')
+    parser.add_argument('--dut', help='target NVMe character device to test '
+                        '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
+    args = parser.parse_args()
+
+    return args
+
+
+def main():
+    """Run tests using fio's io_uring_cmd ioengine to send NVMe pass through commands."""
+
+    args = parse_args()
+
+    artifact_root = args.artifact_root if args.artifact_root else \
+        f"nvmept-test-{time.strftime('%Y%m%d-%H%M%S')}"
+    os.mkdir(artifact_root)
+    print(f"Artifact directory is {artifact_root}")
+
+    if args.fio:
+        fio = str(Path(args.fio).absolute())
+    else:
+        fio = 'fio'
+    print(f"fio path is {fio}")
+
+    test_list = [
+        {
+            "test_id": 1,
+            "rw": 'read',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 2,
+            "rw": 'randread',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 3,
+            "rw": 'write',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 4,
+            "rw": 'randwrite',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 5,
+            "rw": 'trim',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 6,
+            "rw": 'randtrim',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 7,
+            "rw": 'write',
+            "io_size": 1024*1024,
+            "verify": "crc32c",
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 8,
+            "rw": 'randwrite',
+            "io_size": 1024*1024,
+            "verify": "crc32c",
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 9,
+            "rw": 'readwrite',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 10,
+            "rw": 'randrw',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 11,
+            "rw": 'trimwrite',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 12,
+            "rw": 'randtrimwrite',
+            "timebased": 1,
+            "runtime": 3,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 13,
+            "rw": 'randread',
+            "timebased": 1,
+            "runtime": 3,
+            "fixedbufs": 1,
+            "nonvectored": 1,
+            "force_async": 1,
+            "registerfiles": 1,
+            "sqthread_poll": 1,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+        {
+            "test_id": 14,
+            "rw": 'randwrite',
+            "timebased": 1,
+            "runtime": 3,
+            "fixedbufs": 1,
+            "nonvectored": 1,
+            "force_async": 1,
+            "registerfiles": 1,
+            "sqthread_poll": 1,
+            "output-format": "json",
+            "test_obj": PTTest,
+        },
+    ]
+
+    passed = 0
+    failed = 0
+    skipped = 0
+
+    for test in test_list:
+        if (args.skip and test['test_id'] in args.skip) or \
+           (args.run_only and test['test_id'] not in args.run_only):
+            skipped = skipped + 1
+            outcome = 'SKIPPED (User request)'
+        else:
+            test['filename'] = args.dut
+            test_obj = test['test_obj'](artifact_root, test, args.debug)
+            status = test_obj.run_fio(fio)
+            if status:
+                status = test_obj.check()
+            if status:
+                passed = passed + 1
+                outcome = 'PASSED'
+            else:
+                failed = failed + 1
+                outcome = 'FAILED'
+
+        print(f"**********Test {test['test_id']} {outcome}**********")
+
+    print(f"{passed} tests passed, {failed} failed, {skipped} skipped")
+
+    sys.exit(failed)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index 71e3e5a6..c91deed4 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -79,22 +79,22 @@ class FioTest():
 
         self.artifact_root = artifact_root
         self.testnum = testnum
-        self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
+        self.test_dir = os.path.join(artifact_root, f"{testnum:04d}")
         if not os.path.exists(self.test_dir):
             os.mkdir(self.test_dir)
 
         self.command_file = os.path.join(
             self.test_dir,
-            "{0}.command".format(os.path.basename(self.exe_path)))
+            f"{os.path.basename(self.exe_path)}.command")
         self.stdout_file = os.path.join(
             self.test_dir,
-            "{0}.stdout".format(os.path.basename(self.exe_path)))
+            f"{os.path.basename(self.exe_path)}.stdout")
         self.stderr_file = os.path.join(
             self.test_dir,
-            "{0}.stderr".format(os.path.basename(self.exe_path)))
+            f"{os.path.basename(self.exe_path)}.stderr")
         self.exitcode_file = os.path.join(
             self.test_dir,
-            "{0}.exitcode".format(os.path.basename(self.exe_path)))
+            f"{os.path.basename(self.exe_path)}.exitcode")
 
     def run(self):
         """Run the test."""
@@ -126,7 +126,7 @@ class FioExeTest(FioTest):
 
         command = [self.exe_path] + self.parameters
         command_file = open(self.command_file, "w+")
-        command_file.write("%s\n" % command)
+        command_file.write(f"{command}\n")
         command_file.close()
 
         stdout_file = open(self.stdout_file, "w+")
@@ -144,7 +144,7 @@ class FioExeTest(FioTest):
                                     cwd=self.test_dir,
                                     universal_newlines=True)
             proc.communicate(timeout=self.success['timeout'])
-            exitcode_file.write('{0}\n'.format(proc.returncode))
+            exitcode_file.write(f'{proc.returncode}\n')
             logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
             self.output['proc'] = proc
         except subprocess.TimeoutExpired:
@@ -169,7 +169,7 @@ class FioExeTest(FioTest):
 
         if 'proc' not in self.output:
             if self.output['failure'] == 'timeout':
-                self.failure_reason = "{0} timeout,".format(self.failure_reason)
+                self.failure_reason = f"{self.failure_reason} timeout,"
             else:
                 assert self.output['failure'] == 'exception'
                 self.failure_reason = '{0} exception: {1}, {2}'.format(
@@ -183,21 +183,21 @@ class FioExeTest(FioTest):
             if self.success['zero_return']:
                 if self.output['proc'].returncode != 0:
                     self.passed = False
-                    self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
+                    self.failure_reason = f"{self.failure_reason} non-zero return code,"
             else:
                 if self.output['proc'].returncode == 0:
-                    self.failure_reason = "{0} zero return code,".format(self.failure_reason)
+                    self.failure_reason = f"{self.failure_reason} zero return code,"
                     self.passed = False
 
         stderr_size = os.path.getsize(self.stderr_file)
         if 'stderr_empty' in self.success:
             if self.success['stderr_empty']:
                 if stderr_size != 0:
-                    self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
+                    self.failure_reason = f"{self.failure_reason} stderr not empty,"
                     self.passed = False
             else:
                 if stderr_size == 0:
-                    self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
+                    self.failure_reason = f"{self.failure_reason} stderr empty,"
                     self.passed = False
 
 
@@ -223,11 +223,11 @@ class FioJobTest(FioExeTest):
         self.output_format = output_format
         self.precon_failed = False
         self.json_data = None
-        self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
+        self.fio_output = f"{os.path.basename(self.fio_job)}.output"
         self.fio_args = [
             "--max-jobs=16",
-            "--output-format={0}".format(self.output_format),
-            "--output={0}".format(self.fio_output),
+            f"--output-format={self.output_format}",
+            f"--output={self.fio_output}",
             self.fio_job,
             ]
         FioExeTest.__init__(self, fio_path, self.fio_args, success)
@@ -235,20 +235,20 @@ class FioJobTest(FioExeTest):
     def setup(self, artifact_root, testnum):
         """Setup instance variables for fio job test."""
 
-        super(FioJobTest, self).setup(artifact_root, testnum)
+        super().setup(artifact_root, testnum)
 
         self.command_file = os.path.join(
             self.test_dir,
-            "{0}.command".format(os.path.basename(self.fio_job)))
+            f"{os.path.basename(self.fio_job)}.command")
         self.stdout_file = os.path.join(
             self.test_dir,
-            "{0}.stdout".format(os.path.basename(self.fio_job)))
+            f"{os.path.basename(self.fio_job)}.stdout")
         self.stderr_file = os.path.join(
             self.test_dir,
-            "{0}.stderr".format(os.path.basename(self.fio_job)))
+            f"{os.path.basename(self.fio_job)}.stderr")
         self.exitcode_file = os.path.join(
             self.test_dir,
-            "{0}.exitcode".format(os.path.basename(self.fio_job)))
+            f"{os.path.basename(self.fio_job)}.exitcode")
 
     def run_pre_job(self):
         """Run fio job precondition step."""
@@ -269,7 +269,7 @@ class FioJobTest(FioExeTest):
             self.run_pre_job()
 
         if not self.precon_failed:
-            super(FioJobTest, self).run()
+            super().run()
         else:
             logging.debug("Test %d: precondition step failed", self.testnum)
 
@@ -295,7 +295,7 @@ class FioJobTest(FioExeTest):
             with open(filename, "r") as output_file:
                 file_data = output_file.read()
         except OSError:
-            self.failure_reason += " unable to read file {0}".format(filename)
+            self.failure_reason += f" unable to read file {filename}"
             self.passed = False
 
         return file_data
@@ -305,10 +305,10 @@ class FioJobTest(FioExeTest):
 
         if self.precon_failed:
             self.passed = False
-            self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} precondition step failed,"
             return
 
-        super(FioJobTest, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -330,7 +330,7 @@ class FioJobTest(FioExeTest):
         try:
             self.json_data = json.loads(file_data)
         except json.JSONDecodeError:
-            self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
             self.passed = False
 
 
@@ -339,16 +339,16 @@ class FioJobTest_t0005(FioJobTest):
     Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
 
     def check_result(self):
-        super(FioJobTest_t0005, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
 
         if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
-            self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
             self.passed = False
         if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
-            self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} bytes written mismatch,"
             self.passed = False
 
 
@@ -357,7 +357,7 @@ class FioJobTest_t0006(FioJobTest):
     Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
 
     def check_result(self):
-        super(FioJobTest_t0006, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -366,7 +366,7 @@ class FioJobTest_t0006(FioJobTest):
             / self.json_data['jobs'][0]['write']['io_kbytes']
         logging.debug("Test %d: ratio: %f", self.testnum, ratio)
         if ratio < 1.99 or ratio > 2.01:
-            self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} read/write ratio mismatch,"
             self.passed = False
 
 
@@ -375,13 +375,13 @@ class FioJobTest_t0007(FioJobTest):
     Confirm that read['io_kbytes'] = 87040"""
 
     def check_result(self):
-        super(FioJobTest_t0007, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
 
         if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
-            self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
             self.passed = False
 
 
@@ -397,7 +397,7 @@ class FioJobTest_t0008(FioJobTest):
     the blocks originally written will be read."""
 
     def check_result(self):
-        super(FioJobTest_t0008, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -406,10 +406,10 @@ class FioJobTest_t0008(FioJobTest):
         logging.debug("Test %d: ratio: %f", self.testnum, ratio)
 
         if ratio < 0.97 or ratio > 1.03:
-            self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} bytes written mismatch,"
             self.passed = False
         if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
-            self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
             self.passed = False
 
 
@@ -418,7 +418,7 @@ class FioJobTest_t0009(FioJobTest):
     Confirm that runtime >= 60s"""
 
     def check_result(self):
-        super(FioJobTest_t0009, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -426,7 +426,7 @@ class FioJobTest_t0009(FioJobTest):
         logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed'])
 
         if self.json_data['jobs'][0]['elapsed'] < 60:
-            self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} elapsed time mismatch,"
             self.passed = False
 
 
@@ -436,7 +436,7 @@ class FioJobTest_t0012(FioJobTest):
     job1,job2,job3 respectively"""
 
     def check_result(self):
-        super(FioJobTest_t0012, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -484,7 +484,7 @@ class FioJobTest_t0014(FioJobTest):
     re-calibrate the activity dynamically"""
 
     def check_result(self):
-        super(FioJobTest_t0014, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -539,7 +539,7 @@ class FioJobTest_t0015(FioJobTest):
     Confirm that mean(slat) + mean(clat) = mean(tlat)"""
 
     def check_result(self):
-        super(FioJobTest_t0015, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -560,7 +560,7 @@ class FioJobTest_t0019(FioJobTest):
     Confirm that all offsets were touched sequentially"""
 
     def check_result(self):
-        super(FioJobTest_t0019, self).check_result()
+        super().check_result()
 
         bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
         file_data = self.get_file_fail(bw_log_filename)
@@ -576,13 +576,13 @@ class FioJobTest_t0019(FioJobTest):
             cur = int(line.split(',')[4])
             if cur - prev != 4096:
                 self.passed = False
-                self.failure_reason = "offsets {0}, {1} not sequential".format(prev, cur)
+                self.failure_reason = f"offsets {prev}, {cur} not sequential"
                 return
             prev = cur
 
         if cur/4096 != 255:
             self.passed = False
-            self.failure_reason = "unexpected last offset {0}".format(cur)
+            self.failure_reason = f"unexpected last offset {cur}"
 
 
 class FioJobTest_t0020(FioJobTest):
@@ -590,7 +590,7 @@ class FioJobTest_t0020(FioJobTest):
     Confirm that almost all offsets were touched non-sequentially"""
 
     def check_result(self):
-        super(FioJobTest_t0020, self).check_result()
+        super().check_result()
 
         bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
         file_data = self.get_file_fail(bw_log_filename)
@@ -611,14 +611,14 @@ class FioJobTest_t0020(FioJobTest):
 
         if len(offsets) != 256:
             self.passed = False
-            self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets))
+            self.failure_reason += f" number of offsets is {len(offsets)} instead of 256"
 
         for i in range(256):
             if not i in offsets:
                 self.passed = False
-                self.failure_reason += " missing offset {0}".format(i*4096)
+                self.failure_reason += f" missing offset {i * 4096}"
 
-        (z, p) = runstest_1samp(list(offsets))
+        (_, p) = runstest_1samp(list(offsets))
         if p < 0.05:
             self.passed = False
             self.failure_reason += f" runs test failed with p = {p}"
@@ -628,7 +628,7 @@ class FioJobTest_t0022(FioJobTest):
     """Test consists of fio test job t0022"""
 
     def check_result(self):
-        super(FioJobTest_t0022, self).check_result()
+        super().check_result()
 
         bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
         file_data = self.get_file_fail(bw_log_filename)
@@ -655,7 +655,7 @@ class FioJobTest_t0022(FioJobTest):
         # 10 is an arbitrary threshold
         if seq_count > 10:
             self.passed = False
-            self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count)
+            self.failure_reason = f"too many ({seq_count}) consecutive offsets"
 
         if len(offsets) == filesize/bs:
             self.passed = False
@@ -690,7 +690,7 @@ class FioJobTest_t0023(FioJobTest):
                         bw_log_filename, line)
                     break
             else:
-                if ddir != 1:
+                if ddir != 1:   # pylint: disable=no-else-break
                     self.passed = False
                     self.failure_reason += " {0}: trim not preceeded by write: {1}".format(
                         bw_log_filename, line)
@@ -701,11 +701,13 @@ class FioJobTest_t0023(FioJobTest):
                         self.failure_reason += " {0}: block size does not match: {1}".format(
                             bw_log_filename, line)
                         break
+
                     if prev_offset != offset:
                         self.passed = False
                         self.failure_reason += " {0}: offset does not match: {1}".format(
                             bw_log_filename, line)
                         break
+
             prev_ddir = ddir
             prev_bs = bs
             prev_offset = offset
@@ -750,7 +752,7 @@ class FioJobTest_t0023(FioJobTest):
 
 
     def check_result(self):
-        super(FioJobTest_t0023, self).check_result()
+        super().check_result()
 
         filesize = 1024*1024
 
@@ -792,7 +794,7 @@ class FioJobTest_t0024(FioJobTest_t0023):
 class FioJobTest_t0025(FioJobTest):
     """Test experimental verify read backs written data pattern."""
     def check_result(self):
-        super(FioJobTest_t0025, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -802,7 +804,7 @@ class FioJobTest_t0025(FioJobTest):
 
 class FioJobTest_t0027(FioJobTest):
     def setup(self, *args, **kws):
-        super(FioJobTest_t0027, self).setup(*args, **kws)
+        super().setup(*args, **kws)
         self.pattern_file = os.path.join(self.test_dir, "t0027.pattern")
         self.output_file = os.path.join(self.test_dir, "t0027file")
         self.pattern = os.urandom(16 << 10)
@@ -810,7 +812,7 @@ class FioJobTest_t0027(FioJobTest):
             f.write(self.pattern)
 
     def check_result(self):
-        super(FioJobTest_t0027, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -828,7 +830,7 @@ class FioJobTest_iops_rate(FioJobTest):
     With two runs of fio-3.16 I observed a ratio of 8.3"""
 
     def check_result(self):
-        super(FioJobTest_iops_rate, self).check_result()
+        super().check_result()
 
         if not self.passed:
             return
@@ -841,11 +843,11 @@ class FioJobTest_iops_rate(FioJobTest):
         logging.debug("Test %d: ratio: %f", self.testnum, ratio)
 
         if iops1 < 950 or iops1 > 1050:
-            self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} iops value mismatch,"
             self.passed = False
 
         if ratio < 6 or ratio > 10:
-            self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
+            self.failure_reason = f"{self.failure_reason} iops ratio mismatch,"
             self.passed = False
 
 
@@ -863,8 +865,9 @@ class Requirements():
     _not_windows = False
     _unittests = False
     _cpucount4 = False
+    _nvmecdev = False
 
-    def __init__(self, fio_root):
+    def __init__(self, fio_root, args):
         Requirements._not_macos = platform.system() != "Darwin"
         Requirements._not_windows = platform.system() != "Windows"
         Requirements._linux = platform.system() == "Linux"
@@ -873,7 +876,7 @@ class Requirements():
             config_file = os.path.join(fio_root, "config-host.h")
             contents, success = FioJobTest.get_file(config_file)
             if not success:
-                print("Unable to open {0} to check requirements".format(config_file))
+                print(f"Unable to open {config_file} to check requirements")
                 Requirements._zbd = True
             else:
                 Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
@@ -885,7 +888,7 @@ class Requirements():
             else:
                 Requirements._io_uring = "io_uring_setup" in contents
 
-            Requirements._root = (os.geteuid() == 0)
+            Requirements._root = os.geteuid() == 0
             if Requirements._zbd and Requirements._root:
                 try:
                     subprocess.run(["modprobe", "null_blk"],
@@ -904,17 +907,21 @@ class Requirements():
         Requirements._unittests = os.path.exists(unittest_path)
 
         Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
-
-        req_list = [Requirements.linux,
-                    Requirements.libaio,
-                    Requirements.io_uring,
-                    Requirements.zbd,
-                    Requirements.root,
-                    Requirements.zoned_nullb,
-                    Requirements.not_macos,
-                    Requirements.not_windows,
-                    Requirements.unittests,
-                    Requirements.cpucount4]
+        Requirements._nvmecdev = args.nvmecdev
+
+        req_list = [
+                Requirements.linux,
+                Requirements.libaio,
+                Requirements.io_uring,
+                Requirements.zbd,
+                Requirements.root,
+                Requirements.zoned_nullb,
+                Requirements.not_macos,
+                Requirements.not_windows,
+                Requirements.unittests,
+                Requirements.cpucount4,
+                Requirements.nvmecdev,
+                    ]
         for req in req_list:
             value, desc = req()
             logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
@@ -969,6 +976,11 @@ class Requirements():
         """Do we have at least 4 CPUs?"""
         return Requirements._cpucount4, "4+ CPUs required"
 
+    @classmethod
+    def nvmecdev(cls):
+        """Do we have an NVMe character device to test?"""
+        return Requirements._nvmecdev, "NVMe character device test target required"
+
 
 SUCCESS_DEFAULT = {
     'zero_return': True,
@@ -1367,6 +1379,14 @@ TEST_LIST = [
         'success':          SUCCESS_DEFAULT,
         'requirements':     [],
     },
+    {
+        'test_id':          1014,
+        'test_class':       FioExeTest,
+        'exe':              't/nvmept.py',
+        'parameters':       ['-f', '{fio_path}', '--dut', '{nvmecdev}'],
+        'success':          SUCCESS_DEFAULT,
+        'requirements':     [Requirements.linux, Requirements.nvmecdev],
+    },
 ]
 
 
@@ -1390,6 +1410,8 @@ def parse_args():
                         help='skip requirements checking')
     parser.add_argument('-p', '--pass-through', action='append',
                         help='pass-through an argument to an executable test')
+    parser.add_argument('--nvmecdev', action='store', default=None,
+                        help='NVMe character device for **DESTRUCTIVE** testing (e.g., /dev/ng0n1)')
     args = parser.parse_args()
 
     return args
@@ -1408,7 +1430,7 @@ def main():
     if args.pass_through:
         for arg in args.pass_through:
             if not ':' in arg:
-                print("Invalid --pass-through argument '%s'" % arg)
+                print(f"Invalid --pass-through argument '{arg}'")
                 print("Syntax for --pass-through is TESTNUMBER:ARGUMENT")
                 return
             split = arg.split(":", 1)
@@ -1419,7 +1441,7 @@ def main():
         fio_root = args.fio_root
     else:
         fio_root = str(Path(__file__).absolute().parent.parent)
-    print("fio root is %s" % fio_root)
+    print(f"fio root is {fio_root}")
 
     if args.fio:
         fio_path = args.fio
@@ -1429,17 +1451,17 @@ def main():
         else:
             fio_exe = "fio"
         fio_path = os.path.join(fio_root, fio_exe)
-    print("fio path is %s" % fio_path)
+    print(f"fio path is {fio_path}")
     if not shutil.which(fio_path):
         print("Warning: fio executable not found")
 
     artifact_root = args.artifact_root if args.artifact_root else \
-        "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
+        f"fio-test-{time.strftime('%Y%m%d-%H%M%S')}"
     os.mkdir(artifact_root)
-    print("Artifact directory is %s" % artifact_root)
+    print(f"Artifact directory is {artifact_root}")
 
     if not args.skip_req:
-        req = Requirements(fio_root)
+        req = Requirements(fio_root, args)
 
     passed = 0
     failed = 0
@@ -1449,7 +1471,7 @@ def main():
         if (args.skip and config['test_id'] in args.skip) or \
            (args.run_only and config['test_id'] not in args.run_only):
             skipped = skipped + 1
-            print("Test {0} SKIPPED (User request)".format(config['test_id']))
+            print(f"Test {config['test_id']} SKIPPED (User request)")
             continue
 
         if issubclass(config['test_class'], FioJobTest):
@@ -1477,7 +1499,8 @@ def main():
         elif issubclass(config['test_class'], FioExeTest):
             exe_path = os.path.join(fio_root, config['exe'])
             if config['parameters']:
-                parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
+                parameters = [p.format(fio_path=fio_path, nvmecdev=args.nvmecdev)
+                              for p in config['parameters']]
             else:
                 parameters = []
             if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
@@ -1489,7 +1512,7 @@ def main():
                                         config['success'])
             desc = config['exe']
         else:
-            print("Test {0} FAILED: unable to process test config".format(config['test_id']))
+            print(f"Test {config['test_id']} FAILED: unable to process test config")
             failed = failed + 1
             continue
 
@@ -1502,7 +1525,7 @@ def main():
                 if not reqs_met:
                     break
             if not reqs_met:
-                print("Test {0} SKIPPED ({1}) {2}".format(config['test_id'], reason, desc))
+                print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
                 skipped = skipped + 1
                 continue
 
@@ -1520,15 +1543,15 @@ def main():
             result = "PASSED"
             passed = passed + 1
         else:
-            result = "FAILED: {0}".format(test.failure_reason)
+            result = f"FAILED: {test.failure_reason}"
             failed = failed + 1
             contents, _ = FioJobTest.get_file(test.stderr_file)
             logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
             contents, _ = FioJobTest.get_file(test.stdout_file)
             logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
-        print("Test {0} {1} {2}".format(config['test_id'], result, desc))
+        print(f"Test {config['test_id']} {result} {desc}")
 
-    print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
+    print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
 
     sys.exit(failed)
 

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-05-31 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-05-31 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 954b86f71b0718943796192be1a89ffb0da5a97c:

  ci: upload tagged GitHub Actions Windows installers as releases (2023-05-24 09:58:11 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 4820d46cef75f806d8c95afaa77f86ded4e3603e:

  ci: disable tls for msys2 builds (2023-05-26 20:09:53 -0400)

----------------------------------------------------------------
Vincent Fu (1):
      ci: disable tls for msys2 builds

 ci/actions-build.sh | 3 +++
 1 file changed, 3 insertions(+)

---

Diff of recent changes:

diff --git a/ci/actions-build.sh b/ci/actions-build.sh
index 351b8d18..31d3446c 100755
--- a/ci/actions-build.sh
+++ b/ci/actions-build.sh
@@ -53,6 +53,9 @@ main() {
                 "x86_64")
                     ;;
             esac
+            if [ "${CI_TARGET_BUILD}" = "windows-msys2-64" ]; then
+                configure_flags+=("--disable-tls")
+            fi
 	    ;;
     esac
     configure_flags+=(--extra-cflags="${extra_cflags}")

^ permalink raw reply related	[flat|nested] 1350+ messages in thread

* Recent changes (master)
@ 2023-05-25 12:00 Jens Axboe
  0 siblings, 0 replies; 1350+ messages in thread
From: Jens Axboe @ 2023-05-25 12:00 UTC (permalink / raw)
  To: fio

The following changes since commit 5a649e2dddc4d8ad163b0cf57f7cea00a2e94a33:

  Fio 3.35 (2023-05-23 12:33:03 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 954b86f71b0718943796192be1a89ffb0da5a97c:

  ci: upload tagged GitHub Actions Windows installers as releases (2023-05-24 09:58:11 -0400)

----------------------------------------------------------------
Vincent Fu (2):
      ci: stop using AppVeyor for Windows builds
      ci: upload tagged GitHub Actions Windows installers as releases

 .appveyor.yml            | 68 ------------------------------------------------
 .github/workflows/ci.yml |  7 ++++-
 README.rst               | 11 ++++----
 ci/appveyor-install.sh   | 43 ------------------------------
 4 files changed, 12 insertions(+), 117 deletions(-)
 delete mode 100644 .appveyor.yml
 delete mode 100755 ci/appveyor-install.sh

---

Diff of recent changes:

diff --git a/.appveyor.yml b/.appveyor.yml
deleted file mode 100644
index a63cf24f..00000000
--- a/.appveyor.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-clone_depth: 1 # NB: this stops FIO-VERSION-GEN making tag based versions
-
-image:
-  - Visual Studio 2019
-
-environment:
-  CYG_MIRROR: http://cygwin.mirror.constant.com
-  matrix:
-# --disable-tls for the msys2 build to work around
-# breakage with clang/lld 16.0.0-1
-    - ARCHITECTURE: x64
-      CC: clang
-      CONFIGURE_OPTIONS: --enable-pdb --disable-tls
-      DISTRO: msys2
-# Skip 32 bit clang build
-#    - ARCHITECTURE: x86
-#      CC: clang
-#      CONFIGURE_OPTIONS: --enable-pdb
-#      DISTRO: msys2
-    - ARCHITECTURE: x64
-      CONFIGURE_OPTIONS:
-      DISTRO: cygwin
-    - ARCHITECTURE: x86
-      CONFIGURE_OPTIONS: --build-32bit-win
-      DISTRO: cygwin
-
-install:
-  - if %DISTRO%==cygwin (
-      SET "PATH=C:\cygwin64\bin;C:\cygwin64;%PATH%"
-    )
-  - if %DISTRO%==msys2 if %ARCHITECTURE%==x86 (
-      SET "PATH=C:\msys64\mingw32\bin;C:\msys64\usr\bin;%PATH%"
-    )
-  - if %DISTRO%==msys2 if %ARCHITECTURE%==x64 (
-      SET "PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH%"
-    )
-  - SET PATH=C:\Python38-x64;%PATH% # NB: Changed env variables persist to later sections
-  - SET PYTHONUNBUFFERED=TRUE
-  - bash.exe ci\appveyor-install.sh
-
-build_script:
-  - bash.exe configure --extra-cflags=-Werror --disable-native %CONFIGURE_OPTIONS%
-  - make.exe -j2
-
-after_build:
-  - file.exe fio.exe
-  - make.exe test
-  - 'cd os\windows && dobuild.cmd %ARCHITECTURE% && cd ..'
-  - ls.exe ./os/windows/*.msi
-  - ps: Get-ChildItem .\os\windows\*.msi | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name -DeploymentName fio.msi }
-
-test_script:
-  - python.exe t/run-fio-tests.py --artifact-root test-artifacts --debug
-
-deploy:
-  - provider: GitHub
-    description: fio Windows installer
-    auth_token:                      # encrypted token from GitHub
-      secure: Tjj+xRQEV25P6dQgboUblTCKx/LtUOUav2bvzSCtwMhHMAxrrn2adod6nlTf0ItV
-    artifact: fio.msi                # upload installer to release assets
-    draft: false
-    prerelease: false
-    on:
-      APPVEYOR_REPO_TAG: true        # deploy on tag push only
-      DISTRO: cygwin
-
-on_finish:
-  - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -d test-artifacts ] && 7z a -t7z test-artifacts.7z test-artifacts -xr!foo.0.0 -xr!latency.?.0 -xr!fio_jsonplus_clat2csv.test && appveyor PushArtifact test-artifacts.7z'
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dd2997f0..69fedf77 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -108,12 +108,17 @@ jobs:
         dobuild.cmd ${{ matrix.installer_arch }}
         cd ..\..
 
-    - name: Upload installer (Windows)
+    - name: Upload installer as artifact (Windows)
       if: ${{ contains( matrix.build, 'windows' ) }}
       uses: actions/upload-artifact@v3
       with:
         name: ${{ matrix.build }}-installer
         path: os\windows\*.msi
+    - name: Upload installer as release for tagged builds (Windows)
+      uses: softprops/action-gh-release@v1
+      if: ${{ startsWith(github.ref, 'refs/tags/') && startsWith(matrix.build, 'windows-cygwin') }}
+      with:
+        files: os/windows/*.msi
     - name: Remove dependency files to resolve Makefile Cygwin sed issue (Windows)
       if: ${{ startsWith(matrix.build, 'windows-cygwin') }}
       run: rm *.d */*.d */*/*.d
diff --git a/README.rst b/README.rst
index 8f6208e3..dd521daf 100644
-