Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit 20f42c101f7876648705a4fb8a9e2a647dc936ce:

  t/run-fio-tests: restrict t0031 to Linux only (2024-03-21 08:36:14 -0400)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 2b03792ceb7ed00bd50db5b59486fab902295df8:

  Merge branch 'issue-1735' of https://github.com/yygcode/fio (2024-03-22 10:39:37 -0400)

----------------------------------------------------------------
Vincent Fu (4):
      engines/fileoperations: remove extra blank lines
      engines/fileoperations: use local var for ioengine data
      examples: fiograph plots for dir operation ioengines
      Merge branch 'issue-1735' of https://github.com/yygcode/fio

friendy-su (1):
      ioengines: implement dircreate, dirstat, dirdelete engines to fileoperations.c

yonggang.yyg (1):
      iolog: fix disk stats issue

 HOWTO.rst                       |  15 +++++
 engines/fileoperations.c        | 119 ++++++++++++++++++++++++++++++++++++++--
 examples/dircreate-ioengine.fio |  25 +++++++++
 examples/dircreate-ioengine.png | Bin 0 -> 42659 bytes
 examples/dirdelete-ioengine.fio |  18 ++++++
 examples/dirdelete-ioengine.png | Bin 0 -> 45530 bytes
 examples/dirstat-ioengine.fio   |  18 ++++++
 examples/dirstat-ioengine.png   | Bin 0 -> 33597 bytes
 fio.1                           |  15 +++++
 iolog.c                         |   2 +
 10 files changed, 206 insertions(+), 6 deletions(-)
 create mode 100644 examples/dircreate-ioengine.fio
 create mode 100644 examples/dircreate-ioengine.png
 create mode 100644 examples/dirdelete-ioengine.fio
 create mode 100644 examples/dirdelete-ioengine.png
 create mode 100644 examples/dirstat-ioengine.fio
 create mode 100644 examples/dirstat-ioengine.png

---

Diff of recent changes:

diff --git a/HOWTO.rst b/HOWTO.rst
index 4c8ac331..fb067fe5 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2192,6 +2192,21 @@ I/O engine
 			and 'nrfiles', so that the files will be created.
 			This engine is to measure file delete.
 
+		**dircreate**
+			Simply create the directories and do no I/O to them.  You still need to
+			set  `filesize` so that all the accounting still occurs, but no
+			actual I/O will be done other than creating the directories.
+
+		**dirstat**
+			Simply do stat() and do no I/O to the directories. You need to set 'filesize'
+			and 'nrfiles', so that directories will be created.
+			This engine is to measure directory lookup and meta data access.
+
+		**dirdelete**
+			Simply delete the directories by rmdir() and do no I/O to them. You need to set 'filesize'
+			and 'nrfiles', so that the directories will be created.
+			This engine is to measure directory delete.
+
 		**libpmem**
 			Read and write using mmap I/O to a file on a filesystem
 			mounted with DAX on a persistent memory device through the PMDK
diff --git a/engines/fileoperations.c b/engines/fileoperations.c
index 1db60da1..c52f0900 100644
--- a/engines/fileoperations.c
+++ b/engines/fileoperations.c
@@ -1,8 +1,8 @@
 /*
- * fileoperations engine
+ * file/directory operations engine
  *
- * IO engine that doesn't do any IO, just operates files and tracks the latency
- * of the file operation.
+ * IO engine that doesn't do any IO, just operates files/directories
+ * and tracks the latency of the operation.
  */
 #include <stdio.h>
 #include <stdlib.h>
@@ -15,9 +15,15 @@
 #include "../optgroup.h"
 #include "../oslib/statx.h"
 
+enum fio_engine {
+	UNKNOWN_OP_ENGINE = 0,
+	FILE_OP_ENGINE = 1,
+	DIR_OP_ENGINE = 2,
+};
 
 struct fc_data {
 	enum fio_ddir stat_ddir;
+	enum fio_engine op_engine;
 };
 
 struct filestat_options {
@@ -61,11 +67,30 @@ static struct fio_option options[] = {
 	},
 };
 
+static int setup_dirs(struct thread_data *td)
+{
+	int ret = 0;
+	int i;
+	struct fio_file *f;
+
+	for_each_file(td, f, i) {
+		dprint(FD_FILE, "setup directory %s\n", f->file_name);
+		ret = fio_mkdir(f->file_name, 0700);
+		if ((ret && errno != EEXIST)) {
+			log_err("create directory %s failed with %d\n",
+				f->file_name, errno);
+			break;
+		}
+		ret = 0;
+	}
+	return ret;
+}
 
 static int open_file(struct thread_data *td, struct fio_file *f)
 {
 	struct timespec start;
 	int do_lat = !td->o.disable_lat;
+	struct fc_data *fcd = td->io_ops_data;
 
 	dprint(FD_FILE, "fd open %s\n", f->file_name);
 
@@ -81,7 +106,14 @@ static int open_file(struct thread_data *td, struct fio_file *f)
 	if (do_lat)
 		fio_gettime(&start, NULL);
 
-	f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
+	if (fcd->op_engine == FILE_OP_ENGINE)
+		f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
+	else if (fcd->op_engine == DIR_OP_ENGINE)
+		f->fd = fio_mkdir(f->file_name, S_IFDIR);
+	else {
+		log_err("fio: unknown file/directory operation engine\n");
+		return 1;
+	}
 
 	if (f->fd == -1) {
 		char buf[FIO_VERROR_SIZE];
@@ -174,11 +206,11 @@ static int stat_file(struct thread_data *td, struct fio_file *f)
 	return 0;
 }
 
-
 static int delete_file(struct thread_data *td, struct fio_file *f)
 {
 	struct timespec start;
 	int do_lat = !td->o.disable_lat;
+	struct fc_data *fcd = td->io_ops_data;
 	int ret;
 
 	dprint(FD_FILE, "fd delete %s\n", f->file_name);
@@ -195,7 +227,14 @@ static int delete_file(struct thread_data *td, struct fio_file *f)
 	if (do_lat)
 		fio_gettime(&start, NULL);
 
-	ret = unlink(f->file_name);
+	if (fcd->op_engine == FILE_OP_ENGINE)
+		ret = unlink(f->file_name);
+	else if (fcd->op_engine == DIR_OP_ENGINE)
+		ret = rmdir(f->file_name);
+	else {
+		log_err("fio: unknown file/directory operation engine\n");
+		return 1;
+	}
 
 	if (ret == -1) {
 		char buf[FIO_VERROR_SIZE];
@@ -250,6 +289,17 @@ static int init(struct thread_data *td)
 	else if (td_write(td))
 		data->stat_ddir = DDIR_WRITE;
 
+	data->op_engine = UNKNOWN_OP_ENGINE;
+
+	if (!strncmp(td->o.ioengine, "file", 4)) {
+		data->op_engine = FILE_OP_ENGINE;
+		dprint(FD_FILE, "Operate engine type: file\n");
+	}
+	if (!strncmp(td->o.ioengine, "dir", 3)) {
+		data->op_engine = DIR_OP_ENGINE;
+		dprint(FD_FILE, "Operate engine type: directory\n");
+	}
+
 	td->io_ops_data = data;
 	return 0;
 }
@@ -261,6 +311,12 @@ static void cleanup(struct thread_data *td)
 	free(data);
 }
 
+static int remove_dir(struct thread_data *td, struct fio_file *f)
+{
+	dprint(FD_FILE, "remove directory %s\n", f->file_name);
+	return rmdir(f->file_name);
+}
+
 static struct ioengine_ops ioengine_filecreate = {
 	.name		= "filecreate",
 	.version	= FIO_IOOPS_VERSION,
@@ -302,12 +358,60 @@ static struct ioengine_ops ioengine_filedelete = {
 				FIO_NOSTATS | FIO_NOFILEHASH,
 };
 
+static struct ioengine_ops ioengine_dircreate = {
+	.name		= "dircreate",
+	.version	= FIO_IOOPS_VERSION,
+	.init		= init,
+	.cleanup	= cleanup,
+	.queue		= queue_io,
+	.get_file_size	= get_file_size,
+	.open_file	= open_file,
+	.close_file	= generic_close_file,
+	.unlink_file    = remove_dir,
+	.flags		= FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+				FIO_NOSTATS | FIO_NOFILEHASH,
+};
+
+static struct ioengine_ops ioengine_dirstat = {
+	.name		= "dirstat",
+	.version	= FIO_IOOPS_VERSION,
+	.setup		= setup_dirs,
+	.init		= init,
+	.cleanup	= cleanup,
+	.queue		= queue_io,
+	.invalidate	= invalidate_do_nothing,
+	.get_file_size	= generic_get_file_size,
+	.open_file	= stat_file,
+	.unlink_file	= remove_dir,
+	.flags		=  FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+				FIO_NOSTATS | FIO_NOFILEHASH,
+	.options	= options,
+	.option_struct_size = sizeof(struct filestat_options),
+};
+
+static struct ioengine_ops ioengine_dirdelete = {
+	.name		= "dirdelete",
+	.version	= FIO_IOOPS_VERSION,
+	.setup		= setup_dirs,
+	.init		= init,
+	.invalidate	= invalidate_do_nothing,
+	.cleanup	= cleanup,
+	.queue		= queue_io,
+	.get_file_size	= get_file_size,
+	.open_file	= delete_file,
+	.unlink_file	= remove_dir,
+	.flags		= FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+				FIO_NOSTATS | FIO_NOFILEHASH,
+};
 
 static void fio_init fio_fileoperations_register(void)
 {
 	register_ioengine(&ioengine_filecreate);
 	register_ioengine(&ioengine_filestat);
 	register_ioengine(&ioengine_filedelete);
+	register_ioengine(&ioengine_dircreate);
+	register_ioengine(&ioengine_dirstat);
+	register_ioengine(&ioengine_dirdelete);
 }
 
 static void fio_exit fio_fileoperations_unregister(void)
@@ -315,4 +419,7 @@ static void fio_exit fio_fileoperations_unregister(void)
 	unregister_ioengine(&ioengine_filecreate);
 	unregister_ioengine(&ioengine_filestat);
 	unregister_ioengine(&ioengine_filedelete);
+	unregister_ioengine(&ioengine_dircreate);
+	unregister_ioengine(&ioengine_dirstat);
+	unregister_ioengine(&ioengine_dirdelete);
 }
diff --git a/examples/dircreate-ioengine.fio b/examples/dircreate-ioengine.fio
new file mode 100644
index 00000000..c89d9e4d
--- /dev/null
+++ b/examples/dircreate-ioengine.fio
@@ -0,0 +1,25 @@
+# Example dircreate job
+#
+# create_on_open is needed so that the open happens during the run and not the
+# setup.
+#
+# openfiles needs to be set so that you do not exceed the maximum allowed open
+# files.
+#
+# filesize needs to be set to a non zero value so fio will actually run, but the
+# IO will not really be done and the write latency numbers will only reflect the
+# open times.
+[global]
+create_on_open=1
+nrfiles=30
+ioengine=dircreate
+fallocate=none
+filesize=4k
+openfiles=1
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]
diff --git a/examples/dircreate-ioengine.png b/examples/dircreate-ioengine.png
new file mode 100644
index 00000000..da1a8c40
Binary files /dev/null and b/examples/dircreate-ioengine.png differ
diff --git a/examples/dirdelete-ioengine.fio b/examples/dirdelete-ioengine.fio
new file mode 100644
index 00000000..4e5b1e2c
--- /dev/null
+++ b/examples/dirdelete-ioengine.fio
@@ -0,0 +1,18 @@
+# Example dirdelete job
+
+# 'filedelete' engine only do 'rmdir(dirname)'.
+# 'filesize' must be set, then directories will be created at setup stage.
+# 'unlink' is better set to 0, since the directory is deleted in measurement.
+# the options disabled completion latency output such as 'disable_clat' and 'gtod_reduce' must not set.
+[global]
+ioengine=dirdelete
+filesize=4k
+nrfiles=200
+unlink=0
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]
diff --git a/examples/dirdelete-ioengine.png b/examples/dirdelete-ioengine.png
new file mode 100644
index 00000000..af246195
Binary files /dev/null and b/examples/dirdelete-ioengine.png differ
diff --git a/examples/dirstat-ioengine.fio b/examples/dirstat-ioengine.fio
new file mode 100644
index 00000000..1322dd28
--- /dev/null
+++ b/examples/dirstat-ioengine.fio
@@ -0,0 +1,18 @@
+# Example dirstat job
+
+# 'dirstat' engine only do 'stat(dirname)', file will not be open().
+# 'filesize' must be set, then files will be created at setup stage.
+
+[global]
+ioengine=dirstat
+numjobs=10
+filesize=4k
+nrfiles=5
+thread
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]
diff --git a/examples/dirstat-ioengine.png b/examples/dirstat-ioengine.png
new file mode 100644
index 00000000..14b948ba
Binary files /dev/null and b/examples/dirstat-ioengine.png differ
diff --git a/fio.1 b/fio.1
index 09c6b621..63375c62 100644
--- a/fio.1
+++ b/fio.1
@@ -2004,6 +2004,21 @@ Simply delete files by unlink() and do no I/O to the file. You need to set 'file
 and 'nrfiles', so that files will be created.
 This engine is to measure file delete.
 .TP
+.B dircreate
+Simply create the directories and do no I/O to them.  You still need to set
+\fBfilesize\fR so that all the accounting still occurs, but no actual I/O will be
+done other than creating the directories.
+.TP
+.B dirstat
+Simply do stat() and do no I/O to the directory. You need to set 'filesize'
+and 'nrfiles', so that directories will be created.
+This engine is to measure directory lookup and meta data access.
+.TP
+.B dirdelete
+Simply delete directories by unlink() and do no I/O to the directory. You need to set 'filesize'
+and 'nrfiles', so that directories will be created.
+This engine is to measure directory delete.
+.TP
 .B libpmem
 Read and write using mmap I/O to a file on a filesystem
 mounted with DAX on a persistent memory device through the PMDK
diff --git a/iolog.c b/iolog.c
index 251e9d7f..96af4f33 100644
--- a/iolog.c
+++ b/iolog.c
@@ -814,6 +814,8 @@ bool init_iolog(struct thread_data *td)
 	if (!ret)
 		td_verror(td, EINVAL, "failed initializing iolog");
 
+	init_disk_util(td);
+
 	return ret;
 }