Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit ea693b6e4501a1385bf62a01f6fb1f3609d31a4a:

  Revert "Windows: update dobuild.cmd to run the configure/make" (2020-11-05 15:33:00 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to e82ec77644f4fb7eccb3441485762c1c1c574b2f:

  t/latency_percentiles.py: correct terse parse (2020-11-09 12:13:25 -0700)

----------------------------------------------------------------
Eric Sandeen (3):
      fix dynamic engine build
      fix dynamic engine loading for libaio engine etc
      list all available dynamic ioengines with --enghelp

Jens Axboe (6):
      FIO_EXT_ENG_DIR should be default path
      Remove the "libaio over io_uring" mode
      Makefile: ensure that external libs are linked properly with dynamic engine
      configure: remove libaio-uring remnant
      Make sure we do libaio engine compatability names
      t/latency_percentiles.py: correct terse parse

 Makefile                 | 46 +++++++++++++++++++---------------------------
 configure                | 19 +------------------
 ioengines.c              | 42 +++++++++++++++++++++++++++++++++++++++---
 os/os-linux.h            |  2 +-
 t/latency_percentiles.py |  2 +-
 5 files changed, 61 insertions(+), 50 deletions(-)

---

Diff of recent changes:

diff --git a/Makefile b/Makefile
index 8c8a8fce..0d3c877e 100644
--- a/Makefile
+++ b/Makefile
@@ -66,10 +66,10 @@ ifdef CONFIG_LIBHDFS
 endif
 
 ifdef CONFIG_LIBISCSI
-  iscsi_SRCS = engines/libiscsi.c
-  iscsi_LIBS = $(LIBISCSI_LIBS)
-  iscsi_CFLAGS = $(LIBISCSI_CFLAGS)
-  ENGINES += iscsi
+  libiscsi_SRCS = engines/libiscsi.c
+  libiscsi_LIBS = $(LIBISCSI_LIBS)
+  libiscsi_CFLAGS = $(LIBISCSI_CFLAGS)
+  ENGINES += libiscsi
 endif
 
 ifdef CONFIG_LIBNBD
@@ -85,14 +85,9 @@ else ifdef CONFIG_32BIT
   CPPFLAGS += -DBITS_PER_LONG=32
 endif
 ifdef CONFIG_LIBAIO
-  aio_SRCS = engines/libaio.c
-  aio_LIBS = -laio
-  ifdef CONFIG_LIBAIO_URING
-    aio_LIBS = -luring
-  else
-    aio_LIBS = -laio
-  endif
-  ENGINES += aio
+  libaio_SRCS = engines/libaio.c
+  libaio_LIBS = -laio
+  ENGINES += libaio
 endif
 ifdef CONFIG_RDMA
   rdma_SRCS = engines/rdma.c
@@ -179,17 +174,17 @@ ifdef CONFIG_LINUX_DEVDAX
   ENGINES += dev-dax
 endif
 ifdef CONFIG_LIBPMEM
-  pmem_SRCS = engines/libpmem.c
-  pmem_LIBS = -lpmem
-  ENGINES += pmem
+  libpmem_SRCS = engines/libpmem.c
+  libpmem_LIBS = -lpmem
+  ENGINES += libpmem
 endif
 ifdef CONFIG_IME
   SOURCE += engines/ime.c
 endif
 ifdef CONFIG_LIBZBC
-  zbc_SRCS = engines/libzbc.c
-  zbc_LIBS = -lzbc
-  ENGINES += zbc
+  libzbc_SRCS = engines/libzbc.c
+  libzbc_LIBS = -lzbc
+  ENGINES += libzbc
 endif
 
 ifeq ($(CONFIG_TARGET_OS), Linux)
@@ -255,8 +250,10 @@ ifdef CONFIG_DYNAMIC_ENGINES
  DYNAMIC_ENGS := $(ENGINES)
 define engine_template =
 $(1)_OBJS := $$($(1)_SRCS:.c=.o)
-$$($(1)_OBJS): FIO_CFLAGS += -fPIC $$($(1)_CFLAGS)
-ENGS_OBJS += engines/lib$(1).so
+$$($(1)_OBJS): CFLAGS := -fPIC $$($(1)_CFLAGS) $(CFLAGS)
+engines/fio-$(1).so: $$($(1)_OBJS)
+	$$(QUIET_LINK)$(CC) -shared -rdynamic -fPIC -Wl,-soname,fio-$(1).so.1 -o $$@ $$< $$($(1)_LIBS)
+ENGS_OBJS += engines/fio-$(1).so
 endef
 else # !CONFIG_DYNAMIC_ENGINES
 define engine_template =
@@ -266,6 +263,8 @@ CFLAGS += $$($(1)_CFLAGS)
 endef
 endif
 
+override CFLAGS := -DFIO_VERSION='"$(FIO_VERSION)"' $(FIO_CFLAGS) $(CFLAGS)
+
 $(foreach eng,$(ENGINES),$(eval $(call engine_template,$(eng))))
 
 OBJS := $(SOURCE:.c=.o)
@@ -438,8 +437,6 @@ FIO-VERSION-FILE: FORCE
 	@$(SHELL) $(SRCDIR)/FIO-VERSION-GEN
 -include FIO-VERSION-FILE
 
-override CFLAGS := -DFIO_VERSION='"$(FIO_VERSION)"' $(FIO_CFLAGS) $(CFLAGS)
-
 %.o : %.c
 	@mkdir -p $(dir $@)
 	$(QUIET_CC)$(CC) -o $@ $(CFLAGS) $(CPPFLAGS) -c $<
@@ -567,11 +564,6 @@ unittests/unittest: $(UT_OBJS) $(UT_TARGET_OBJS)
 	$(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(UT_OBJS) $(UT_TARGET_OBJS) -lcunit $(LIBS)
 endif
 
-ifdef CONFIG_DYNAMIC_ENGINES
-engines/lib$(1).so: $$($(1)_OBJS)
-	$$(QUIET_LINK)$(CC) -shared -rdynamic -fPIC -Wl,-soname,fio-$(1).so.1 $$($(1)_LIBS) -o $$@ $$<
-endif
-
 clean: FORCE
 	@rm -f .depend $(FIO_OBJS) $(GFIO_OBJS) $(OBJS) $(T_OBJS) $(UT_OBJS) $(PROGS) $(T_PROGS) $(T_TEST_PROGS) core.* core gfio unittests/unittest FIO-VERSION-FILE *.[do] lib/*.d oslib/*.[do] crc/*.d engines/*.[do] engines/*.so profiles/*.[do] t/*.[do] unittests/*.[do] unittests/*/*.[do] config-host.mak config-host.h y.tab.[ch] lex.yy.c exp/*.[do] lexer.h
 	@rm -f t/fio-btrace2fio t/io_uring t/read-to-pipe-async
diff --git a/configure b/configure
index 39a9248d..d2ca8934 100755
--- a/configure
+++ b/configure
@@ -168,7 +168,6 @@ disable_native="no"
 march_set="no"
 libiscsi="no"
 libnbd="no"
-libaio_uring="no"
 libzbc=""
 dynamic_engines="no"
 prefix=/usr/local
@@ -237,8 +236,6 @@ for opt do
   ;;
   --disable-tcmalloc) disable_tcmalloc="yes"
   ;;
-  --enable-libaio-uring) libaio_uring="yes"
-  ;;
   --dynamic-libengines) dynamic_engines="yes"
   ;;
   --help)
@@ -281,7 +278,6 @@ if test "$show_help" = "yes" ; then
   echo "--enable-libnbd         Enable libnbd (NBD engine) support"
   echo "--disable-libzbc        Disable libzbc even if found"
   echo "--disable-tcmalloc	Disable tcmalloc support"
-  echo "--enable-libaio-uring   Enable libaio emulated over io_uring"
   echo "--dynamic-libengines	Lib-based ioengines as dynamic libraries"
   exit $exit_val
 fi
@@ -653,22 +649,13 @@ int main(void)
   return 0;
 }
 EOF
-  if test "$libaio_uring" = "yes"; then
-    if compile_prog "" "-luring" "libaio io_uring" ; then
-      libaio=yes
-      LIBS="-luring $LIBS"
-    else
-      feature_not_found "libaio io_uring" ""
-    fi
-  elif compile_prog "" "-laio" "libaio" ; then
+  if compile_prog "" "-laio" "libaio" ; then
     libaio=yes
-    libaio_uring=no
   else
     if test "$libaio" = "yes" ; then
       feature_not_found "linux AIO" "libaio-dev or libaio-devel"
     fi
     libaio=no
-    libaio_uring=no
   fi
 
   cat > $TMPC <<EOF
@@ -689,7 +676,6 @@ EOF
 fi
 print_config "Linux AIO support" "$libaio"
 print_config "Linux AIO support rw flags" "$libaio_rw_flags"
-print_config "Linux AIO over io_uring" "$libaio_uring"
 
 ##########################################
 # posix aio probe
@@ -2722,9 +2708,6 @@ if test "$libaio" = "yes" ; then
   if test "$libaio_rw_flags" = "yes" ; then
     output_sym "CONFIG_LIBAIO_RW_FLAGS"
   fi
-  if test "$libaio_uring" = "yes" ; then
-    output_sym "CONFIG_LIBAIO_URING"
-  fi
 fi
 if test "$posix_aio" = "yes" ; then
   output_sym "CONFIG_POSIXAIO"
diff --git a/ioengines.c b/ioengines.c
index 3e43ef2f..5ac512ae 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -15,6 +15,8 @@
 #include <dlfcn.h>
 #include <fcntl.h>
 #include <assert.h>
+#include <sys/types.h>
+#include <dirent.h>
 
 #include "fio.h"
 #include "diskutil.h"
@@ -91,7 +93,7 @@ static void *dlopen_external(struct thread_data *td, const char *engine)
 	char engine_path[PATH_MAX];
 	void *dlhandle;
 
-	sprintf(engine_path, "%s/lib%s.so", FIO_EXT_ENG_DIR, engine);
+	sprintf(engine_path, "%s/fio-%s.so", FIO_EXT_ENG_DIR, engine);
 
 	dlhandle = dlopen(engine_path, RTLD_LAZY);
 	if (!dlhandle)
@@ -110,6 +112,10 @@ static struct ioengine_ops *dlopen_ioengine(struct thread_data *td,
 	struct ioengine_ops *ops;
 	void *dlhandle;
 
+	if (!strncmp(engine_lib, "linuxaio", 8) ||
+	    !strncmp(engine_lib, "aio", 3))
+		engine_lib = "libaio";
+
 	dprint(FD_IO, "dload engine %s\n", engine_lib);
 
 	dlerror();
@@ -158,7 +164,7 @@ static struct ioengine_ops *__load_ioengine(const char *engine)
 	/*
 	 * linux libaio has alias names, so convert to what we want
 	 */
-	if (!strncmp(engine, "linuxaio", 8)) {
+	if (!strncmp(engine, "linuxaio", 8) || !strncmp(engine, "aio", 3)) {
 		dprint(FD_IO, "converting ioengine name: %s -> libaio\n",
 		       engine);
 		engine = "libaio";
@@ -630,6 +636,34 @@ int td_io_get_file_size(struct thread_data *td, struct fio_file *f)
 	return td->io_ops->get_file_size(td, f);
 }
 
+#ifdef CONFIG_DYNAMIC_ENGINES
+/* Load all dynamic engines in FIO_EXT_ENG_DIR for enghelp command */
+static void
+fio_load_dynamic_engines(struct thread_data *td)
+{
+	DIR *dirhandle = NULL;
+	struct dirent *dirent = NULL;
+	char engine_path[PATH_MAX];
+
+	dirhandle = opendir(FIO_EXT_ENG_DIR);
+	if (!dirhandle)
+		return;
+
+	while ((dirent = readdir(dirhandle)) != NULL) {
+		if (!strcmp(dirent->d_name, ".") ||
+		    !strcmp(dirent->d_name, ".."))
+			continue;
+
+		sprintf(engine_path, "%s/%s", FIO_EXT_ENG_DIR, dirent->d_name);
+		dlopen_ioengine(td, engine_path);
+	}
+
+	closedir(dirhandle);
+}
+#else
+#define fio_load_dynamic_engines(td) do { } while (0)
+#endif
+
 int fio_show_ioengine_help(const char *engine)
 {
 	struct flist_head *entry;
@@ -638,8 +672,11 @@ int fio_show_ioengine_help(const char *engine)
 	char *sep;
 	int ret = 1;
 
+	memset(&td, 0, sizeof(struct thread_data));
+
 	if (!engine || !*engine) {
 		log_info("Available IO engines:\n");
+		fio_load_dynamic_engines(&td);
 		flist_for_each(entry, &engine_list) {
 			io_ops = flist_entry(entry, struct ioengine_ops, list);
 			log_info("\t%s\n", io_ops->name);
@@ -652,7 +689,6 @@ int fio_show_ioengine_help(const char *engine)
 		sep++;
 	}
 
-	memset(&td, 0, sizeof(struct thread_data));
 	td.o.ioengine = (char *)engine;
 	io_ops = load_ioengine(&td);
 
diff --git a/os/os-linux.h b/os/os-linux.h
index 65d3b429..5562b0da 100644
--- a/os/os-linux.h
+++ b/os/os-linux.h
@@ -58,7 +58,7 @@
 
 #define OS_MAP_ANON		MAP_ANONYMOUS
 
-#define FIO_EXT_ENG_DIR	"/usr/lib/fio"
+#define FIO_EXT_ENG_DIR	"/usr/local/lib/fio"
 
 typedef cpu_set_t os_cpu_mask_t;
 
diff --git a/t/latency_percentiles.py b/t/latency_percentiles.py
index 6ce4579a..a9aee019 100755
--- a/t/latency_percentiles.py
+++ b/t/latency_percentiles.py
@@ -224,7 +224,7 @@ class FioLatTest():
         lines = file_data.splitlines()
         for i in range(8):
             file_data = lines[i]
-            if file_data.startswith('3;fio-'):
+            if file_data.startswith('3;;latency'):
                 self.terse_data = file_data.split(';')
                 return True