Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit ca4eefc1b55d7f9fc03bf113d63e3d0b2d7b38ae:

  Merge branch 'evelu-fixes2' of https://github.com/ErwanAliasr1/fio (2021-10-01 13:55:52 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 24a24c12a04c45174c2d68ffb7fcb3f367e40dee:

  t/io_uring: clean up aio wait loop (2021-10-04 17:04:04 -0600)

----------------------------------------------------------------
Jens Axboe (6):
      t/io_uring: remove extra add_stat() call
      t/io_uring: add support for legacy AIO
      t/io_uring: don't print partial IOPS etc output if exit signal was received
      t/io_uring: don't track IO latencies the first second of runtime
      t/io_uring: check for valid clock_index and finish state for stats
      t/io_uring: clean up aio wait loop

 t/io_uring.c | 277 +++++++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 253 insertions(+), 24 deletions(-)

---

Diff of recent changes:

diff --git a/t/io_uring.c b/t/io_uring.c
index 2ec4caeb..f27a12c7 100644
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -7,6 +7,10 @@
 #include <inttypes.h>
 #include <math.h>
 
+#ifdef CONFIG_LIBAIO
+#include <libaio.h>
+#endif
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/ioctl.h>
@@ -86,6 +90,10 @@ struct submitter {
 	int clock_index;
 	unsigned long *plat;
 
+#ifdef CONFIG_LIBAIO
+	io_context_t aio_ctx;
+#endif
+
 	struct file files[MAX_FDS];
 	unsigned nr_files;
 	unsigned cur_file;
@@ -94,6 +102,7 @@ struct submitter {
 
 static struct submitter *submitter;
 static volatile int finish;
+static int stats_running;
 
 static int depth = DEPTH;
 static int batch_submit = BATCH_SUBMIT;
@@ -108,6 +117,8 @@ static int sq_thread_cpu = -1;	/* pin above thread to this CPU */
 static int do_nop = 0;		/* no-op SQ ring commands */
 static int nthreads = 1;
 static int stats = 0;		/* generate IO stats */
+static int aio = 0;		/* use libaio */
+
 static unsigned long tsc_rate;
 
 #define TSC_RATE_FILE	"tsc-rate"
@@ -298,10 +309,12 @@ static void add_stat(struct submitter *s, int clock_index, int nr)
 	unsigned long cycles;
 	unsigned int pidx;
 
-	cycles = get_cpu_clock();
-	cycles -= s->clock_batch[clock_index];
-	pidx = plat_val_to_idx(cycles);
-	s->plat[pidx] += nr;
+	if (!s->finish && clock_index) {
+		cycles = get_cpu_clock();
+		cycles -= s->clock_batch[clock_index];
+		pidx = plat_val_to_idx(cycles);
+		s->plat[pidx] += nr;
+	}
 #endif
 }
 
@@ -432,11 +445,11 @@ static void init_io(struct submitter *s, unsigned index)
 	sqe->ioprio = 0;
 	sqe->off = offset;
 	sqe->user_data = (unsigned long) f->fileno;
-	if (stats)
+	if (stats && stats_running)
 		sqe->user_data |= ((unsigned long)s->clock_index << 32);
 }
 
-static int prep_more_ios(struct submitter *s, int max_ios)
+static int prep_more_ios_uring(struct submitter *s, int max_ios)
 {
 	struct io_sq_ring *ring = &s->sq_ring;
 	unsigned index, tail, next_tail, prepped = 0;
@@ -481,7 +494,7 @@ static int get_file_size(struct file *f)
 	return -1;
 }
 
-static int reap_events(struct submitter *s)
+static int reap_events_uring(struct submitter *s)
 {
 	struct io_cq_ring *ring = &s->cq_ring;
 	struct io_uring_cqe *cqe;
@@ -517,9 +530,8 @@ static int reap_events(struct submitter *s)
 					stat_nr = 0;
 				}
 				last_idx = clock_index;
-			}
-			stat_nr++;
-			add_stat(s, clock_index, 1);
+			} else if (clock_index)
+				stat_nr++;
 		}
 		reaped++;
 		head++;
@@ -535,11 +547,9 @@ static int reap_events(struct submitter *s)
 	return reaped;
 }
 
-static void *submitter_fn(void *data)
+static int submitter_init(struct submitter *s)
 {
-	struct submitter *s = data;
-	struct io_sq_ring *ring = &s->sq_ring;
-	int i, ret, prepped, nr_batch;
+	int i, nr_batch;
 
 	s->tid = gettid();
 	printf("submitter=%d\n", s->tid);
@@ -552,7 +562,7 @@ static void *submitter_fn(void *data)
 	if (stats) {
 		nr_batch = roundup_pow2(depth / batch_submit);
 		s->clock_batch = calloc(nr_batch, sizeof(unsigned long));
-		s->clock_index = 0;
+		s->clock_index = 1;
 
 		s->plat = calloc(PLAT_NR, sizeof(unsigned long));
 	} else {
@@ -561,6 +571,170 @@ static void *submitter_fn(void *data)
 		nr_batch = 0;
 	}
 
+	return nr_batch;
+}
+
+#ifdef CONFIG_LIBAIO
+static int prep_more_ios_aio(struct submitter *s, int max_ios, struct iocb *iocbs)
+{
+	unsigned long offset, data;
+	struct file *f;
+	unsigned index;
+	long r;
+
+	index = 0;
+	while (index < max_ios) {
+		struct iocb *iocb = &iocbs[index];
+
+		if (s->nr_files == 1) {
+			f = &s->files[0];
+		} else {
+			f = &s->files[s->cur_file];
+			if (f->pending_ios >= file_depth(s)) {
+				s->cur_file++;
+				if (s->cur_file == s->nr_files)
+					s->cur_file = 0;
+				f = &s->files[s->cur_file];
+			}
+		}
+		f->pending_ios++;
+
+		r = lrand48();
+		offset = (r % (f->max_blocks - 1)) * bs;
+		io_prep_pread(iocb, f->real_fd, s->iovecs[index].iov_base,
+				s->iovecs[index].iov_len, offset);
+
+		data = f->fileno;
+		if (stats && stats_running)
+			data |= ((unsigned long) s->clock_index << 32);
+		iocb->data = (void *) (uintptr_t) data;
+		index++;
+	}
+	return index;
+}
+
+static int reap_events_aio(struct submitter *s, struct io_event *events, int evs)
+{
+	int last_idx = -1, stat_nr = 0;
+	int reaped = 0;
+
+	while (evs) {
+		unsigned long data = (uintptr_t) events[reaped].data;
+		struct file *f = &s->files[data & 0xffffffff];
+
+		f->pending_ios--;
+		if (events[reaped].res != bs) {
+			printf("io: unexpected ret=%ld\n", events[reaped].res);
+			return -1;
+		}
+		if (stats) {
+			int clock_index = data >> 32;
+
+			if (last_idx != clock_index) {
+				if (last_idx != -1) {
+					add_stat(s, last_idx, stat_nr);
+					stat_nr = 0;
+				}
+				last_idx = clock_index;
+			} else if (clock_index)
+				stat_nr++;
+		}
+		reaped++;
+		evs--;
+	}
+
+	if (stat_nr)
+		add_stat(s, last_idx, stat_nr);
+
+	s->inflight -= reaped;
+	s->done += reaped;
+	return reaped;
+}
+
+static void *submitter_aio_fn(void *data)
+{
+	struct submitter *s = data;
+	int i, ret, prepped, nr_batch;
+	struct iocb **iocbsptr;
+	struct iocb *iocbs;
+	struct io_event *events;
+
+	nr_batch = submitter_init(s);
+
+	iocbsptr = calloc(depth, sizeof(struct iocb *));
+	iocbs = calloc(depth, sizeof(struct iocb));
+	events = calloc(depth, sizeof(struct io_event));
+
+	for (i = 0; i < depth; i++)
+		iocbsptr[i] = &iocbs[i];
+
+	prepped = 0;
+	do {
+		int to_wait, to_submit, to_prep;
+
+		if (!prepped && s->inflight < depth) {
+			to_prep = min(depth - s->inflight, batch_submit);
+			prepped = prep_more_ios_aio(s, to_prep, iocbs);
+#ifdef ARCH_HAVE_CPU_CLOCK
+			if (prepped && stats) {
+				s->clock_batch[s->clock_index] = get_cpu_clock();
+				s->clock_index = (s->clock_index + 1) & (nr_batch - 1);
+			}
+#endif
+		}
+		s->inflight += prepped;
+		to_submit = prepped;
+
+		if (to_submit && (s->inflight + to_submit <= depth))
+			to_wait = 0;
+		else
+			to_wait = min(s->inflight + to_submit, batch_complete);
+
+		ret = io_submit(s->aio_ctx, to_submit, iocbsptr);
+		s->calls++;
+		if (ret < 0) {
+			perror("io_submit");
+			break;
+		} else if (ret != to_submit) {
+			printf("submitted %d, wanted %d\n", ret, to_submit);
+			break;
+		}
+		prepped = 0;
+
+		while (to_wait) {
+			int r;
+
+			s->calls++;
+			r = io_getevents(s->aio_ctx, to_wait, to_wait, events, NULL);
+			if (r < 0) {
+				perror("io_getevents");
+				break;
+			} else if (r != to_wait) {
+				printf("r=%d, wait=%d\n", r, to_wait);
+				break;
+			}
+			r = reap_events_aio(s, events, r);
+			s->reaps += r;
+			to_wait -= r;
+		}
+	} while (!s->finish);
+
+	free(iocbsptr);
+	free(iocbs);
+	free(events);
+	finish = 1;
+	return NULL;
+}
+#endif
+
+static void *submitter_uring_fn(void *data)
+{
+	struct submitter *s = data;
+	struct io_sq_ring *ring = &s->sq_ring;
+	int ret, prepped, nr_batch;
+
+	nr_batch = submitter_init(s);
+
 	prepped = 0;
 	do {
 		int to_wait, to_submit, this_reap, to_prep;
@@ -568,7 +742,7 @@ static void *submitter_fn(void *data)
 
 		if (!prepped && s->inflight < depth) {
 			to_prep = min(depth - s->inflight, batch_submit);
-			prepped = prep_more_ios(s, to_prep);
+			prepped = prep_more_ios_uring(s, to_prep);
 #ifdef ARCH_HAVE_CPU_CLOCK
 			if (prepped && stats) {
 				s->clock_batch[s->clock_index] = get_cpu_clock();
@@ -613,7 +787,8 @@ submit:
 		this_reap = 0;
 		do {
 			int r;
-			r = reap_events(s);
+
+			r = reap_events_uring(s);
 			if (r == -1) {
 				s->finish = 1;
 				break;
@@ -693,6 +868,34 @@ static void arm_sig_int(void)
 #endif
 }
 
+static int setup_aio(struct submitter *s)
+{
+#ifdef CONFIG_LIBAIO
+	if (polled) {
+		fprintf(stderr, "aio does not support polled IO\n");
+		polled = 0;
+	}
+	if (sq_thread_poll) {
+		fprintf(stderr, "aio does not support SQPOLL IO\n");
+		sq_thread_poll = 0;
+	}
+	if (do_nop) {
+		fprintf(stderr, "aio does not support polled IO\n");
+		do_nop = 0;
+	}
+	if (fixedbufs || register_files) {
+		fprintf(stderr, "aio does not support registered files or buffers\n");
+		fixedbufs = register_files = 0;
+	}
+
+	return io_queue_init(depth, &s->aio_ctx);
+#else
+	fprintf(stderr, "Legacy AIO not available on this system/build\n");
+	errno = EINVAL;
+	return -1;
+#endif
+}
+
 static int setup_ring(struct submitter *s)
 {
 	struct io_sq_ring *sring = &s->sq_ring;
@@ -812,9 +1015,10 @@ static void usage(char *argv, int status)
 		" -O <bool> : Use O_DIRECT, default %d\n"
 		" -N <bool> : Perform just no-op requests, default %d\n"
 		" -t <bool> : Track IO latencies, default %d\n"
-		" -T <int>  : TSC rate in HZ\n",
+		" -T <int>  : TSC rate in HZ\n"
+		" -a <bool> : Use legacy aio, default %d\n",
 		argv, DEPTH, BATCH_SUBMIT, BATCH_COMPLETE, BS, polled,
-		fixedbufs, register_files, nthreads, !buffered, do_nop, stats);
+		fixedbufs, register_files, nthreads, !buffered, do_nop, stats, aio);
 	exit(status);
 }
 
@@ -874,8 +1078,11 @@ int main(int argc, char *argv[])
 	if (!do_nop && argc < 2)
 		usage(argv[0], 1);
 
-	while ((opt = getopt(argc, argv, "d:s:c:b:p:B:F:n:N:O:t:T:h?")) != -1) {
+	while ((opt = getopt(argc, argv, "d:s:c:b:p:B:F:n:N:O:t:T:a:h?")) != -1) {
 		switch (opt) {
+		case 'a':
+			aio = !!atoi(optarg);
+			break;
 		case 'd':
 			depth = atoi(optarg);
 			break;
@@ -1033,19 +1240,32 @@ int main(int argc, char *argv[])
 	for (j = 0; j < nthreads; j++) {
 		s = get_submitter(j);
 
-		err = setup_ring(s);
+		if (!aio)
+			err = setup_ring(s);
+		else
+			err = setup_aio(s);
 		if (err) {
 			printf("ring setup failed: %s, %d\n", strerror(errno), err);
 			return 1;
 		}
 	}
 	s = get_submitter(0);
-	printf("polled=%d, fixedbufs=%d, register_files=%d, buffered=%d", polled, fixedbufs, register_files, buffered);
-	printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", depth, *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);
+	printf("polled=%d, fixedbufs=%d, register_files=%d, buffered=%d, QD=%d\n", polled, fixedbufs, register_files, buffered, depth);
+	if (!aio)
+		printf("Engine=io_uring, sq_ring=%d, cq_ring=%d\n", *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);
+#ifdef CONFIG_LIBAIO
+	else
+		printf("Engine=aio, ctx=%p\n", &s->aio_ctx);
+#endif
 
 	for (j = 0; j < nthreads; j++) {
 		s = get_submitter(j);
-		pthread_create(&s->thread, NULL, submitter_fn, s);
+		if (!aio)
+			pthread_create(&s->thread, NULL, submitter_uring_fn, s);
+#ifdef CONFIG_LIBAIO
+		else
+			pthread_create(&s->thread, NULL, submitter_aio_fn, s);
+#endif
 	}
 
 	fdepths = malloc(8 * s->nr_files * nthreads);
@@ -1058,6 +1278,15 @@ int main(int argc, char *argv[])
 		unsigned long iops, bw;
 
 		sleep(1);
+
+		/* don't print partial run, if interrupted by signal */
+		if (finish)
+			break;
+
+		/* one second in to the run, enable stats */
+		if (stats)
+			stats_running = 1;
+
 		for (j = 0; j < nthreads; j++) {
 			this_done += s->done;
 			this_call += s->calls;