From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from bombadil.infradead.org ([198.137.202.133]:33464 "EHLO bombadil.infradead.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1729460AbeLMNR7 (ORCPT ); Thu, 13 Dec 2018 08:17:59 -0500 Received: from [216.160.245.99] (helo=kernel.dk) by bombadil.infradead.org with esmtpsa (Exim 4.90_1 #2 (Red Hat Linux)) id 1gXQsH-0005b8-L7 for fio@vger.kernel.org; Thu, 13 Dec 2018 13:17:57 +0000 Subject: Recent changes (master) From: Jens Axboe Message-Id: <20181213130002.168362C0051@kernel.dk> Date: Thu, 13 Dec 2018 06:00:02 -0700 (MST) Sender: fio-owner@vger.kernel.org List-Id: fio@vger.kernel.org To: fio@vger.kernel.org The following changes since commit 0527b23f9112b59dc63f3b40a7f40d45c48ec60d: t/aio-ring: updates/cleanups (2018-12-10 15:14:36 -0700) are available in the Git repository at: git://git.kernel.dk/fio.git master for you to fetch changes up to 702906e9e3e03e9836421d5e5b5eaae3cd99d398: engines/libaio: remove features deprecated from old interface (2018-12-12 22:02:16 -0700) ---------------------------------------------------------------- Jens Axboe (10): t/aio-ring: update to newer API t/aio-ring: update to new io_setup2(2) t/aio-ring: set nr_events after clear ioengine: remove ancient alias for libaio Add aioring engine t/aio-ring: update for new API aioring: hide it if archs don't define syscalls aioring: check for arch support AFTER including the headers aioring: remove qd > 1 restriction engines/libaio: remove features deprecated from old interface Makefile | 3 + arch/arch-x86_64.h | 4 + engines/aioring.c | 547 +++++++++++++++++++++++++++++++++++++++++++++++++++++ engines/libaio.c | 150 ++------------- ioengines.c | 2 +- options.c | 7 + t/aio-ring.c | 72 ++++--- 7 files changed, 621 insertions(+), 164 deletions(-) create mode 100644 engines/aioring.c --- Diff of recent changes: diff --git a/Makefile b/Makefile index 284621d3..f111ae6a 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,9 @@ endif ifdef CONFIG_LIBAIO SOURCE += engines/libaio.c endif +ifdef CONFIG_LIBAIO + SOURCE += engines/aioring.c +endif ifdef CONFIG_RDMA SOURCE += engines/rdma.c endif diff --git a/arch/arch-x86_64.h b/arch/arch-x86_64.h index ac670d08..d49bcd7f 100644 --- a/arch/arch-x86_64.h +++ b/arch/arch-x86_64.h @@ -4,6 +4,9 @@ #ifndef __NR_sys_io_setup2 #define __NR_sys_io_setup2 335 #endif +#ifndef __NR_sys_io_ring_enter +#define __NR_sys_io_ring_enter 336 +#endif static inline void do_cpuid(unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx) @@ -41,6 +44,7 @@ static inline unsigned long long get_cpu_clock(void) #define ARCH_HAVE_FFZ #define ARCH_HAVE_SSE4_2 #define ARCH_HAVE_CPU_CLOCK +#define ARCH_HAVE_AIORING #define RDRAND_LONG ".byte 0x48,0x0f,0xc7,0xf0" #define RDSEED_LONG ".byte 0x48,0x0f,0xc7,0xf8" diff --git a/engines/aioring.c b/engines/aioring.c new file mode 100644 index 00000000..1598cc12 --- /dev/null +++ b/engines/aioring.c @@ -0,0 +1,547 @@ +/* + * aioring engine + * + * IO engine using the new native Linux libaio ring interface + * + */ +#include +#include +#include +#include +#include +#include + +#include "../fio.h" +#include "../lib/pow2.h" +#include "../optgroup.h" +#include "../lib/memalign.h" + +#ifdef ARCH_HAVE_AIORING + +#ifndef IOCB_FLAG_HIPRI +#define IOCB_FLAG_HIPRI (1 << 2) +#endif + +/* + * io_setup2(2) flags + */ +#ifndef IOCTX_FLAG_IOPOLL +#define IOCTX_FLAG_IOPOLL (1 << 0) +#endif +#ifndef IOCTX_FLAG_SCQRING +#define IOCTX_FLAG_SCQRING (1 << 1) +#endif +#ifndef IOCTX_FLAG_FIXEDBUFS +#define IOCTX_FLAG_FIXEDBUFS (1 << 2) +#endif +#ifndef IOCTX_FLAG_SQTHREAD +#define IOCTX_FLAG_SQTHREAD (1 << 3) +#endif +#ifndef IOCTX_FLAG_SQWQ +#define IOCTX_FLAG_SQWQ (1 << 4) +#endif + +/* + * io_ring_enter(2) flags + */ +#ifndef IORING_FLAG_SUBMIT +#define IORING_FLAG_SUBMIT (1 << 0) +#endif +#ifndef IORING_FLAG_GETEVENTS +#define IORING_FLAG_GETEVENTS (1 << 1) +#endif + +typedef uint64_t u64; +typedef uint32_t u32; +typedef uint16_t u16; + +struct aio_sq_ring { + union { + struct { + u32 head; + u32 tail; + u32 nr_events; + u16 sq_thread_cpu; + u64 iocbs; + }; + u32 pad[16]; + }; + u32 array[0]; +}; + +struct aio_cq_ring { + union { + struct { + u32 head; + u32 tail; + u32 nr_events; + }; + struct io_event pad; + }; + struct io_event events[0]; +}; + +struct aioring_data { + io_context_t aio_ctx; + struct io_u **io_us; + struct io_u **io_u_index; + + struct aio_sq_ring *sq_ring; + struct iocb *iocbs; + + struct aio_cq_ring *cq_ring; + struct io_event *events; + + int queued; + int cq_ring_off; +}; + +struct aioring_options { + void *pad; + unsigned int hipri; + unsigned int fixedbufs; +}; + +static struct fio_option options[] = { + { + .name = "hipri", + .lname = "High Priority", + .type = FIO_OPT_STR_SET, + .off1 = offsetof(struct aioring_options, hipri), + .help = "Use polled IO completions", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_LIBAIO, + }, + { + .name = "fixedbufs", + .lname = "Fixed (pre-mapped) IO buffers", + .type = FIO_OPT_STR_SET, + .off1 = offsetof(struct aioring_options, fixedbufs), + .help = "Pre map IO buffers", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_LIBAIO, + }, + { + .name = NULL, + }, +}; + +static int fio_aioring_commit(struct thread_data *td); + +static int io_ring_enter(io_context_t ctx, unsigned int to_submit, + unsigned int min_complete, unsigned int flags) +{ +#ifdef __NR_sys_io_ring_enter + return syscall(__NR_sys_io_ring_enter, ctx, to_submit, min_complete, + flags); +#else + return -1; +#endif +} + +static int fio_aioring_prep(struct thread_data *td, struct io_u *io_u) +{ + struct aioring_data *ld = td->io_ops_data; + struct fio_file *f = io_u->file; + struct aioring_options *o = td->eo; + struct iocb *iocb; + + iocb = &ld->iocbs[io_u->index]; + + if (io_u->ddir == DDIR_READ) { + if (o->fixedbufs) { + iocb->aio_fildes = f->fd; + iocb->aio_lio_opcode = IO_CMD_PREAD; + iocb->u.c.offset = io_u->offset; + } else { + io_prep_pread(iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); + if (o->hipri) + iocb->u.c.flags |= IOCB_FLAG_HIPRI; + } + } else if (io_u->ddir == DDIR_WRITE) { + if (o->fixedbufs) { + iocb->aio_fildes = f->fd; + iocb->aio_lio_opcode = IO_CMD_PWRITE; + iocb->u.c.offset = io_u->offset; + } else { + io_prep_pwrite(iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); + if (o->hipri) + iocb->u.c.flags |= IOCB_FLAG_HIPRI; + } + } else if (ddir_sync(io_u->ddir)) + io_prep_fsync(iocb, f->fd); + + iocb->data = io_u; + return 0; +} + +static struct io_u *fio_aioring_event(struct thread_data *td, int event) +{ + struct aioring_data *ld = td->io_ops_data; + struct io_event *ev; + struct io_u *io_u; + int index; + + index = event + ld->cq_ring_off; + if (index >= ld->cq_ring->nr_events) + index -= ld->cq_ring->nr_events; + + ev = &ld->cq_ring->events[index]; + io_u = ev->data; + + if (ev->res != io_u->xfer_buflen) { + if (ev->res > io_u->xfer_buflen) + io_u->error = -ev->res; + else + io_u->resid = io_u->xfer_buflen - ev->res; + } else + io_u->error = 0; + + return io_u; +} + +static int fio_aioring_cqring_reap(struct thread_data *td, unsigned int events, + unsigned int max) +{ + struct aioring_data *ld = td->io_ops_data; + struct aio_cq_ring *ring = ld->cq_ring; + u32 head, reaped = 0; + + head = ring->head; + do { + read_barrier(); + if (head == ring->tail) + break; + reaped++; + head++; + if (head == ring->nr_events) + head = 0; + } while (reaped + events < max); + + ring->head = head; + write_barrier(); + return reaped; +} + +static int fio_aioring_getevents(struct thread_data *td, unsigned int min, + unsigned int max, const struct timespec *t) +{ + struct aioring_data *ld = td->io_ops_data; + unsigned actual_min = td->o.iodepth_batch_complete_min == 0 ? 0 : min; + struct aio_cq_ring *ring = ld->cq_ring; + int r, events = 0; + + ld->cq_ring_off = ring->head; + do { + r = fio_aioring_cqring_reap(td, events, max); + if (r) { + events += r; + continue; + } + + r = io_ring_enter(ld->aio_ctx, 0, actual_min, + IORING_FLAG_GETEVENTS); + if (r < 0) { + if (errno == EAGAIN) + continue; + perror("ring enter"); + break; + } + } while (events < min); + + return r < 0 ? r : events; +} + +static enum fio_q_status fio_aioring_queue(struct thread_data *td, + struct io_u *io_u) +{ + struct aioring_data *ld = td->io_ops_data; + struct aio_sq_ring *ring = ld->sq_ring; + unsigned tail, next_tail; + + fio_ro_check(td, io_u); + + if (ld->queued == td->o.iodepth) + return FIO_Q_BUSY; + + /* + * fsync is tricky, since it can fail and we need to do it + * serialized with other io. the reason is that linux doesn't + * support aio fsync yet. So return busy for the case where we + * have pending io, to let fio complete those first. + */ + if (ddir_sync(io_u->ddir)) { + if (ld->queued) + return FIO_Q_BUSY; + + do_io_u_sync(td, io_u); + return FIO_Q_COMPLETED; + } + + if (io_u->ddir == DDIR_TRIM) { + if (ld->queued) + return FIO_Q_BUSY; + + do_io_u_trim(td, io_u); + io_u_mark_submit(td, 1); + io_u_mark_complete(td, 1); + return FIO_Q_COMPLETED; + } + + tail = ring->tail; + next_tail = tail + 1; + if (next_tail == ring->nr_events) + next_tail = 0; + read_barrier(); + if (next_tail == ring->head) + return FIO_Q_BUSY; + + ring->array[tail] = io_u->index; + ring->tail = next_tail; + write_barrier(); + + ld->queued++; + return FIO_Q_QUEUED; +} + +static void fio_aioring_queued(struct thread_data *td, int start, int nr) +{ + struct aioring_data *ld = td->io_ops_data; + struct timespec now; + + if (!fio_fill_issue_time(td)) + return; + + fio_gettime(&now, NULL); + + while (nr--) { + int index = ld->sq_ring->array[start]; + struct io_u *io_u = io_u = ld->io_u_index[index]; + + memcpy(&io_u->issue_time, &now, sizeof(now)); + io_u_queued(td, io_u); + + start++; + if (start == ld->sq_ring->nr_events) + start = 0; + } +} + +static int fio_aioring_commit(struct thread_data *td) +{ + struct aioring_data *ld = td->io_ops_data; + int ret; + + if (!ld->queued) + return 0; + + do { + int start = ld->sq_ring->head; + long nr = ld->queued; + + ret = io_ring_enter(ld->aio_ctx, nr, 0, IORING_FLAG_SUBMIT | + IORING_FLAG_GETEVENTS); + if (ret == -1) + perror("io_ring_enter"); + if (ret > 0) { + fio_aioring_queued(td, start, ret); + io_u_mark_submit(td, ret); + + ld->queued -= ret; + ret = 0; + } else if (ret == -EINTR || !ret) { + if (!ret) + io_u_mark_submit(td, ret); + continue; + } else if (ret == -EAGAIN) { + /* + * If we get EAGAIN, we should break out without + * error and let the upper layer reap some + * events for us. If we have no queued IO, we + * must loop here. If we loop for more than 30s, + * just error out, something must be buggy in the + * IO path. + */ + if (ld->queued) { + ret = 0; + break; + } + usleep(1); + continue; + } else if (ret == -ENOMEM) { + /* + * If we get -ENOMEM, reap events if we can. If + * we cannot, treat it as a fatal event since there's + * nothing we can do about it. + */ + if (ld->queued) + ret = 0; + break; + } else + break; + } while (ld->queued); + + return ret; +} + +static size_t aioring_cq_size(struct thread_data *td) +{ + return sizeof(struct aio_cq_ring) + 2 * td->o.iodepth * sizeof(struct io_event); +} + +static size_t aioring_sq_iocb(struct thread_data *td) +{ + return sizeof(struct iocb) * td->o.iodepth; +} + +static size_t aioring_sq_size(struct thread_data *td) +{ + return sizeof(struct aio_sq_ring) + td->o.iodepth * sizeof(u32); +} + +static void fio_aioring_cleanup(struct thread_data *td) +{ + struct aioring_data *ld = td->io_ops_data; + + if (ld) { + /* + * Work-around to avoid huge RCU stalls at exit time. If we + * don't do this here, then it'll be torn down by exit_aio(). + * But for that case we can parallellize the freeing, thus + * speeding it up a lot. + */ + if (!(td->flags & TD_F_CHILD)) + io_destroy(ld->aio_ctx); + free(ld->io_u_index); + free(ld->io_us); + fio_memfree(ld->sq_ring, aioring_sq_size(td), false); + fio_memfree(ld->iocbs, aioring_sq_iocb(td), false); + fio_memfree(ld->cq_ring, aioring_cq_size(td), false); + free(ld); + } +} + +static int fio_aioring_queue_init(struct thread_data *td) +{ +#ifdef __NR_sys_io_setup2 + struct aioring_data *ld = td->io_ops_data; + struct aioring_options *o = td->eo; + int flags = IOCTX_FLAG_SCQRING; + int depth = td->o.iodepth; + + if (o->hipri) + flags |= IOCTX_FLAG_IOPOLL; + if (o->fixedbufs) { + struct rlimit rlim = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + setrlimit(RLIMIT_MEMLOCK, &rlim); + flags |= IOCTX_FLAG_FIXEDBUFS; + } + + return syscall(__NR_sys_io_setup2, depth, flags, + ld->sq_ring, ld->cq_ring, &ld->aio_ctx); +#else + return -1; +#endif +} + +static int fio_aioring_post_init(struct thread_data *td) +{ + struct aioring_data *ld = td->io_ops_data; + struct aioring_options *o = td->eo; + struct io_u *io_u; + struct iocb *iocb; + int err = 0; + + if (o->fixedbufs) { + int i; + + for (i = 0; i < td->o.iodepth; i++) { + io_u = ld->io_u_index[i]; + iocb = &ld->iocbs[i]; + iocb->u.c.buf = io_u->buf; + iocb->u.c.nbytes = td_max_bs(td); + + if (o->hipri) + iocb->u.c.flags |= IOCB_FLAG_HIPRI; + } + } + + err = fio_aioring_queue_init(td); + if (err) { + td_verror(td, -err, "io_queue_init"); + return 1; + } + + return 0; +} + +static int fio_aioring_init(struct thread_data *td) +{ + struct aioring_data *ld; + + /* ring needs an extra entry, add one to achieve QD set */ + td->o.iodepth++; + + ld = calloc(1, sizeof(*ld)); + + /* io_u index */ + ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *)); + ld->io_us = calloc(td->o.iodepth, sizeof(struct io_u *)); + + ld->iocbs = fio_memalign(page_size, aioring_sq_iocb(td), false); + memset(ld->iocbs, 0, aioring_sq_iocb(td)); + + ld->sq_ring = fio_memalign(page_size, aioring_sq_size(td), false); + memset(ld->sq_ring, 0, aioring_sq_size(td)); + ld->sq_ring->nr_events = td->o.iodepth; + ld->sq_ring->iocbs = (u64) (uintptr_t) ld->iocbs; + + ld->cq_ring = fio_memalign(page_size, aioring_cq_size(td), false); + memset(ld->cq_ring, 0, aioring_cq_size(td)); + ld->cq_ring->nr_events = td->o.iodepth * 2; + + td->io_ops_data = ld; + return 0; +} + +static int fio_aioring_io_u_init(struct thread_data *td, struct io_u *io_u) +{ + struct aioring_data *ld = td->io_ops_data; + + ld->io_u_index[io_u->index] = io_u; + return 0; +} + +static struct ioengine_ops ioengine = { + .name = "aio-ring", + .version = FIO_IOOPS_VERSION, + .init = fio_aioring_init, + .post_init = fio_aioring_post_init, + .io_u_init = fio_aioring_io_u_init, + .prep = fio_aioring_prep, + .queue = fio_aioring_queue, + .commit = fio_aioring_commit, + .getevents = fio_aioring_getevents, + .event = fio_aioring_event, + .cleanup = fio_aioring_cleanup, + .open_file = generic_open_file, + .close_file = generic_close_file, + .get_file_size = generic_get_file_size, + .options = options, + .option_struct_size = sizeof(struct aioring_options), +}; + +static void fio_init fio_aioring_register(void) +{ + register_ioengine(&ioengine); +} + +static void fio_exit fio_aioring_unregister(void) +{ + unregister_ioengine(&ioengine); +} +#endif diff --git a/engines/libaio.c b/engines/libaio.c index 03335094..8844ac8b 100644 --- a/engines/libaio.c +++ b/engines/libaio.c @@ -20,14 +20,8 @@ #define IOCB_FLAG_HIPRI (1 << 2) #endif -#ifndef IOCTX_FLAG_USERIOCB -#define IOCTX_FLAG_USERIOCB (1 << 0) -#endif #ifndef IOCTX_FLAG_IOPOLL -#define IOCTX_FLAG_IOPOLL (1 << 1) -#endif -#ifndef IOCTX_FLAG_FIXEDBUFS -#define IOCTX_FLAG_FIXEDBUFS (1 << 2) +#define IOCTX_FLAG_IOPOLL (1 << 0) #endif static int fio_libaio_commit(struct thread_data *td); @@ -38,7 +32,6 @@ struct libaio_data { struct iocb **iocbs; struct io_u **io_us; - struct iocb *user_iocbs; struct io_u **io_u_index; /* @@ -60,8 +53,6 @@ struct libaio_options { void *pad; unsigned int userspace_reap; unsigned int hipri; - unsigned int useriocb; - unsigned int fixedbufs; }; static struct fio_option options[] = { @@ -83,24 +74,6 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_LIBAIO, }, - { - .name = "useriocb", - .lname = "User IOCBs", - .type = FIO_OPT_STR_SET, - .off1 = offsetof(struct libaio_options, useriocb), - .help = "Use user mapped IOCBs", - .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, - }, - { - .name = "fixedbufs", - .lname = "Fixed (pre-mapped) IO buffers", - .type = FIO_OPT_STR_SET, - .off1 = offsetof(struct libaio_options, fixedbufs), - .help = "Pre map IO buffers", - .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, - }, { .name = NULL, }, @@ -117,36 +90,20 @@ static inline void ring_inc(struct libaio_data *ld, unsigned int *val, static int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u) { - struct libaio_data *ld = td->io_ops_data; struct fio_file *f = io_u->file; struct libaio_options *o = td->eo; struct iocb *iocb; - if (o->useriocb) - iocb = &ld->user_iocbs[io_u->index]; - else - iocb = &io_u->iocb; + iocb = &io_u->iocb; if (io_u->ddir == DDIR_READ) { - if (o->fixedbufs) { - iocb->aio_fildes = f->fd; - iocb->aio_lio_opcode = IO_CMD_PREAD; - iocb->u.c.offset = io_u->offset; - } else { - io_prep_pread(iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); - if (o->hipri) - iocb->u.c.flags |= IOCB_FLAG_HIPRI; - } + io_prep_pread(iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); + if (o->hipri) + iocb->u.c.flags |= IOCB_FLAG_HIPRI; } else if (io_u->ddir == DDIR_WRITE) { - if (o->fixedbufs) { - iocb->aio_fildes = f->fd; - iocb->aio_lio_opcode = IO_CMD_PWRITE; - iocb->u.c.offset = io_u->offset; - } else { - io_prep_pwrite(iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); - if (o->hipri) - iocb->u.c.flags |= IOCB_FLAG_HIPRI; - } + io_prep_pwrite(iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); + if (o->hipri) + iocb->u.c.flags |= IOCB_FLAG_HIPRI; } else if (ddir_sync(io_u->ddir)) io_prep_fsync(iocb, f->fd); @@ -156,16 +113,11 @@ static int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u) static struct io_u *fio_libaio_event(struct thread_data *td, int event) { struct libaio_data *ld = td->io_ops_data; - struct libaio_options *o = td->eo; struct io_event *ev; struct io_u *io_u; ev = ld->aio_events + event; - if (o->useriocb) { - int index = (int) (uintptr_t) ev->obj; - io_u = ld->io_u_index[index]; - } else - io_u = container_of(ev->obj, struct io_u, iocb); + io_u = container_of(ev->obj, struct io_u, iocb); if (ev->res != io_u->xfer_buflen) { if (ev->res > io_u->xfer_buflen) @@ -261,7 +213,6 @@ static enum fio_q_status fio_libaio_queue(struct thread_data *td, struct io_u *io_u) { struct libaio_data *ld = td->io_ops_data; - struct libaio_options *o = td->eo; fio_ro_check(td, io_u); @@ -292,11 +243,7 @@ static enum fio_q_status fio_libaio_queue(struct thread_data *td, return FIO_Q_COMPLETED; } - if (o->useriocb) - ld->iocbs[ld->head] = (struct iocb *) (uintptr_t) io_u->index; - else - ld->iocbs[ld->head] = &io_u->iocb; - + ld->iocbs[ld->head] = &io_u->iocb; ld->io_us[ld->head] = io_u; ring_inc(ld, &ld->head, 1); ld->queued++; @@ -415,87 +362,46 @@ static void fio_libaio_cleanup(struct thread_data *td) free(ld->aio_events); free(ld->iocbs); free(ld->io_us); - if (ld->user_iocbs) { - size_t size = td->o.iodepth * sizeof(struct iocb); - fio_memfree(ld->user_iocbs, size, false); - } free(ld); } } static int fio_libaio_old_queue_init(struct libaio_data *ld, unsigned int depth, - bool hipri, bool useriocb, bool fixedbufs) + bool hipri) { if (hipri) { log_err("fio: polled aio not available on your platform\n"); return 1; } - if (useriocb) { - log_err("fio: user mapped iocbs not available on your platform\n"); - return 1; - } - if (fixedbufs) { - log_err("fio: fixed buffers not available on your platform\n"); - return 1; - } return io_queue_init(depth, &ld->aio_ctx); } static int fio_libaio_queue_init(struct libaio_data *ld, unsigned int depth, - bool hipri, bool useriocb, bool fixedbufs) + bool hipri) { #ifdef __NR_sys_io_setup2 int ret, flags = 0; if (hipri) flags |= IOCTX_FLAG_IOPOLL; - if (useriocb) - flags |= IOCTX_FLAG_USERIOCB; - if (fixedbufs) { - struct rlimit rlim = { - .rlim_cur = RLIM_INFINITY, - .rlim_max = RLIM_INFINITY, - }; - - setrlimit(RLIMIT_MEMLOCK, &rlim); - flags |= IOCTX_FLAG_FIXEDBUFS; - } - ret = syscall(__NR_sys_io_setup2, depth, flags, ld->user_iocbs, - NULL, NULL, &ld->aio_ctx); + ret = syscall(__NR_sys_io_setup2, depth, flags, NULL, NULL, + &ld->aio_ctx); if (!ret) return 0; /* fall through to old syscall */ #endif - return fio_libaio_old_queue_init(ld, depth, hipri, useriocb, fixedbufs); + return fio_libaio_old_queue_init(ld, depth, hipri); } static int fio_libaio_post_init(struct thread_data *td) { struct libaio_data *ld = td->io_ops_data; struct libaio_options *o = td->eo; - struct io_u *io_u; - struct iocb *iocb; int err = 0; - if (o->fixedbufs) { - int i; - - for (i = 0; i < td->o.iodepth; i++) { - io_u = ld->io_u_index[i]; - iocb = &ld->user_iocbs[i]; - iocb->u.c.buf = io_u->buf; - iocb->u.c.nbytes = td_max_bs(td); - - iocb->u.c.flags = 0; - if (o->hipri) - iocb->u.c.flags |= IOCB_FLAG_HIPRI; - } - } - - err = fio_libaio_queue_init(ld, td->o.iodepth, o->hipri, o->useriocb, - o->fixedbufs); + err = fio_libaio_queue_init(ld, td->o.iodepth, o->hipri); if (err) { td_verror(td, -err, "io_queue_init"); return 1; @@ -506,20 +412,10 @@ static int fio_libaio_post_init(struct thread_data *td) static int fio_libaio_init(struct thread_data *td) { - struct libaio_options *o = td->eo; struct libaio_data *ld; ld = calloc(1, sizeof(*ld)); - if (o->useriocb) { - size_t size; - - ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *)); - size = td->o.iodepth * sizeof(struct iocb); - ld->user_iocbs = fio_memalign(page_size, size, false); - memset(ld->user_iocbs, 0, size); - } - ld->entries = td->o.iodepth; ld->is_pow2 = is_power_of_2(ld->entries); ld->aio_events = calloc(ld->entries, sizeof(struct io_event)); @@ -530,25 +426,11 @@ static int fio_libaio_init(struct thread_data *td) return 0; } -static int fio_libaio_io_u_init(struct thread_data *td, struct io_u *io_u) -{ - struct libaio_options *o = td->eo; - - if (o->useriocb) { - struct libaio_data *ld = td->io_ops_data; - - ld->io_u_index[io_u->index] = io_u; - } - - return 0; -} - static struct ioengine_ops ioengine = { .name = "libaio", .version = FIO_IOOPS_VERSION, .init = fio_libaio_init, .post_init = fio_libaio_post_init, - .io_u_init = fio_libaio_io_u_init, .prep = fio_libaio_prep, .queue = fio_libaio_queue, .commit = fio_libaio_commit, diff --git a/ioengines.c b/ioengines.c index b7df8608..45e769e6 100644 --- a/ioengines.c +++ b/ioengines.c @@ -131,7 +131,7 @@ static struct ioengine_ops *__load_ioengine(const char *name) /* * linux libaio has alias names, so convert to what we want */ - if (!strncmp(engine, "linuxaio", 8) || !strncmp(engine, "aio", 3)) { + if (!strncmp(engine, "linuxaio", 8)) { dprint(FD_IO, "converting ioengine name: %s -> libaio\n", name); strcpy(engine, "libaio"); } diff --git a/options.c b/options.c index 7a7006c1..626c7c17 100644 --- a/options.c +++ b/options.c @@ -1773,6 +1773,13 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .help = "Linux native asynchronous IO", }, #endif +#ifdef CONFIG_LIBAIO +#ifdef ARCH_HAVE_AIORING + { .ival = "aio-ring", + .help = "Linux native asynchronous IO", + }, +#endif +#endif #ifdef CONFIG_POSIXAIO { .ival = "posixaio", .help = "POSIX asynchronous IO", diff --git a/t/aio-ring.c b/t/aio-ring.c index c6106348..322f2ffa 100644 --- a/t/aio-ring.c +++ b/t/aio-ring.c @@ -24,38 +24,42 @@ #define IOCB_FLAG_HIPRI (1 << 2) -#define IOCTX_FLAG_IOPOLL (1 << 1) -#define IOCTX_FLAG_USERIOCB (1 << 0) +#define IOCTX_FLAG_IOPOLL (1 << 0) +#define IOCTX_FLAG_SCQRING (1 << 1) /* Use SQ/CQ rings */ #define IOCTX_FLAG_FIXEDBUFS (1 << 2) -#define IOCTX_FLAG_SCQRING (1 << 3) /* Use SQ/CQ rings */ -#define IOCTX_FLAG_SQTHREAD (1 << 4) /* Use SQ thread */ -#define IOCTX_FLAG_SQWQ (1 << 5) /* Use SQ wq */ +#define IOCTX_FLAG_SQTHREAD (1 << 3) /* Use SQ thread */ +#define IOCTX_FLAG_SQWQ (1 << 4) /* Use SQ wq */ #define barrier() __asm__ __volatile__("": : :"memory") #define min(a, b) ((a < b) ? (a) : (b)) +typedef uint64_t u64; typedef uint32_t u32; +typedef uint16_t u16; -struct aio_iocb_ring { +struct aio_sq_ring { union { struct { - u32 head, tail; + u32 head; + u32 tail; u32 nr_events; - u32 sq_thread_cpu; + u16 sq_thread_cpu; + u64 iocbs; }; - struct iocb pad_iocb; + u32 pad[16]; }; - struct iocb iocbs[0]; + u32 array[0]; }; -struct aio_io_event_ring { +struct aio_cq_ring { union { struct { - u32 head, tail; + u32 head; + u32 tail; u32 nr_events; }; - struct io_event pad_event; + struct io_event pad; }; struct io_event events[0]; }; @@ -76,8 +80,9 @@ struct submitter { unsigned long max_blocks; io_context_t ioc; struct drand48_data rand; - struct aio_iocb_ring *sq_ring; - struct aio_io_event_ring *cq_ring; + struct aio_sq_ring *sq_ring; + struct iocb *iocbs; + struct aio_cq_ring *cq_ring; int inflight; unsigned long reaps; unsigned long done; @@ -96,10 +101,10 @@ static int sq_thread = 0; /* use kernel submission thread */ static int sq_thread_cpu = 0; /* pin above thread to this CPU */ static int io_setup2(unsigned int nr_events, unsigned int flags, - struct iocb *iocbs, struct aio_iocb_ring *sq_ring, - struct aio_io_event_ring *cq_ring, io_context_t *ctx_idp) + struct aio_sq_ring *sq_ring, struct aio_cq_ring *cq_ring, + io_context_t *ctx_idp) { - return syscall(335, nr_events, flags, iocbs, sq_ring, cq_ring, ctx_idp); + return syscall(335, nr_events, flags, sq_ring, cq_ring, ctx_idp); } static int io_ring_enter(io_context_t ctx, unsigned int to_submit, @@ -132,8 +137,7 @@ static void init_io(struct submitter *s, int fd, struct iocb *iocb) static int prep_more_ios(struct submitter *s, int fd, int max_ios) { - struct aio_iocb_ring *ring = s->sq_ring; - struct iocb *iocb; + struct aio_sq_ring *ring = s->sq_ring; u32 tail, next_tail, prepped = 0; next_tail = tail = ring->tail; @@ -146,8 +150,8 @@ static int prep_more_ios(struct submitter *s, int fd, int max_ios) if (next_tail == ring->head) break; - iocb = &s->sq_ring->iocbs[tail]; - init_io(s, fd, iocb); + init_io(s, fd, &s->iocbs[tail]); + s->sq_ring->array[tail] = tail; prepped++; tail = next_tail; } while (prepped < max_ios); @@ -185,7 +189,7 @@ static int get_file_size(int fd, unsigned long *blocks) static int reap_events(struct submitter *s) { - struct aio_io_event_ring *ring = s->cq_ring; + struct aio_cq_ring *ring = s->cq_ring; struct io_event *ev; u32 head, reaped = 0; @@ -196,8 +200,7 @@ static int reap_events(struct submitter *s) break; ev = &ring->events[head]; if (ev->res != BS) { - int index = (int) (uintptr_t) ev->obj; - struct iocb *iocb = &s->sq_ring->iocbs[index]; + struct iocb *iocb = ev->obj; printf("io: unexpected ret=%ld\n", ev->res); printf("offset=%lu, size=%lu\n", (unsigned long) iocb->u.c.offset, (unsigned long) iocb->u.c.nbytes); @@ -351,20 +354,31 @@ int main(int argc, char *argv[]) arm_sig_int(); - size = sizeof(struct aio_iocb_ring) + RING_SIZE * sizeof(struct iocb); + size = sizeof(struct iocb) * RING_SIZE; + if (posix_memalign(&p, 4096, size)) + return 1; + memset(p, 0, size); + s->iocbs = p; + + size = sizeof(struct aio_sq_ring) + RING_SIZE * sizeof(u32); if (posix_memalign(&p, 4096, size)) return 1; s->sq_ring = p; memset(p, 0, size); + s->sq_ring->nr_events = RING_SIZE; + s->sq_ring->iocbs = (u64) s->iocbs; - size = sizeof(struct aio_io_event_ring) + RING_SIZE * sizeof(struct io_event); + /* CQ ring must be twice as big */ + size = sizeof(struct aio_cq_ring) + + 2 * RING_SIZE * sizeof(struct io_event); if (posix_memalign(&p, 4096, size)) return 1; s->cq_ring = p; memset(p, 0, size); + s->cq_ring->nr_events = 2 * RING_SIZE; for (j = 0; j < RING_SIZE; j++) { - struct iocb *iocb = &s->sq_ring->iocbs[j]; + struct iocb *iocb = &s->iocbs[j]; if (posix_memalign(&iocb->u.c.buf, BS, BS)) { printf("failed alloc\n"); @@ -385,7 +399,7 @@ int main(int argc, char *argv[]) s->sq_ring->sq_thread_cpu = sq_thread_cpu; } - err = io_setup2(RING_SIZE, flags, s->sq_ring->iocbs, s->sq_ring, s->cq_ring, &s->ioc); + err = io_setup2(RING_SIZE, flags, s->sq_ring, s->cq_ring, &s->ioc); if (err) { printf("ctx_init failed: %s, %d\n", strerror(errno), err); return 1;