* [PATCH 0/4] io_uring: report locked memory usage
@ 2020-06-16 23:36 Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 1/4] io_uring: add wrappers for memory accounting Bijan Mottahedeh
` (4 more replies)
0 siblings, 5 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
This patch set adds support for reporting of locked memory usage.
Patches 1 and 2 are prep patches to facilitate the reporting.
Patch 3 reports all locked memory as pinned.
Patch 4 reports ring memory as locked and registered memory as pinned.
This seems more appropriate but kept it a separate patch in case it
should be dropped.
Bijan Mottahedeh (4):
io_uring: add wrappers for memory accounting
io_uring: rename ctx->account_mem field
io_uring: report pinned memory usage
io_uring: separate reporting of ring pages from registered pages
fs/io_uring.c | 93 ++++++++++++++++++++++++++++++++++++++++++-----------------
1 file changed, 66 insertions(+), 27 deletions(-)
--
1.8.3.1
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 1/4] io_uring: add wrappers for memory accounting
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
@ 2020-06-16 23:36 ` Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 2/4] io_uring: rename ctx->account_mem field Bijan Mottahedeh
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
Facilitate separation of locked memory usage reporting vs. limiting for
upcoming patches. No functional changes.
Signed-off-by: Bijan Mottahedeh <bijan.mottahedeh@oracle.com>
---
fs/io_uring.c | 48 ++++++++++++++++++++++++++++--------------------
1 file changed, 28 insertions(+), 20 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index a07c44e..ebd3f62 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -7028,12 +7028,14 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx,
return ret;
}
-static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
+static inline void __io_unaccount_mem(struct user_struct *user,
+ unsigned long nr_pages)
{
atomic_long_sub(nr_pages, &user->locked_vm);
}
-static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
+static inline int __io_account_mem(struct user_struct *user,
+ unsigned long nr_pages)
{
unsigned long page_limit, cur_pages, new_pages;
@@ -7051,6 +7053,20 @@ static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
return 0;
}
+static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+{
+ if (ctx->account_mem)
+ __io_unaccount_mem(ctx->user, nr_pages);
+}
+
+static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+{
+ if (ctx->account_mem)
+ return (__io_account_mem(ctx->user, nr_pages));
+
+ return 0;
+}
+
static void io_mem_free(void *ptr)
{
struct page *page;
@@ -7125,8 +7141,7 @@ static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
for (j = 0; j < imu->nr_bvecs; j++)
unpin_user_page(imu->bvec[j].bv_page);
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user, imu->nr_bvecs);
+ io_unaccount_mem(ctx, imu->nr_bvecs);
kvfree(imu->bvec);
imu->nr_bvecs = 0;
}
@@ -7209,11 +7224,9 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
start = ubuf >> PAGE_SHIFT;
nr_pages = end - start;
- if (ctx->account_mem) {
- ret = io_account_mem(ctx->user, nr_pages);
- if (ret)
- goto err;
- }
+ ret = io_account_mem(ctx, nr_pages);
+ if (ret)
+ goto err;
ret = 0;
if (!pages || nr_pages > got_pages) {
@@ -7226,8 +7239,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
GFP_KERNEL);
if (!pages || !vmas) {
ret = -ENOMEM;
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user, nr_pages);
+ io_unaccount_mem(ctx, nr_pages);
goto err;
}
got_pages = nr_pages;
@@ -7237,8 +7249,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
GFP_KERNEL);
ret = -ENOMEM;
if (!imu->bvec) {
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user, nr_pages);
+ io_unaccount_mem(ctx, nr_pages);
goto err;
}
@@ -7269,8 +7280,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
*/
if (pret > 0)
unpin_user_pages(pages, pret);
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user, nr_pages);
+ io_unaccount_mem(ctx, nr_pages);
kvfree(imu->bvec);
goto err;
}
@@ -7375,9 +7385,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)
io_mem_free(ctx->sq_sqes);
percpu_ref_exit(&ctx->refs);
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user,
- ring_pages(ctx->sq_entries, ctx->cq_entries));
+ io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries));
free_uid(ctx->user);
put_cred(ctx->creds);
kfree(ctx->cancel_hash);
@@ -7916,7 +7924,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
account_mem = !capable(CAP_IPC_LOCK);
if (account_mem) {
- ret = io_account_mem(user,
+ ret = __io_account_mem(user,
ring_pages(p->sq_entries, p->cq_entries));
if (ret) {
free_uid(user);
@@ -7927,7 +7935,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
ctx = io_ring_ctx_alloc(p);
if (!ctx) {
if (account_mem)
- io_unaccount_mem(user, ring_pages(p->sq_entries,
+ __io_unaccount_mem(user, ring_pages(p->sq_entries,
p->cq_entries));
free_uid(user);
return -ENOMEM;
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PATCH 2/4] io_uring: rename ctx->account_mem field
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 1/4] io_uring: add wrappers for memory accounting Bijan Mottahedeh
@ 2020-06-16 23:36 ` Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 3/4] io_uring: report pinned memory usage Bijan Mottahedeh
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
Rename account_mem to limit_name to clarify its purpose.
Signed-off-by: Bijan Mottahedeh <bijan.mottahedeh@oracle.com>
---
fs/io_uring.c | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index ebd3f62..0bcbc1a 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -226,7 +226,7 @@ struct io_ring_ctx {
struct {
unsigned int flags;
unsigned int compat: 1;
- unsigned int account_mem: 1;
+ unsigned int limit_mem: 1;
unsigned int cq_overflow_flushed: 1;
unsigned int drain_next: 1;
unsigned int eventfd_async: 1;
@@ -7055,13 +7055,13 @@ static inline int __io_account_mem(struct user_struct *user,
static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{
- if (ctx->account_mem)
+ if (ctx->limit_mem)
__io_unaccount_mem(ctx->user, nr_pages);
}
static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{
- if (ctx->account_mem)
+ if (ctx->limit_mem)
return (__io_account_mem(ctx->user, nr_pages));
return 0;
@@ -7882,7 +7882,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
{
struct user_struct *user = NULL;
struct io_ring_ctx *ctx;
- bool account_mem;
+ bool limit_mem;
int ret;
if (!entries)
@@ -7921,9 +7921,9 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
}
user = get_uid(current_user());
- account_mem = !capable(CAP_IPC_LOCK);
+ limit_mem = !capable(CAP_IPC_LOCK);
- if (account_mem) {
+ if (limit_mem) {
ret = __io_account_mem(user,
ring_pages(p->sq_entries, p->cq_entries));
if (ret) {
@@ -7934,14 +7934,14 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
ctx = io_ring_ctx_alloc(p);
if (!ctx) {
- if (account_mem)
+ if (limit_mem)
__io_unaccount_mem(user, ring_pages(p->sq_entries,
p->cq_entries));
free_uid(user);
return -ENOMEM;
}
ctx->compat = in_compat_syscall();
- ctx->account_mem = account_mem;
+ ctx->limit_mem = limit_mem;
ctx->user = user;
ctx->creds = get_current_cred();
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PATCH 3/4] io_uring: report pinned memory usage
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 1/4] io_uring: add wrappers for memory accounting Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 2/4] io_uring: rename ctx->account_mem field Bijan Mottahedeh
@ 2020-06-16 23:36 ` Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 4/4] io_uring: separate reporting of ring pages from registered pages Bijan Mottahedeh
2020-06-17 15:35 ` [PATCH 0/4] io_uring: report locked memory usage Jens Axboe
4 siblings, 0 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
Report pinned memory usage always, regardless of whether locked memory
limit is enforced.
Signed-off-by: Bijan Mottahedeh <bijan.mottahedeh@oracle.com>
---
fs/io_uring.c | 22 ++++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 0bcbc1a..851ff21 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -7057,12 +7057,23 @@ static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{
if (ctx->limit_mem)
__io_unaccount_mem(ctx->user, nr_pages);
+
+ if (ctx->sqo_mm)
+ atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
}
static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{
- if (ctx->limit_mem)
- return (__io_account_mem(ctx->user, nr_pages));
+ int ret;
+
+ if (ctx->limit_mem) {
+ ret = __io_account_mem(ctx->user, nr_pages);
+ if (ret)
+ return ret;
+ }
+
+ if (ctx->sqo_mm)
+ atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
return 0;
}
@@ -7364,8 +7375,10 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx)
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
io_finish_async(ctx);
- if (ctx->sqo_mm)
+ if (ctx->sqo_mm) {
mmdrop(ctx->sqo_mm);
+ ctx->sqo_mm = NULL;
+ }
io_iopoll_reap_events(ctx);
io_sqe_buffer_unregister(ctx);
@@ -7941,7 +7954,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
return -ENOMEM;
}
ctx->compat = in_compat_syscall();
- ctx->limit_mem = limit_mem;
ctx->user = user;
ctx->creds = get_current_cred();
@@ -7988,6 +8000,8 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
goto err;
trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
+ io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries));
+ ctx->limit_mem = limit_mem;
return ret;
err:
io_ring_ctx_wait_and_kill(ctx);
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PATCH 4/4] io_uring: separate reporting of ring pages from registered pages
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
` (2 preceding siblings ...)
2020-06-16 23:36 ` [PATCH 3/4] io_uring: report pinned memory usage Bijan Mottahedeh
@ 2020-06-16 23:36 ` Bijan Mottahedeh
2020-06-17 15:35 ` [PATCH 0/4] io_uring: report locked memory usage Jens Axboe
4 siblings, 0 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
Ring pages are not pinned so it is more appropriate to report them
as locked.
Signed-off-by: Bijan Mottahedeh <bijan.mottahedeh@oracle.com>
---
fs/io_uring.c | 43 ++++++++++++++++++++++++++++++-------------
1 file changed, 30 insertions(+), 13 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 851ff21..c308dad 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -876,6 +876,11 @@ struct io_op_def {
},
};
+enum io_mem_account {
+ ACCT_LOCKED,
+ ACCT_PINNED,
+};
+
static void io_wq_submit_work(struct io_wq_work **workptr);
static void io_cqring_fill_event(struct io_kiocb *req, long res);
static void io_put_req(struct io_kiocb *req);
@@ -7053,16 +7058,22 @@ static inline int __io_account_mem(struct user_struct *user,
return 0;
}
-static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
+ enum io_mem_account acct)
{
if (ctx->limit_mem)
__io_unaccount_mem(ctx->user, nr_pages);
- if (ctx->sqo_mm)
- atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
+ if (ctx->sqo_mm) {
+ if (acct == ACCT_LOCKED)
+ ctx->sqo_mm->locked_vm -= nr_pages;
+ else if (acct == ACCT_PINNED)
+ atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
+ }
}
-static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
+ enum io_mem_account acct)
{
int ret;
@@ -7072,8 +7083,12 @@ static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
return ret;
}
- if (ctx->sqo_mm)
- atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
+ if (ctx->sqo_mm) {
+ if (acct == ACCT_LOCKED)
+ ctx->sqo_mm->locked_vm += nr_pages;
+ else if (acct == ACCT_PINNED)
+ atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
+ }
return 0;
}
@@ -7152,7 +7167,7 @@ static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
for (j = 0; j < imu->nr_bvecs; j++)
unpin_user_page(imu->bvec[j].bv_page);
- io_unaccount_mem(ctx, imu->nr_bvecs);
+ io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
kvfree(imu->bvec);
imu->nr_bvecs = 0;
}
@@ -7235,7 +7250,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
start = ubuf >> PAGE_SHIFT;
nr_pages = end - start;
- ret = io_account_mem(ctx, nr_pages);
+ ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
if (ret)
goto err;
@@ -7250,7 +7265,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
GFP_KERNEL);
if (!pages || !vmas) {
ret = -ENOMEM;
- io_unaccount_mem(ctx, nr_pages);
+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
goto err;
}
got_pages = nr_pages;
@@ -7260,7 +7275,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
GFP_KERNEL);
ret = -ENOMEM;
if (!imu->bvec) {
- io_unaccount_mem(ctx, nr_pages);
+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
goto err;
}
@@ -7291,7 +7306,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
*/
if (pret > 0)
unpin_user_pages(pages, pret);
- io_unaccount_mem(ctx, nr_pages);
+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
kvfree(imu->bvec);
goto err;
}
@@ -7398,7 +7413,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)
io_mem_free(ctx->sq_sqes);
percpu_ref_exit(&ctx->refs);
- io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries));
+ io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
+ ACCT_LOCKED);
free_uid(ctx->user);
put_cred(ctx->creds);
kfree(ctx->cancel_hash);
@@ -8000,7 +8016,8 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
goto err;
trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
- io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries));
+ io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
+ ACCT_LOCKED);
ctx->limit_mem = limit_mem;
return ret;
err:
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH 0/4] io_uring: report locked memory usage
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
` (3 preceding siblings ...)
2020-06-16 23:36 ` [PATCH 4/4] io_uring: separate reporting of ring pages from registered pages Bijan Mottahedeh
@ 2020-06-17 15:35 ` Jens Axboe
4 siblings, 0 replies; 6+ messages in thread
From: Jens Axboe @ 2020-06-17 15:35 UTC (permalink / raw)
To: Bijan Mottahedeh; +Cc: io-uring
On 6/16/20 5:36 PM, Bijan Mottahedeh wrote:
> This patch set adds support for reporting of locked memory usage.
>
> Patches 1 and 2 are prep patches to facilitate the reporting.
>
> Patch 3 reports all locked memory as pinned.
>
> Patch 4 reports ring memory as locked and registered memory as pinned.
> This seems more appropriate but kept it a separate patch in case it
> should be dropped.
>
> Bijan Mottahedeh (4):
> io_uring: add wrappers for memory accounting
> io_uring: rename ctx->account_mem field
> io_uring: report pinned memory usage
> io_uring: separate reporting of ring pages from registered pages
>
> fs/io_uring.c | 93 ++++++++++++++++++++++++++++++++++++++++++-----------------
> 1 file changed, 66 insertions(+), 27 deletions(-)
Applied for 5.9, thanks.
--
Jens Axboe
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2020-06-17 15:35 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 1/4] io_uring: add wrappers for memory accounting Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 2/4] io_uring: rename ctx->account_mem field Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 3/4] io_uring: report pinned memory usage Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 4/4] io_uring: separate reporting of ring pages from registered pages Bijan Mottahedeh
2020-06-17 15:35 ` [PATCH 0/4] io_uring: report locked memory usage Jens Axboe
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.