From: David Howells <dhowells@redhat.com>
To: Trond Myklebust <trondmy@hammerspace.com>,
Anna Schumaker <anna.schumaker@netapp.com>,
Steve French <sfrench@samba.org>,
Jeff Layton <jlayton@redhat.com>
Cc: dhowells@redhat.com, Matthew Wilcox <willy@infradead.org>,
Alexander Viro <viro@zeniv.linux.org.uk>,
linux-afs@lists.infradead.org, linux-nfs@vger.kernel.org,
linux-cifs@vger.kernel.org, ceph-devel@vger.kernel.org,
v9fs-developer@lists.sourceforge.net,
linux-fsdevel@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [RFC PATCH 21/61] fscache: Provide a simple thread pool for running ops asynchronously
Date: Mon, 04 May 2020 18:10:41 +0100 [thread overview]
Message-ID: <158861224099.340223.12658954309881846480.stgit@warthog.procyon.org.uk> (raw)
In-Reply-To: <158861203563.340223.7585359869938129395.stgit@warthog.procyon.org.uk>
Provide a simple thread pool that can be used to run cookie management
operations in the background and a dispatcher infrastructure to punt
operations to the pool if threads are available or to just run the
operation in the calling thread if not.
A future patch will replace all the object state machine stuff with whole
routines that do all the work in one go without trying to interleave bits
from various objects.
Signed-off-by: David Howells <dhowells@redhat.com>
---
fs/fscache/Makefile | 1
fs/fscache/dispatcher.c | 144 ++++++++++++++++++++++++++++++++++++++++
fs/fscache/internal.h | 8 ++
fs/fscache/main.c | 7 ++
include/trace/events/fscache.h | 6 +-
5 files changed, 165 insertions(+), 1 deletion(-)
create mode 100644 fs/fscache/dispatcher.c
diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile
index ac3fcd909fff..7b10c6aad157 100644
--- a/fs/fscache/Makefile
+++ b/fs/fscache/Makefile
@@ -6,6 +6,7 @@
fscache-y := \
cache.o \
cookie.o \
+ dispatcher.o \
fsdef.o \
main.o \
netfs.o \
diff --git a/fs/fscache/dispatcher.c b/fs/fscache/dispatcher.c
new file mode 100644
index 000000000000..fba71b99c951
--- /dev/null
+++ b/fs/fscache/dispatcher.c
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Object dispatcher
+ *
+ * Copyright (C) 2019 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ */
+
+#define FSCACHE_DEBUG_LEVEL OPERATION
+#include <linux/kthread.h>
+#include <linux/slab.h>
+#include <linux/wait.h>
+#include <linux/freezer.h>
+#include "internal.h"
+
+#define FSCACHE_DISPATCHER_POOL_SIZE 8
+
+static LIST_HEAD(fscache_pending_work);
+static DEFINE_SPINLOCK(fscache_work_lock);
+static DECLARE_WAIT_QUEUE_HEAD(fscache_dispatcher_pool);
+static struct completion fscache_dispatcher_pool_done[FSCACHE_DISPATCHER_POOL_SIZE];
+static bool fscache_dispatcher_stop;
+
+struct fscache_work {
+ struct list_head link;
+ struct fscache_cookie *cookie;
+ struct fscache_object *object;
+ int param;
+ void (*func)(struct fscache_cookie *, struct fscache_object *, int);
+};
+
+/*
+ * Attempt to queue some work to do. If there's too much asynchronous work
+ * already queued, we'll do it here in this thread instead.
+ */
+void fscache_dispatch(struct fscache_cookie *cookie,
+ struct fscache_object *object,
+ int param,
+ void (*func)(struct fscache_cookie *,
+ struct fscache_object *, int))
+{
+ struct fscache_work *work;
+ bool queued = false;
+
+ work = kzalloc(sizeof(struct fscache_work), GFP_KERNEL);
+ if (work) {
+ work->cookie = cookie;
+ work->object = object;
+ work->param = param;
+ work->func = func;
+
+ spin_lock(&fscache_work_lock);
+ if (waitqueue_active(&fscache_dispatcher_pool) ||
+ list_empty(&fscache_pending_work)) {
+ fscache_cookie_get(cookie, fscache_cookie_get_work);
+ list_add_tail(&work->link, &fscache_pending_work);
+ wake_up(&fscache_dispatcher_pool);
+ queued = true;
+ }
+ spin_unlock(&fscache_work_lock);
+ }
+
+ if (!queued) {
+ kfree(work);
+ func(cookie, object, param);
+ }
+}
+
+/*
+ * A dispatcher thread.
+ */
+static int fscache_dispatcher(void *data)
+{
+ struct completion *done = data;
+
+ for (;;) {
+ if (!list_empty(&fscache_pending_work)) {
+ struct fscache_work *work = NULL;
+
+ spin_lock(&fscache_work_lock);
+ if (!list_empty(&fscache_pending_work)) {
+ work = list_entry(fscache_pending_work.next,
+ struct fscache_work, link);
+ list_del_init(&work->link);
+ }
+ spin_unlock(&fscache_work_lock);
+
+ if (work) {
+ work->func(work->cookie, work->object, work->param);
+ fscache_cookie_put(work->cookie, fscache_cookie_put_work);
+ kfree(work);
+ }
+ continue;
+ } else if (fscache_dispatcher_stop) {
+ break;
+ }
+
+ wait_event_freezable(fscache_dispatcher_pool,
+ (fscache_dispatcher_stop ||
+ !list_empty(&fscache_pending_work)));
+ }
+
+ complete_and_exit(done, 0);
+}
+
+/*
+ * Start up the dispatcher threads.
+ */
+int fscache_init_dispatchers(void)
+{
+ struct task_struct *t;
+ int i;
+
+ for (i = 0; i < FSCACHE_DISPATCHER_POOL_SIZE; i++) {
+ t = kthread_create(fscache_dispatcher,
+ &fscache_dispatcher_pool_done[i],
+ "kfsc/%d", i);
+ if (IS_ERR(t))
+ goto failed;
+ wake_up_process(t);
+ }
+
+ return 0;
+
+failed:
+ fscache_dispatcher_stop = true;
+ wake_up_all(&fscache_dispatcher_pool);
+ for (i--; i >= 0; i--)
+ wait_for_completion(&fscache_dispatcher_pool_done[i]);
+ return PTR_ERR(t);
+}
+
+/*
+ * Kill off the dispatcher threads.
+ */
+void fscache_kill_dispatchers(void)
+{
+ int i;
+
+ fscache_dispatcher_stop = true;
+ wake_up_all(&fscache_dispatcher_pool);
+
+ for (i = 0; i < FSCACHE_DISPATCHER_POOL_SIZE; i++)
+ wait_for_completion(&fscache_dispatcher_pool_done[i]);
+}
diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h
index bc5539d2157b..2100e2222884 100644
--- a/fs/fscache/internal.h
+++ b/fs/fscache/internal.h
@@ -75,6 +75,14 @@ extern struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *);
extern void fscache_cookie_put(struct fscache_cookie *,
enum fscache_cookie_trace);
+/*
+ * dispatcher.c
+ */
+extern void fscache_dispatch(struct fscache_cookie *, struct fscache_object *, int,
+ void (*func)(struct fscache_cookie *, struct fscache_object *, int));
+extern int fscache_init_dispatchers(void);
+extern void fscache_kill_dispatchers(void);
+
/*
* fsdef.c
*/
diff --git a/fs/fscache/main.c b/fs/fscache/main.c
index 59c2494efda3..6f225ae0fd99 100644
--- a/fs/fscache/main.c
+++ b/fs/fscache/main.c
@@ -126,6 +126,10 @@ static int __init fscache_init(void)
for_each_possible_cpu(cpu)
init_waitqueue_head(&per_cpu(fscache_object_cong_wait, cpu));
+ ret = fscache_init_dispatchers();
+ if (ret < 0)
+ goto error_dispatchers;
+
ret = fscache_proc_init();
if (ret < 0)
goto error_proc;
@@ -160,6 +164,8 @@ static int __init fscache_init(void)
unregister_sysctl_table(fscache_sysctl_header);
error_sysctl:
#endif
+ fscache_kill_dispatchers();
+error_dispatchers:
fscache_proc_cleanup();
error_proc:
destroy_workqueue(fscache_op_wq);
@@ -184,6 +190,7 @@ static void __exit fscache_exit(void)
unregister_sysctl_table(fscache_sysctl_header);
#endif
fscache_proc_cleanup();
+ fscache_kill_dispatchers();
destroy_workqueue(fscache_op_wq);
destroy_workqueue(fscache_object_wq);
pr_notice("Unloaded\n");
diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h
index 08d7de72409d..fb3fdf2921ee 100644
--- a/include/trace/events/fscache.h
+++ b/include/trace/events/fscache.h
@@ -26,11 +26,13 @@ enum fscache_cookie_trace {
fscache_cookie_get_attach_object,
fscache_cookie_get_reacquire,
fscache_cookie_get_register_netfs,
+ fscache_cookie_get_work,
fscache_cookie_put_acquire_nobufs,
fscache_cookie_put_dup_netfs,
fscache_cookie_put_relinquish,
fscache_cookie_put_object,
fscache_cookie_put_parent,
+ fscache_cookie_put_work,
};
#endif
@@ -45,11 +47,13 @@ enum fscache_cookie_trace {
EM(fscache_cookie_get_attach_object, "GET obj") \
EM(fscache_cookie_get_reacquire, "GET raq") \
EM(fscache_cookie_get_register_netfs, "GET net") \
+ EM(fscache_cookie_get_work, "GET wrk") \
EM(fscache_cookie_put_acquire_nobufs, "PUT nbf") \
EM(fscache_cookie_put_dup_netfs, "PUT dnt") \
EM(fscache_cookie_put_relinquish, "PUT rlq") \
EM(fscache_cookie_put_object, "PUT obj") \
- E_(fscache_cookie_put_parent, "PUT prn")
+ EM(fscache_cookie_put_parent, "PUT prn") \
+ E_(fscache_cookie_put_work, "PUT wrk")
/*
* Export enum symbols via userspace.
next prev parent reply other threads:[~2020-05-04 17:10 UTC|newest]
Thread overview: 68+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-05-04 17:07 [RFC PATCH 00/61] fscache, cachefiles: Rewrite the I/O interface in terms of kiocb/iov_iter David Howells
2020-05-04 17:07 ` [RFC PATCH 01/61] afs: Make afs_zap_data() static David Howells
2020-05-04 17:07 ` [RFC PATCH 02/61] iov_iter: Add ITER_MAPPING David Howells
2020-05-04 17:07 ` [RFC PATCH 03/61] vm: Add wait/unlock functions for PG_fscache David Howells
2020-05-04 17:08 ` [RFC PATCH 04/61] vfs: Export rw_verify_area() for use by cachefiles David Howells
2020-05-04 17:08 ` [RFC PATCH 05/61] vfs: Provide S_CACHE_FILE inode flag David Howells
2020-05-04 17:08 ` [RFC PATCH 06/61] afs: Disable use of the fscache I/O routines David Howells
2020-05-04 17:08 ` [RFC PATCH 07/61] fscache: Add a cookie debug ID and use that in traces David Howells
2020-05-04 17:08 ` [RFC PATCH 08/61] fscache: Procfile to display cookies David Howells
2020-05-04 17:08 ` [RFC PATCH 09/61] fscache: Temporarily disable network filesystems' use of fscache David Howells
2020-05-04 17:08 ` [RFC PATCH 10/61] fscache: Remove the old I/O API David Howells
2020-05-04 17:09 ` [RFC PATCH 11/61] fscache: Remove the netfs data from the cookie David Howells
2020-05-04 17:09 ` [RFC PATCH 12/61] fscache: Remove struct fscache_cookie_def David Howells
2020-05-04 17:09 ` [RFC PATCH 13/61] fscache: Remove store_limit* from struct fscache_object David Howells
2020-05-04 17:09 ` [RFC PATCH 14/61] fscache: Remove fscache_check_consistency() David Howells
2020-05-04 17:09 ` [RFC PATCH 15/61] fscache: Remove fscache_attr_changed() David Howells
2020-05-04 17:09 ` [RFC PATCH 16/61] fscache: Remove obsolete stats David Howells
2020-05-04 17:10 ` [RFC PATCH 17/61] fscache: Remove old I/O tracepoints David Howells
2020-05-04 17:10 ` [RFC PATCH 18/61] fscache: Temporarily disable fscache_invalidate() David Howells
2020-05-04 17:10 ` [RFC PATCH 19/61] fscache: Remove the I/O operation manager David Howells
2020-05-04 17:10 ` [RFC PATCH 20/61] cachefiles: Remove tree of active files and use S_CACHE_FILE inode flag David Howells
2020-05-04 17:10 ` David Howells [this message]
2020-05-04 17:10 ` [RFC PATCH 23/61] fscache: Rewrite the I/O API based on iov_iter David Howells
2020-05-04 17:11 ` [RFC PATCH 24/61] fscache: Remove fscache_wait_on_invalidate() David Howells
2020-05-04 17:11 ` [RFC PATCH 25/61] fscache: Keep track of size of a file last set independently on the server David Howells
2020-05-04 17:11 ` [RFC PATCH 26/61] fscache, cachefiles: Fix disabled histogram warnings David Howells
2020-05-04 17:11 ` [RFC PATCH 27/61] fscache: Recast assertion in terms of cookie not being an index David Howells
2020-05-04 17:11 ` [RFC PATCH 28/61] cachefiles: Remove some redundant checks on unsigned values David Howells
2020-05-04 17:11 ` [RFC PATCH 29/61] cachefiles: trace: Log coherency checks David Howells
2020-05-04 17:12 ` [RFC PATCH 30/61] cachefiles: Split cachefiles_drop_object() up a bit David Howells
2020-05-04 17:12 ` [RFC PATCH 31/61] cachefiles: Implement new fscache I/O backend API David Howells
2020-05-04 17:12 ` [RFC PATCH 32/61] cachefiles: Merge object->backer into object->dentry David Howells
2020-05-04 17:12 ` [RFC PATCH 33/61] cachefiles: Implement a content-present indicator and bitmap David Howells
2020-05-04 17:12 ` [RFC PATCH 34/61] cachefiles: Implement extent shaper David Howells
2020-05-04 17:12 ` [RFC PATCH 35/61] cachefiles: Round the cachefile size up to DIO block size David Howells
2020-05-04 17:12 ` [RFC PATCH 36/61] cachefiles: Implement read and write parts of new I/O API David Howells
2020-05-04 17:13 ` [RFC PATCH 37/61] cachefiles: Add I/O tracepoints David Howells
2020-05-04 17:13 ` [RFC PATCH 38/61] fscache: Add read helper David Howells
2020-05-04 17:13 ` [RFC PATCH 39/61] fscache: Display cache-specific data in /proc/fs/fscache/objects David Howells
2020-05-04 17:13 ` [RFC PATCH 40/61] fscache: Remove more obsolete stats David Howells
2020-05-04 17:13 ` [RFC PATCH 41/61] fscache: New stats David Howells
2020-05-04 17:13 ` [RFC PATCH 42/61] fscache, cachefiles: Rewrite invalidation David Howells
2020-05-04 17:13 ` [RFC PATCH 43/61] fscache: Implement "will_modify" parameter on fscache_use_cookie() David Howells
2020-05-04 17:14 ` [RFC PATCH 44/61] fscache: Provide resize operation David Howells
2020-05-04 17:14 ` [RFC PATCH 45/61] fscache: Remove the update operation David Howells
2020-05-04 17:14 ` [RFC PATCH 46/61] cachefiles: Shape write requests David Howells
2020-05-04 17:14 ` [RFC PATCH 47/61] afs: Remove afs_zero_fid as it's not used David Howells
2020-05-04 17:14 ` [RFC PATCH 48/61] afs: Move key to afs_read struct David Howells
2020-05-04 17:14 ` [RFC PATCH 49/61] afs: Don't truncate iter during data fetch David Howells
2020-05-04 17:15 ` [RFC PATCH 50/61] afs: Set up the iov_iter before calling afs_extract_data() David Howells
2020-05-04 17:15 ` [RFC PATCH 51/61] afs: Use ITER_MAPPING for writing David Howells
2020-05-04 17:15 ` [RFC PATCH 52/61] afs: Interpose struct fscache_io_request into struct afs_read David Howells
2020-05-04 17:15 ` [RFC PATCH 53/61] afs: Note the amount transferred in fetch-data delivery David Howells
2020-05-04 17:15 ` [RFC PATCH 54/61] afs: Wait on PG_fscache before modifying/releasing a page David Howells
2020-05-05 11:59 ` Matthew Wilcox
2020-05-06 7:57 ` David Howells
2020-05-06 11:09 ` Matthew Wilcox
2020-05-06 14:24 ` David Howells
2020-05-08 14:39 ` David Howells
2020-05-04 17:15 ` [RFC PATCH 55/61] afs: Use new fscache I/O API David Howells
2020-05-04 17:15 ` [RFC PATCH 56/61] afs: Copy local writes to the cache when writing to the server David Howells
2020-05-04 17:16 ` [RFC PATCH 57/61] afs: Invoke fscache_resize_cookie() when handling ATTR_SIZE for setattr David Howells
2020-05-04 17:16 ` [RFC PATCH 58/61] fscache: Rewrite the main document David Howells
2020-05-04 17:16 ` [RFC PATCH 59/61] fscache: Remove the obsolete API bits from the documentation David Howells
2020-05-04 17:16 ` [RFC PATCH 60/61] fscache: Document the new netfs API David Howells
2020-05-04 17:16 ` [RFC PATCH 61/61] fscache: Document the rewritten cache backend API David Howells
2020-05-04 17:54 ` [RFC PATCH 00/61] fscache, cachefiles: Rewrite the I/O interface in terms of kiocb/iov_iter Jeff Layton
2020-05-05 6:05 ` Christoph Hellwig
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=158861224099.340223.12658954309881846480.stgit@warthog.procyon.org.uk \
--to=dhowells@redhat.com \
--cc=anna.schumaker@netapp.com \
--cc=ceph-devel@vger.kernel.org \
--cc=jlayton@redhat.com \
--cc=linux-afs@lists.infradead.org \
--cc=linux-cifs@vger.kernel.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nfs@vger.kernel.org \
--cc=sfrench@samba.org \
--cc=trondmy@hammerspace.com \
--cc=v9fs-developer@lists.sourceforge.net \
--cc=viro@zeniv.linux.org.uk \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).