CEPH-Devel Archive on lore.kernel.org
 help / color / Atom feed
From: David Howells <dhowells@redhat.com>
To: Trond Myklebust <trondmy@hammerspace.com>,
	Anna Schumaker <anna.schumaker@netapp.com>,
	Steve French <sfrench@samba.org>,
	Dominique Martinet <asmadeus@codewreck.org>
Cc: dhowells@redhat.com, Jeff Layton <jlayton@redhat.com>,
	Matthew Wilcox <willy@infradead.org>,
	Alexander Viro <viro@zeniv.linux.org.uk>,
	linux-cachefs@redhat.com, linux-afs@lists.infradead.org,
	linux-nfs@vger.kernel.org, linux-cifs@vger.kernel.org,
	ceph-devel@vger.kernel.org, v9fs-developer@lists.sourceforge.net,
	linux-fsdevel@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [RFC PATCH 24/76] fscache: Provide a simple thread pool for running ops asynchronously
Date: Fri, 20 Nov 2020 15:07:43 +0000
Message-ID: <160588486318.3465195.8936971598684341062.stgit@warthog.procyon.org.uk> (raw)
In-Reply-To: <160588455242.3465195.3214733858273019178.stgit@warthog.procyon.org.uk>

Provide a simple thread pool that can be used to run cookie management
operations in the background and a dispatcher infrastructure to punt
operations to the pool if threads are available or to just run the
operation in the calling thread if not.

A future patch will replace all the object state machine stuff with whole
routines that do all the work in one go without trying to interleave bits
from various objects.

Signed-off-by: David Howells <dhowells@redhat.com>
---

 fs/fscache/Makefile            |    1 
 fs/fscache/dispatcher.c        |  145 ++++++++++++++++++++++++++++++++++++++++
 fs/fscache/internal.h          |    8 ++
 fs/fscache/main.c              |    7 ++
 include/trace/events/fscache.h |    6 +-
 5 files changed, 166 insertions(+), 1 deletion(-)
 create mode 100644 fs/fscache/dispatcher.c

diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile
index ac3fcd909fff..7b10c6aad157 100644
--- a/fs/fscache/Makefile
+++ b/fs/fscache/Makefile
@@ -6,6 +6,7 @@
 fscache-y := \
 	cache.o \
 	cookie.o \
+	dispatcher.o \
 	fsdef.o \
 	main.o \
 	netfs.o \
diff --git a/fs/fscache/dispatcher.c b/fs/fscache/dispatcher.c
new file mode 100644
index 000000000000..3d957e499da3
--- /dev/null
+++ b/fs/fscache/dispatcher.c
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Object dispatcher
+ *
+ * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ */
+
+#define FSCACHE_DEBUG_LEVEL OPERATION
+#include <linux/kthread.h>
+#include <linux/slab.h>
+#include <linux/wait.h>
+#include <linux/freezer.h>
+#include "internal.h"
+
+#define FSCACHE_DISPATCHER_POOL_SIZE 8
+
+static LIST_HEAD(fscache_pending_work);
+static DEFINE_SPINLOCK(fscache_work_lock);
+static DECLARE_WAIT_QUEUE_HEAD(fscache_dispatcher_pool);
+static struct completion fscache_dispatcher_pool_done[FSCACHE_DISPATCHER_POOL_SIZE];
+static bool fscache_dispatcher_stop;
+
+struct fscache_work {
+	struct list_head	link;
+	struct fscache_cookie	*cookie;
+	struct fscache_object	*object;
+	int			param;
+	void (*func)(struct fscache_cookie *, struct fscache_object *, int);
+};
+
+/*
+ * Attempt to queue some work to do.  If there's too much asynchronous work
+ * already queued, we'll do it here in this thread instead.
+ */
+void fscache_dispatch(struct fscache_cookie *cookie,
+		      struct fscache_object *object,
+		      int param,
+		      void (*func)(struct fscache_cookie *,
+				   struct fscache_object *, int))
+{
+	struct fscache_work *work;
+	bool queued = false;
+
+	work = kzalloc(sizeof(struct fscache_work), GFP_KERNEL);
+	if (work) {
+		work->cookie = cookie;
+		work->object = object;
+		work->param = param;
+		work->func = func;
+
+		spin_lock(&fscache_work_lock);
+		if (waitqueue_active(&fscache_dispatcher_pool) ||
+		    list_empty(&fscache_pending_work)) {
+			fscache_cookie_get(cookie, fscache_cookie_get_work);
+			list_add_tail(&work->link, &fscache_pending_work);
+			wake_up(&fscache_dispatcher_pool);
+			queued = true;
+		}
+		spin_unlock(&fscache_work_lock);
+	}
+
+	if (!queued) {
+		kfree(work);
+		func(cookie, object, param);
+	}
+}
+
+/*
+ * A dispatcher thread.
+ */
+static int fscache_dispatcher(void *data)
+{
+	struct completion *done = data;
+
+	for (;;) {
+		if (!list_empty(&fscache_pending_work)) {
+			struct fscache_work *work = NULL;
+
+			spin_lock(&fscache_work_lock);
+			if (!list_empty(&fscache_pending_work)) {
+				work = list_entry(fscache_pending_work.next,
+						  struct fscache_work, link);
+				list_del_init(&work->link);
+			}
+			spin_unlock(&fscache_work_lock);
+
+			if (work) {
+				work->func(work->cookie, work->object, work->param);
+				fscache_cookie_put(work->cookie, fscache_cookie_put_work);
+				kfree(work);
+			}
+			continue;
+		} else if (fscache_dispatcher_stop) {
+			break;
+		}
+
+		wait_event_freezable(fscache_dispatcher_pool,
+				     (fscache_dispatcher_stop ||
+				      !list_empty(&fscache_pending_work)));
+	}
+
+	complete_and_exit(done, 0);
+}
+
+/*
+ * Start up the dispatcher threads.
+ */
+int fscache_init_dispatchers(void)
+{
+	struct task_struct *t;
+	int i;
+
+	for (i = 0; i < FSCACHE_DISPATCHER_POOL_SIZE; i++) {
+		init_completion(&fscache_dispatcher_pool_done[i]);
+		t = kthread_create(fscache_dispatcher,
+				   &fscache_dispatcher_pool_done[i],
+				   "kfsc/%d", i);
+		if (IS_ERR(t))
+			goto failed;
+		wake_up_process(t);
+	}
+
+	return 0;
+
+failed:
+	fscache_dispatcher_stop = true;
+	wake_up_all(&fscache_dispatcher_pool);
+	for (i--; i >= 0; i--)
+		wait_for_completion(&fscache_dispatcher_pool_done[i]);
+	return PTR_ERR(t);
+}
+
+/*
+ * Kill off the dispatcher threads.
+ */
+void fscache_kill_dispatchers(void)
+{
+	int i;
+
+	fscache_dispatcher_stop = true;
+	wake_up_all(&fscache_dispatcher_pool);
+
+	for (i = 0; i < FSCACHE_DISPATCHER_POOL_SIZE; i++)
+		wait_for_completion(&fscache_dispatcher_pool_done[i]);
+}
diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h
index bc5539d2157b..2100e2222884 100644
--- a/fs/fscache/internal.h
+++ b/fs/fscache/internal.h
@@ -75,6 +75,14 @@ extern struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *);
 extern void fscache_cookie_put(struct fscache_cookie *,
 			       enum fscache_cookie_trace);
 
+/*
+ * dispatcher.c
+ */
+extern void fscache_dispatch(struct fscache_cookie *, struct fscache_object *, int,
+			     void (*func)(struct fscache_cookie *, struct fscache_object *, int));
+extern int fscache_init_dispatchers(void);
+extern void fscache_kill_dispatchers(void);
+
 /*
  * fsdef.c
  */
diff --git a/fs/fscache/main.c b/fs/fscache/main.c
index c1e6cc9091aa..c8f1beafa8e1 100644
--- a/fs/fscache/main.c
+++ b/fs/fscache/main.c
@@ -125,6 +125,10 @@ static int __init fscache_init(void)
 	for_each_possible_cpu(cpu)
 		init_waitqueue_head(&per_cpu(fscache_object_cong_wait, cpu));
 
+	ret = fscache_init_dispatchers();
+	if (ret < 0)
+		goto error_dispatchers;
+
 	ret = fscache_proc_init();
 	if (ret < 0)
 		goto error_proc;
@@ -159,6 +163,8 @@ static int __init fscache_init(void)
 	unregister_sysctl_table(fscache_sysctl_header);
 error_sysctl:
 #endif
+	fscache_kill_dispatchers();
+error_dispatchers:
 	fscache_proc_cleanup();
 error_proc:
 	destroy_workqueue(fscache_op_wq);
@@ -183,6 +189,7 @@ static void __exit fscache_exit(void)
 	unregister_sysctl_table(fscache_sysctl_header);
 #endif
 	fscache_proc_cleanup();
+	fscache_kill_dispatchers();
 	destroy_workqueue(fscache_op_wq);
 	destroy_workqueue(fscache_object_wq);
 	pr_notice("Unloaded\n");
diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h
index 08d7de72409d..fb3fdf2921ee 100644
--- a/include/trace/events/fscache.h
+++ b/include/trace/events/fscache.h
@@ -26,11 +26,13 @@ enum fscache_cookie_trace {
 	fscache_cookie_get_attach_object,
 	fscache_cookie_get_reacquire,
 	fscache_cookie_get_register_netfs,
+	fscache_cookie_get_work,
 	fscache_cookie_put_acquire_nobufs,
 	fscache_cookie_put_dup_netfs,
 	fscache_cookie_put_relinquish,
 	fscache_cookie_put_object,
 	fscache_cookie_put_parent,
+	fscache_cookie_put_work,
 };
 
 #endif
@@ -45,11 +47,13 @@ enum fscache_cookie_trace {
 	EM(fscache_cookie_get_attach_object,	"GET obj")		\
 	EM(fscache_cookie_get_reacquire,	"GET raq")		\
 	EM(fscache_cookie_get_register_netfs,	"GET net")		\
+	EM(fscache_cookie_get_work,		"GET wrk")		\
 	EM(fscache_cookie_put_acquire_nobufs,	"PUT nbf")		\
 	EM(fscache_cookie_put_dup_netfs,	"PUT dnt")		\
 	EM(fscache_cookie_put_relinquish,	"PUT rlq")		\
 	EM(fscache_cookie_put_object,		"PUT obj")		\
-	E_(fscache_cookie_put_parent,		"PUT prn")
+	EM(fscache_cookie_put_parent,		"PUT prn")		\
+	E_(fscache_cookie_put_work,		"PUT wrk")
 
 /*
  * Export enum symbols via userspace.



  parent reply index

Thread overview: 76+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-11-20 15:02 [RFC PATCH 00/76] fscache: Modernisation David Howells
2020-11-20 15:02 ` [RFC PATCH 01/76] nfs, cifs, ceph, 9p: Disable use of fscache prior to its rewrite David Howells
2020-11-20 15:02 ` [RFC PATCH 02/76] afs: Disable use of the fscache I/O routines David Howells
2020-11-20 15:03 ` [RFC PATCH 03/76] fscache: Add a cookie debug ID and use that in traces David Howells
2020-11-20 15:03 ` [RFC PATCH 04/76] fscache: Procfile to display cookies David Howells
2020-11-20 15:03 ` [RFC PATCH 05/76] fscache: Remove the old I/O API David Howells
2020-11-20 15:03 ` [RFC PATCH 06/76] fscache: Remove the netfs data from the cookie David Howells
2020-11-20 15:04 ` [RFC PATCH 07/76] fscache: Remove struct fscache_cookie_def David Howells
2020-11-20 15:04 ` [RFC PATCH 08/76] fscache: Remove store_limit* from struct fscache_object David Howells
2020-11-20 15:04 ` [RFC PATCH 09/76] fscache: Remove fscache_check_consistency() David Howells
2020-11-20 15:04 ` [RFC PATCH 10/76] fscache: Remove fscache_attr_changed() David Howells
2020-11-20 15:04 ` [RFC PATCH 11/76] fscache: Remove obsolete stats David Howells
2020-11-20 15:05 ` [RFC PATCH 12/76] fscache: Remove old I/O tracepoints David Howells
2020-11-20 15:05 ` [RFC PATCH 13/76] fscache: Temporarily disable fscache_invalidate() David Howells
2020-11-20 15:05 ` [RFC PATCH 14/76] fscache: Remove the I/O operation manager David Howells
2020-11-20 15:05 ` [RFC PATCH 15/76] fscache: Change %p in format strings to something else David Howells
2020-11-20 15:06 ` [RFC PATCH 16/76] cachefiles: " David Howells
2020-11-20 15:06 ` [RFC PATCH 17/76] iov_iter: Add ITER_XARRAY David Howells
2020-11-20 15:06 ` [RFC PATCH 18/76] vm: Add wait/unlock functions for PG_fscache David Howells
2020-11-20 15:06 ` [RFC PATCH 19/76] mm: Implement readahead_control pageset expansion David Howells
2020-11-20 15:06 ` [RFC PATCH 20/76] mm: Stop generic_file_buffered_read() from grabbing a superfluous page David Howells
2020-11-20 15:07 ` [RFC PATCH 21/76] vfs: Export rw_verify_area() for use by cachefiles David Howells
2020-11-20 15:07 ` [RFC PATCH 22/76] vfs: Provide S_CACHE_FILE inode flag David Howells
2020-11-20 15:07 ` [RFC PATCH 23/76] cachefiles: Remove tree of active files and use " David Howells
2020-11-20 15:07 ` David Howells [this message]
2020-11-20 15:08 ` [RFC PATCH 26/76] fscache: Rewrite the I/O API based on iov_iter David Howells
2020-11-20 15:08 ` [RFC PATCH 27/76] fscache: Keep track of size of a file last set independently on the server David Howells
2020-11-20 15:08 ` [RFC PATCH 28/76] fscache, cachefiles: Fix disabled histogram warnings David Howells
2020-11-20 15:08 ` [RFC PATCH 29/76] fscache: Recast assertion in terms of cookie not being an index David Howells
2020-11-20 15:08 ` [RFC PATCH 30/76] vfs, fscache: Force ->write_inode() to occur if cookie pinned for writeback David Howells
2020-11-20 15:09 ` [RFC PATCH 31/76] fscache: Allow ->put_super() to be used to wait for cache operations David Howells
2020-11-20 15:09 ` [RFC PATCH 32/76] netfs: Make a netfs helper module David Howells
2020-11-20 15:09 ` [RFC PATCH 33/76] netfs: Provide readahead and readpage netfs helpers David Howells
2020-11-20 15:09 ` [RFC PATCH 34/76] netfs: Use the cache David Howells
2020-11-20 15:10 ` [RFC PATCH 35/76] fscache: read-helper: Add tracepoints David Howells
2020-11-20 15:10 ` [RFC PATCH 36/76] cachefiles: Remove some redundant checks on unsigned values David Howells
2020-11-20 15:10 ` [RFC PATCH 37/76] cachefiles: trace: Log coherency checks David Howells
2020-11-20 15:10 ` [RFC PATCH 38/76] cachefiles: Split cachefiles_drop_object() up a bit David Howells
2020-11-20 15:11 ` [RFC PATCH 39/76] cachefiles: Implement new fscache I/O backend API David Howells
2020-11-20 15:11 ` [RFC PATCH 40/76] cachefiles: Merge object->backer into object->dentry David Howells
2020-11-20 15:11 ` [RFC PATCH 41/76] cachefiles: Implement a content-present indicator and bitmap David Howells
2020-11-20 15:11 ` [RFC PATCH 42/76] cachefiles: Shape requests from the fscache read helper David Howells
2020-11-20 15:11 ` [RFC PATCH 43/76] cachefiles: Round the cachefile size up to DIO block size David Howells
2020-11-20 15:12 ` [RFC PATCH 44/76] cachefiles: Implement read and write parts of new I/O API David Howells
2020-11-20 15:12 ` [RFC PATCH 45/76] cachefiles: Add I/O tracepoints David Howells
2020-11-20 15:12 ` [RFC PATCH 46/76] fscache: Display cache-specific data in /proc/fs/fscache/objects David Howells
2020-11-20 15:12 ` [RFC PATCH 47/76] fscache: Remove more obsolete stats David Howells
2020-11-20 15:12 ` [RFC PATCH 48/76] fscache: Always create /proc/fs/fscache/stats if configured David Howells
2020-11-20 15:13 ` [RFC PATCH 49/76] netfs: Stats David Howells
2020-11-20 15:13 ` [RFC PATCH 50/76] fscache: New stats David Howells
2020-11-20 15:13 ` [RFC PATCH 51/76] fscache, cachefiles: Rewrite invalidation David Howells
2020-11-20 15:13 ` [RFC PATCH 52/76] fscache: disable cookie when doing an invalidation for DIO write David Howells
2020-11-20 15:13 ` [RFC PATCH 53/76] fscache: Implement "will_modify" parameter on fscache_use_cookie() David Howells
2020-11-20 15:14 ` [RFC PATCH 54/76] fscache: Provide resize operation David Howells
2020-11-20 15:14 ` [RFC PATCH 55/76] fscache: Remove the update operation David Howells
2020-11-20 15:14 ` [RFC PATCH 56/76] afs: Pass page into dirty region helpers to provide THP size David Howells
2020-11-20 15:14 ` [RFC PATCH 57/76] afs: Print the operation debug_id when logging an unexpected data version David Howells
2020-11-20 15:14 ` [RFC PATCH 58/76] afs: Move key to afs_read struct David Howells
2020-11-20 15:15 ` [RFC PATCH 59/76] afs: Don't truncate iter during data fetch David Howells
2020-11-20 15:15 ` [RFC PATCH 60/76] afs: Log remote unmarshalling errors David Howells
2020-11-20 15:15 ` [RFC PATCH 61/76] afs: Set up the iov_iter before calling afs_extract_data() David Howells
2020-11-20 15:15 ` [RFC PATCH 62/76] afs: Use ITER_XARRAY for writing David Howells
2020-11-20 15:16 ` [RFC PATCH 63/76] afs: Wait on PG_fscache before modifying/releasing a page David Howells
2020-11-20 15:16 ` [RFC PATCH 64/76] afs: Extract writeback extension into its own function David Howells
2020-11-20 15:16 ` [RFC PATCH 65/76] afs: Prepare for use of THPs David Howells
2020-11-20 15:16 ` [RFC PATCH 66/76] afs: Use the fs operation ops to handle FetchData completion David Howells
2020-11-20 15:17 ` [RFC PATCH 67/76] afs: Use new fscache read helper API David Howells
2020-11-20 15:17 ` [RFC PATCH 68/76] netfs: Add write_begin helper David Howells
2020-11-20 15:17 ` [RFC PATCH 69/76] fscache: Add support for writing to the cache David Howells
2020-11-20 15:17 ` [RFC PATCH 70/76] afs: Use the fscache_write_begin() helper David Howells
2020-11-20 15:18 ` [RFC PATCH 71/76] afs: Copy local writes to the cache when writing to the server David Howells
2020-11-20 15:18 ` [RFC PATCH 72/76] afs: Invoke fscache_resize_cookie() when handling ATTR_SIZE for setattr David Howells
2020-11-20 15:18 ` [RFC PATCH 73/76] afs: Add O_DIRECT read support David Howells
2020-11-20 15:18 ` [RFC PATCH 74/76] afs: Skip truncation on the server of data we haven't written yet David Howells
2020-11-20 15:19 ` [RFC PATCH 75/76] afs: Make afs_write_begin() return the THP subpage David Howells
2020-11-20 15:19 ` [RFC PATCH 76/76] afs: Fix speculative status fetch going out of order wrt to modifications David Howells

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=160588486318.3465195.8936971598684341062.stgit@warthog.procyon.org.uk \
    --to=dhowells@redhat.com \
    --cc=anna.schumaker@netapp.com \
    --cc=asmadeus@codewreck.org \
    --cc=ceph-devel@vger.kernel.org \
    --cc=jlayton@redhat.com \
    --cc=linux-afs@lists.infradead.org \
    --cc=linux-cachefs@redhat.com \
    --cc=linux-cifs@vger.kernel.org \
    --cc=linux-fsdevel@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-nfs@vger.kernel.org \
    --cc=sfrench@samba.org \
    --cc=trondmy@hammerspace.com \
    --cc=v9fs-developer@lists.sourceforge.net \
    --cc=viro@zeniv.linux.org.uk \
    --cc=willy@infradead.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

CEPH-Devel Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/ceph-devel/0 ceph-devel/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 ceph-devel ceph-devel/ https://lore.kernel.org/ceph-devel \
		ceph-devel@vger.kernel.org
	public-inbox-index ceph-devel

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.ceph-devel


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git