From: Dave Chinner <david@fromorbit.com>
To: linux-xfs@vger.kernel.org
Subject: [PATCH 04/12] xfs: introduce per-cpu CIL tracking sructure
Date: Thu, 25 Feb 2021 14:37:17 +1100 [thread overview]
Message-ID: <20210225033725.3558450-5-david@fromorbit.com> (raw)
In-Reply-To: <20210225033725.3558450-1-david@fromorbit.com>
From: Dave Chinner <dchinner@redhat.com>
The CIL push lock is highly contended on larger machines, becoming a
hard bottleneck that about 700,000 transaction commits/s on >16p
machines. To address this, start moving the CIL tracking
infrastructure to utilise per-CPU structures.
We need to track the space used, the amount of log reservation space
reserved to write the CIL, the log items in the CIL and the busy
extents that need to be completed by the CIL commit. This requires
a couple of per-cpu counters, an unordered per-cpu list and a
globally ordered per-cpu list.
Create a per-cpu structure to hold these and all the management
interfaces needed, as well as the hooks to handle hotplug CPUs.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
---
fs/xfs/xfs_log_cil.c | 94 ++++++++++++++++++++++++++++++++++++++
fs/xfs/xfs_log_priv.h | 15 ++++++
include/linux/cpuhotplug.h | 1 +
3 files changed, 110 insertions(+)
diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c
index ecd2f085e572..06aca398f560 100644
--- a/fs/xfs/xfs_log_cil.c
+++ b/fs/xfs/xfs_log_cil.c
@@ -1360,6 +1360,93 @@ xfs_log_item_in_current_chkpt(
return true;
}
+#ifdef CONFIG_HOTPLUG_CPU
+static LIST_HEAD(xlog_cil_pcp_list);
+static DEFINE_SPINLOCK(xlog_cil_pcp_lock);
+static bool xlog_cil_pcp_init;
+
+static int
+xlog_cil_pcp_dead(
+ unsigned int cpu)
+{
+ struct xfs_cil *cil;
+
+ spin_lock(&xlog_cil_pcp_lock);
+ list_for_each_entry(cil, &xlog_cil_pcp_list, xc_pcp_list) {
+ /* move stuff on dead CPU to context */
+ }
+ spin_unlock(&xlog_cil_pcp_lock);
+ return 0;
+}
+
+static int
+xlog_cil_pcp_hpadd(
+ struct xfs_cil *cil)
+{
+ if (!xlog_cil_pcp_init) {
+ int ret;
+ ret = cpuhp_setup_state_nocalls(CPUHP_XFS_CIL_DEAD,
+ "xfs/cil_pcp:dead", NULL,
+ xlog_cil_pcp_dead);
+ if (ret < 0) {
+ xfs_warn(cil->xc_log->l_mp,
+ "Failed to initialise CIL hotplug, error %d. XFS is non-functional.",
+ ret);
+ ASSERT(0);
+ return -ENOMEM;
+ }
+ xlog_cil_pcp_init = true;
+ }
+
+ INIT_LIST_HEAD(&cil->xc_pcp_list);
+ spin_lock(&xlog_cil_pcp_lock);
+ list_add(&cil->xc_pcp_list, &xlog_cil_pcp_list);
+ spin_unlock(&xlog_cil_pcp_lock);
+ return 0;
+}
+
+static void
+xlog_cil_pcp_hpremove(
+ struct xfs_cil *cil)
+{
+ spin_lock(&xlog_cil_pcp_lock);
+ list_del(&cil->xc_pcp_list);
+ spin_unlock(&xlog_cil_pcp_lock);
+}
+
+#else /* !CONFIG_HOTPLUG_CPU */
+static inline void xlog_cil_pcp_hpadd(struct xfs_cil *cil) {}
+static inline void xlog_cil_pcp_hpremove(struct xfs_cil *cil) {}
+#endif
+
+static void __percpu *
+xlog_cil_pcp_alloc(
+ struct xfs_cil *cil)
+{
+ struct xlog_cil_pcp *cilpcp;
+
+ cilpcp = alloc_percpu(struct xlog_cil_pcp);
+ if (!cilpcp)
+ return NULL;
+
+ if (xlog_cil_pcp_hpadd(cil) < 0) {
+ free_percpu(cilpcp);
+ return NULL;
+ }
+ return cilpcp;
+}
+
+static void
+xlog_cil_pcp_free(
+ struct xfs_cil *cil,
+ struct xlog_cil_pcp *cilpcp)
+{
+ if (!cilpcp)
+ return;
+ xlog_cil_pcp_hpremove(cil);
+ free_percpu(cilpcp);
+}
+
/*
* Perform initial CIL structure initialisation.
*/
@@ -1374,6 +1461,12 @@ xlog_cil_init(
if (!cil)
return -ENOMEM;
+ cil->xc_pcp = xlog_cil_pcp_alloc(cil);
+ if (!cil->xc_pcp) {
+ kmem_free(cil);
+ return -ENOMEM;
+ }
+
INIT_LIST_HEAD(&cil->xc_cil);
INIT_LIST_HEAD(&cil->xc_committing);
spin_lock_init(&cil->xc_cil_lock);
@@ -1404,6 +1497,7 @@ xlog_cil_destroy(
ASSERT(list_empty(&cil->xc_cil));
ASSERT(test_bit(XLOG_CIL_EMPTY, &cil->xc_flags));
+ xlog_cil_pcp_free(cil, cil->xc_pcp);
kmem_free(cil);
}
diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h
index 7852a60e8f86..fbf2a665c62a 100644
--- a/fs/xfs/xfs_log_priv.h
+++ b/fs/xfs/xfs_log_priv.h
@@ -231,6 +231,16 @@ struct xfs_cil_ctx {
struct work_struct push_work;
};
+/*
+ * Per-cpu CIL tracking items
+ */
+struct xlog_cil_pcp {
+ uint32_t space_used;
+ uint32_t curr_res;
+ struct list_head busy_extents;
+ struct list_head log_items;
+};
+
/*
* Committed Item List structure
*
@@ -264,6 +274,11 @@ struct xfs_cil {
wait_queue_head_t xc_commit_wait;
uint64_t xc_current_sequence;
wait_queue_head_t xc_push_wait; /* background push throttle */
+
+ struct xlog_cil_pcp __percpu *xc_pcp;
+#ifdef CONFIG_HOTPLUG_CPU
+ struct list_head xc_pcp_list;
+#endif
} ____cacheline_aligned_in_smp;
/* xc_flags bit values */
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 0042ef362511..22813626787f 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -52,6 +52,7 @@ enum cpuhp_state {
CPUHP_FS_BUFF_DEAD,
CPUHP_PRINTK_DEAD,
CPUHP_MM_MEMCQ_DEAD,
+ CPUHP_XFS_CIL_DEAD,
CPUHP_PERCPU_CNT_DEAD,
CPUHP_RADIX_DEAD,
CPUHP_PAGE_ALLOC_DEAD,
--
2.28.0
next prev parent reply other threads:[~2021-02-25 3:38 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-02-25 3:37 [PATCH 00/12] xfs: Improve CIL scalability Dave Chinner
2021-02-25 3:37 ` [PATCH 01/12] xfs: use the CIL space used counter for emptiness checks Dave Chinner
2021-02-25 3:37 ` [PATCH 02/12] xfs: lift init CIL reservation out of xc_cil_lock Dave Chinner
2021-02-25 3:37 ` [PATCH 03/12] xfs: rework per-iclog header CIL reservation Dave Chinner
2021-02-25 3:37 ` Dave Chinner [this message]
2021-02-25 3:37 ` [PATCH 05/12] xfs: implement percpu cil space used calculation Dave Chinner
2021-02-25 3:37 ` [PATCH 06/12] xfs: track CIL ticket reservation in percpu structure Dave Chinner
2021-02-25 3:37 ` [PATCH 07/12] xfs: convert CIL busy extents to per-cpu Dave Chinner
2021-02-25 3:37 ` [PATCH 08/12] xfs: Add order IDs to log items in CIL Dave Chinner
2021-02-25 3:37 ` [PATCH 09/12] xfs: convert CIL to unordered per cpu lists Dave Chinner
2021-02-25 3:37 ` [PATCH 10/12] xfs: move CIL ordering to the logvec chain Dave Chinner
2021-02-25 3:37 ` [PATCH 11/12] xfs: __percpu_counter_compare() inode count debug too expensive Dave Chinner
2021-02-25 3:37 ` [PATCH 12/12] xfs: avoid cil push lock if possible Dave Chinner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210225033725.3558450-5-david@fromorbit.com \
--to=david@fromorbit.com \
--cc=linux-xfs@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.