From: "Paul E. McKenney" <paulmck@kernel.org>
To: rcu@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com,
rostedt@goodmis.org, hch@lst.de,
"Paul E. McKenney" <paulmck@kernel.org>,
"Rafael J. Wysocki" <rafael.j.wysocki@intel.com>,
"Michał Mirosław" <mirq-linux@rere.qmqm.pl>,
"Dmitry Osipenko" <dmitry.osipenko@collabora.com>
Subject: [PATCH RFC rcu 03/19] srcu: Begin offloading srcu_struct fields to srcu_update
Date: Thu, 23 Mar 2023 17:19:22 -0700 [thread overview]
Message-ID: <20230324001938.3443499-3-paulmck@kernel.org> (raw)
In-Reply-To: <3db82572-f156-4a5d-b711-841aa28bd996@paulmck-laptop>
The current srcu_struct structure is on the order of 200 bytes in size
(depending on architecture and .config), which is much better than the
old-style 26K bytes, but still all too inconvenient when one is trying
to achieve good cache locality on a fastpath involving SRCU readers.
However, only a few fields in srcu_struct are used by SRCU readers.
The remaining fields could be offloaded to a new srcu_update
structure, thus shrinking the srcu_struct structure down to a few
tens of bytes. This commit begins this noble quest, a quest that is
complicated by open-coded initialization of the srcu_struct within the
srcu_notifier_head structure. This complication is addressed by updating
the srcu_notifier_head structure's open coding, given that there does
not appear to be a straightforward way of abstracting that initialization.
This commit moves only the ->node pointer to srcu_update. Later commits
will move additional fields.
[ paulmck: Fold in qiang1.zhang@intel.com's memory-leak fix. ]
Link: https://lore.kernel.org/all/20230320055751.4120251-1-qiang1.zhang@intel.com/
Suggested-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
Cc: "Michał Mirosław" <mirq-linux@rere.qmqm.pl>
Cc: Dmitry Osipenko <dmitry.osipenko@collabora.com>
---
include/linux/notifier.h | 5 ++++-
include/linux/srcutiny.h | 6 +++---
include/linux/srcutree.h | 27 ++++++++++++++++++---------
kernel/rcu/rcu.h | 6 ++++--
kernel/rcu/srcutree.c | 28 +++++++++++++++++++---------
5 files changed, 48 insertions(+), 24 deletions(-)
diff --git a/include/linux/notifier.h b/include/linux/notifier.h
index aef88c2d1173..2aba75145144 100644
--- a/include/linux/notifier.h
+++ b/include/linux/notifier.h
@@ -73,6 +73,9 @@ struct raw_notifier_head {
struct srcu_notifier_head {
struct mutex mutex;
+#ifdef CONFIG_TREE_SRCU
+ struct srcu_usage srcuu;
+#endif
struct srcu_struct srcu;
struct notifier_block __rcu *head;
};
@@ -107,7 +110,7 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
{ \
.mutex = __MUTEX_INITIALIZER(name.mutex), \
.head = NULL, \
- .srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \
+ .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \
}
#define ATOMIC_NOTIFIER_HEAD(name) \
diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 5aa5e0faf6a1..ebd72491af99 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -31,7 +31,7 @@ struct srcu_struct {
void srcu_drive_gp(struct work_struct *wp);
-#define __SRCU_STRUCT_INIT(name, __ignored) \
+#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored) \
{ \
.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \
.srcu_cb_tail = &name.srcu_cb_head, \
@@ -44,9 +44,9 @@ void srcu_drive_gp(struct work_struct *wp);
* Tree SRCU, which needs some per-CPU data.
*/
#define DEFINE_SRCU(name) \
- struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)
+ struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)
#define DEFINE_STATIC_SRCU(name) \
- static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)
+ static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)
void synchronize_srcu(struct srcu_struct *ssp);
diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 428480152375..2689e64024bb 100644
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -57,11 +57,17 @@ struct srcu_node {
int grphi; /* Biggest CPU for node. */
};
+/*
+ * Per-SRCU-domain structure, update-side data linked from srcu_struct.
+ */
+struct srcu_usage {
+ struct srcu_node *node; /* Combining tree. */
+};
+
/*
* Per-SRCU-domain structure, similar in function to rcu_state.
*/
struct srcu_struct {
- struct srcu_node *node; /* Combining tree. */
struct srcu_node *level[RCU_NUM_LVLS + 1];
/* First node at each level. */
int srcu_size_state; /* Small-to-big transition state. */
@@ -90,6 +96,7 @@ struct srcu_struct {
unsigned long reschedule_count;
struct delayed_work work;
struct lockdep_map dep_map;
+ struct srcu_usage *srcu_sup; /* Update-side data. */
};
// Values for size state variable (->srcu_size_state). Once the state
@@ -121,24 +128,24 @@ struct srcu_struct {
#define SRCU_STATE_SCAN1 1
#define SRCU_STATE_SCAN2 2
-#define __SRCU_STRUCT_INIT_COMMON(name) \
+#define __SRCU_STRUCT_INIT_COMMON(name, usage_name) \
.lock = __SPIN_LOCK_UNLOCKED(name.lock), \
.srcu_gp_seq_needed = -1UL, \
.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \
+ .srcu_sup = &usage_name, \
__SRCU_DEP_MAP_INIT(name)
-#define __SRCU_STRUCT_INIT_MODULE(name) \
+#define __SRCU_STRUCT_INIT_MODULE(name, usage_name) \
{ \
- __SRCU_STRUCT_INIT_COMMON(name) \
+ __SRCU_STRUCT_INIT_COMMON(name, usage_name) \
}
-#define __SRCU_STRUCT_INIT(name, pcpu_name) \
+#define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name) \
{ \
.sda = &pcpu_name, \
- __SRCU_STRUCT_INIT_COMMON(name) \
+ __SRCU_STRUCT_INIT_COMMON(name, usage_name) \
}
-
/*
* Define and initialize a srcu struct at build time.
* Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it.
@@ -160,15 +167,17 @@ struct srcu_struct {
*/
#ifdef MODULE
# define __DEFINE_SRCU(name, is_static) \
- is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name); \
+ static struct srcu_usage name##_srcu_usage; \
+ is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage); \
extern struct srcu_struct * const __srcu_struct_##name; \
struct srcu_struct * const __srcu_struct_##name \
__section("___srcu_struct_ptrs") = &name
#else
# define __DEFINE_SRCU(name, is_static) \
static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data); \
+ static struct srcu_usage name##_srcu_usage; \
is_static struct srcu_struct name = \
- __SRCU_STRUCT_INIT(name, name##_srcu_data)
+ __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data)
#endif
#define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index a3adcf9a9919..4a1b9622598b 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -378,11 +378,13 @@ extern void rcu_init_geometry(void);
* specified state structure (for SRCU) or the only rcu_state structure
* (for RCU).
*/
-#define srcu_for_each_node_breadth_first(sp, rnp) \
+#define _rcu_for_each_node_breadth_first(sp, rnp) \
for ((rnp) = &(sp)->node[0]; \
(rnp) < &(sp)->node[rcu_num_nodes]; (rnp)++)
#define rcu_for_each_node_breadth_first(rnp) \
- srcu_for_each_node_breadth_first(&rcu_state, rnp)
+ _rcu_for_each_node_breadth_first(&rcu_state, rnp)
+#define srcu_for_each_node_breadth_first(ssp, rnp) \
+ _rcu_for_each_node_breadth_first(ssp->srcu_sup, rnp)
/*
* Scan the leaves of the rcu_node hierarchy for the rcu_state structure.
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 7a6d9452a5d0..ad1d5ca42a99 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -173,12 +173,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
/* Initialize geometry if it has not already been initialized. */
rcu_init_geometry();
- ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), gfp_flags);
- if (!ssp->node)
+ ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags);
+ if (!ssp->srcu_sup->node)
return false;
/* Work out the overall tree geometry. */
- ssp->level[0] = &ssp->node[0];
+ ssp->level[0] = &ssp->srcu_sup->node[0];
for (i = 1; i < rcu_num_lvls; i++)
ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1];
rcu_init_levelspread(levelspread, num_rcu_lvl);
@@ -195,7 +195,7 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ;
snp->grplo = -1;
snp->grphi = -1;
- if (snp == &ssp->node[0]) {
+ if (snp == &ssp->srcu_sup->node[0]) {
/* Root node, special case. */
snp->srcu_parent = NULL;
continue;
@@ -236,8 +236,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
*/
static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
{
+ if (!is_static)
+ ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);
+ if (!ssp->srcu_sup)
+ return -ENOMEM;
ssp->srcu_size_state = SRCU_SIZE_SMALL;
- ssp->node = NULL;
+ ssp->srcu_sup->node = NULL;
mutex_init(&ssp->srcu_cb_mutex);
mutex_init(&ssp->srcu_gp_mutex);
ssp->srcu_idx = 0;
@@ -249,8 +253,11 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
ssp->sda_is_static = is_static;
if (!is_static)
ssp->sda = alloc_percpu(struct srcu_data);
- if (!ssp->sda)
+ if (!ssp->sda) {
+ if (!is_static)
+ kfree(ssp->srcu_sup);
return -ENOMEM;
+ }
init_srcu_struct_data(ssp);
ssp->srcu_gp_seq_needed_exp = 0;
ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();
@@ -259,6 +266,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
if (!ssp->sda_is_static) {
free_percpu(ssp->sda);
ssp->sda = NULL;
+ kfree(ssp->srcu_sup);
return -ENOMEM;
}
} else {
@@ -656,13 +664,15 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
rcu_seq_current(&ssp->srcu_gp_seq), ssp->srcu_gp_seq_needed);
return; /* Caller forgot to stop doing call_srcu()? */
}
+ kfree(ssp->srcu_sup->node);
+ ssp->srcu_sup->node = NULL;
+ ssp->srcu_size_state = SRCU_SIZE_SMALL;
if (!ssp->sda_is_static) {
free_percpu(ssp->sda);
ssp->sda = NULL;
+ kfree(ssp->srcu_sup);
+ ssp->srcu_sup = NULL;
}
- kfree(ssp->node);
- ssp->node = NULL;
- ssp->srcu_size_state = SRCU_SIZE_SMALL;
}
EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
--
2.40.0.rc2
next prev parent reply other threads:[~2023-03-24 0:19 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-03-24 0:19 [PATCH RFC rcu 0/19] Further shrink srcu_struct to promote cache locality Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 01/19] srcu: Add whitespace to __SRCU_STRUCT_INIT() & __DEFINE_SRCU() Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 02/19] srcu: Use static init for statically allocated in-module srcu_struct Paul E. McKenney
2023-03-30 4:11 ` Zhang, Qiang1
2023-03-30 14:55 ` Paul E. McKenney
2023-03-30 15:03 ` Zhang, Qiang1
2023-03-30 15:20 ` Zhang, Qiang1
2023-03-30 17:05 ` Paul E. McKenney
2023-03-24 0:19 ` Paul E. McKenney [this message]
2023-03-24 19:10 ` [PATCH RFC rcu 03/19] srcu: Begin offloading srcu_struct fields to srcu_update Wysocki, Rafael J
2023-03-24 20:11 ` Paul E. McKenney
2023-03-26 23:18 ` Christoph Hellwig
2023-03-27 3:29 ` Paul E. McKenney
2023-04-24 21:06 ` Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 04/19] srcu: Move ->level from srcu_struct to srcu_usage Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 05/19] srcu: Move ->srcu_size_state " Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 06/19] srcu: Move ->srcu_cb_mutex " Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 07/19] srcu: Move ->lock initialization after srcu_usage allocation Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 08/19] srcu: Move ->lock from srcu_struct to srcu_usage Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 09/19] srcu: Move ->srcu_gp_mutex " Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 10/19] srcu: Move grace-period fields " Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 11/19] srcu: Move heuristics " Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 12/19] srcu: Move ->sda_is_static " Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 13/19] srcu: Move srcu_barrier() fields " Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 14/19] srcu: Move work-scheduling " Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 15/19] srcu: Fix long lines in srcu_get_delay() Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 16/19] srcu: Fix long lines in cleanup_srcu_struct() Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 17/19] srcu: Fix long lines in srcu_gp_end() Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 18/19] srcu: Fix long lines in srcu_funnel_gp_start() Paul E. McKenney
2023-03-24 0:19 ` [PATCH RFC rcu 19/19] srcu: Remove extraneous parentheses from srcu_read_lock() etc Paul E. McKenney
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230324001938.3443499-3-paulmck@kernel.org \
--to=paulmck@kernel.org \
--cc=dmitry.osipenko@collabora.com \
--cc=hch@lst.de \
--cc=kernel-team@meta.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mirq-linux@rere.qmqm.pl \
--cc=rafael.j.wysocki@intel.com \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).