From: Dennis Zhou <dennis@kernel.org>
To: Jens Axboe <axboe@kernel.dk>, Tejun Heo <tj@kernel.org>,
Johannes Weiner <hannes@cmpxchg.org>,
Josef Bacik <josef@toxicpanda.com>
Cc: kernel-team@fb.com, linux-block@vger.kernel.org,
cgroups@vger.kernel.org, linux-kernel@vger.kernel.org,
Dennis Zhou <dennis@kernel.org>
Subject: [PATCH 13/14] blkcg: change blkg reference counting to use percpu_ref
Date: Tue, 4 Dec 2018 13:35:59 -0500 [thread overview]
Message-ID: <20181204183600.99746-14-dennis@kernel.org> (raw)
In-Reply-To: <20181204183600.99746-1-dennis@kernel.org>
Every bio is now associated with a blkg putting blkg_get, blkg_try_get,
and blkg_put on the hot path. Switch over the refcnt in blkg to use
percpu_ref.
Signed-off-by: Dennis Zhou <dennis@kernel.org>
Acked-by: Tejun Heo <tj@kernel.org>
---
block/blk-cgroup.c | 41 ++++++++++++++++++++++++++++++++++++--
include/linux/blk-cgroup.h | 15 +++++---------
2 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 120f2e2835fb..2ca7611fe274 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -81,6 +81,37 @@ static void blkg_free(struct blkcg_gq *blkg)
kfree(blkg);
}
+static void __blkg_release(struct rcu_head *rcu)
+{
+ struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
+
+ percpu_ref_exit(&blkg->refcnt);
+
+ /* release the blkcg and parent blkg refs this blkg has been holding */
+ css_put(&blkg->blkcg->css);
+ if (blkg->parent)
+ blkg_put(blkg->parent);
+
+ wb_congested_put(blkg->wb_congested);
+
+ blkg_free(blkg);
+}
+
+/*
+ * A group is RCU protected, but having an rcu lock does not mean that one
+ * can access all the fields of blkg and assume these are valid. For
+ * example, don't try to follow throtl_data and request queue links.
+ *
+ * Having a reference to blkg under an rcu allows accesses to only values
+ * local to groups like group stats and group rate limits.
+ */
+static void blkg_release(struct percpu_ref *ref)
+{
+ struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
+
+ call_rcu(&blkg->rcu_head, __blkg_release);
+}
+
/**
* blkg_alloc - allocate a blkg
* @blkcg: block cgroup the new blkg is associated with
@@ -107,7 +138,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
blkg->q = q;
INIT_LIST_HEAD(&blkg->q_node);
blkg->blkcg = blkcg;
- atomic_set(&blkg->refcnt, 1);
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
@@ -207,6 +237,11 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
blkg_get(blkg->parent);
}
+ ret = percpu_ref_init(&blkg->refcnt, blkg_release, 0,
+ GFP_NOWAIT | __GFP_NOWARN);
+ if (ret)
+ goto err_cancel_ref;
+
/* invoke per-policy init */
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
@@ -239,6 +274,8 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
blkg_put(blkg);
return ERR_PTR(ret);
+err_cancel_ref:
+ percpu_ref_exit(&blkg->refcnt);
err_put_congested:
wb_congested_put(wb_congested);
err_put_css:
@@ -367,7 +404,7 @@ static void blkg_destroy(struct blkcg_gq *blkg)
* Put the reference taken at the time of creation so that when all
* queues are gone, group can be destroyed.
*/
- blkg_put(blkg);
+ percpu_ref_kill(&blkg->refcnt);
}
/**
diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h
index 68cae7961060..05909cf31ed1 100644
--- a/include/linux/blk-cgroup.h
+++ b/include/linux/blk-cgroup.h
@@ -123,7 +123,7 @@ struct blkcg_gq {
struct blkcg_gq *parent;
/* reference count */
- atomic_t refcnt;
+ struct percpu_ref refcnt;
/* is this blkg online? protected by both blkcg and q locks */
bool online;
@@ -486,8 +486,7 @@ static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
*/
static inline void blkg_get(struct blkcg_gq *blkg)
{
- WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
- atomic_inc(&blkg->refcnt);
+ percpu_ref_get(&blkg->refcnt);
}
/**
@@ -499,7 +498,7 @@ static inline void blkg_get(struct blkcg_gq *blkg)
*/
static inline struct blkcg_gq *blkg_try_get(struct blkcg_gq *blkg)
{
- if (atomic_inc_not_zero(&blkg->refcnt))
+ if (percpu_ref_tryget(&blkg->refcnt))
return blkg;
return NULL;
}
@@ -513,23 +512,19 @@ static inline struct blkcg_gq *blkg_try_get(struct blkcg_gq *blkg)
*/
static inline struct blkcg_gq *blkg_try_get_closest(struct blkcg_gq *blkg)
{
- while (!atomic_inc_not_zero(&blkg->refcnt))
+ while (!percpu_ref_tryget(&blkg->refcnt))
blkg = blkg->parent;
return blkg;
}
-void __blkg_release_rcu(struct rcu_head *rcu);
-
/**
* blkg_put - put a blkg reference
* @blkg: blkg to put
*/
static inline void blkg_put(struct blkcg_gq *blkg)
{
- WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
- if (atomic_dec_and_test(&blkg->refcnt))
- call_rcu(&blkg->rcu_head, __blkg_release_rcu);
+ percpu_ref_put(&blkg->refcnt);
}
/**
--
2.17.1
next prev parent reply other threads:[~2018-12-04 18:36 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-12-04 18:35 [PATCH v5 00/14] block: always associate blkg and refcount cleanup Dennis Zhou
2018-12-04 18:35 ` [PATCH 01/14] blkcg: fix ref count issue with bio_blkcg() using task_css Dennis Zhou
2018-12-04 18:35 ` [PATCH 02/14] blkcg: update blkg_lookup_create() to do locking Dennis Zhou
2018-12-04 18:35 ` [PATCH 03/14] blkcg: convert blkg_lookup_create() to find closest blkg Dennis Zhou
2018-12-04 18:35 ` [PATCH 04/14] blkcg: introduce common blkg association logic Dennis Zhou
2018-12-04 18:35 ` [PATCH 05/14] dm: set the static flush bio device on demand Dennis Zhou
2018-12-04 20:28 ` Mike Snitzer
2018-12-04 22:15 ` Dennis Zhou
2018-12-04 18:35 ` [PATCH 06/14] blkcg: associate blkg when associating a device Dennis Zhou
2018-12-04 19:48 ` Josef Bacik
2018-12-04 18:35 ` [PATCH 07/14] blkcg: consolidate bio_issue_init() to be a part of core Dennis Zhou
2018-12-04 19:50 ` Josef Bacik
2018-12-04 18:35 ` [PATCH 08/14] blkcg: associate a blkg for pages being evicted by swap Dennis Zhou
2018-12-04 18:35 ` [PATCH 09/14] blkcg: associate writeback bios with a blkg Dennis Zhou
2018-12-04 18:35 ` [PATCH 10/14] blkcg: remove bio->bi_css and instead use bio->bi_blkg Dennis Zhou
2018-12-04 18:35 ` [PATCH 11/14] blkcg: remove additional reference to the css Dennis Zhou
2018-12-04 19:53 ` Josef Bacik
2018-12-04 18:35 ` [PATCH 12/14] blkcg: remove bio_disassociate_task() Dennis Zhou
2018-12-04 19:53 ` Josef Bacik
2018-12-04 18:35 ` Dennis Zhou [this message]
2018-12-04 19:54 ` [PATCH 13/14] blkcg: change blkg reference counting to use percpu_ref Josef Bacik
2018-12-04 18:36 ` [PATCH 14/14] blkcg: rename blkg_try_get() to blkg_tryget() Dennis Zhou
2018-12-05 17:10 [PATCH v6 00/14] block: always associate blkg and refcount cleanup Dennis Zhou
2018-12-05 17:10 ` [PATCH 13/14] blkcg: change blkg reference counting to use percpu_ref Dennis Zhou
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20181204183600.99746-14-dennis@kernel.org \
--to=dennis@kernel.org \
--cc=axboe@kernel.dk \
--cc=cgroups@vger.kernel.org \
--cc=hannes@cmpxchg.org \
--cc=josef@toxicpanda.com \
--cc=kernel-team@fb.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=tj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).