From: Muchun Song <songmuchun@bytedance.com> To: hannes@cmpxchg.org, mhocko@kernel.org, roman.gushchin@linux.dev, shakeelb@google.com Cc: cgroups@vger.kernel.org, linux-mm@kvack.org, linux-kernel@vger.kernel.org, duanxiongchun@bytedance.com, longman@redhat.com, Muchun Song <songmuchun@bytedance.com> Subject: [PATCH v4 08/11] mm: memcontrol: introduce memcg_reparent_ops Date: Tue, 24 May 2022 14:05:48 +0800 [thread overview] Message-ID: <20220524060551.80037-9-songmuchun@bytedance.com> (raw) In-Reply-To: <20220524060551.80037-1-songmuchun@bytedance.com> In the previous patch, we know how to make the lruvec lock safe when LRU pages are reparented. We should do something like following. memcg_reparent_objcgs(memcg) 1) lock // lruvec belongs to memcg and lruvec_parent belongs to parent memcg. spin_lock(&lruvec->lru_lock); spin_lock(&lruvec_parent->lru_lock); 2) relocate from current memcg to its parent // Move all the pages from the lruvec list to the parent lruvec list. 3) unlock spin_unlock(&lruvec_parent->lru_lock); spin_unlock(&lruvec->lru_lock); Apart from the page lruvec lock, the deferred split queue lock (THP only) also needs to do something similar. So we extract the necessary three steps in the memcg_reparent_objcgs(). memcg_reparent_objcgs(memcg) 1) lock memcg_reparent_ops->lock(memcg, parent); 2) relocate memcg_reparent_ops->relocate(memcg, reparent); 3) unlock memcg_reparent_ops->unlock(memcg, reparent); Now there are two different locks (e.g. lruvec lock and deferred split queue lock) need to use this infrastructure. In the next patch, we will use those APIs to make those locks safe when the LRU pages reparented. Signed-off-by: Muchun Song <songmuchun@bytedance.com> --- include/linux/memcontrol.h | 20 +++++++++++++++ mm/memcontrol.c | 62 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 3a0e2592434e..e806e743a1fc 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -347,6 +347,26 @@ struct mem_cgroup { struct mem_cgroup_per_node *nodeinfo[]; }; +struct memcg_reparent_ops { + /* + * Note that interrupt is disabled before calling those callbacks, + * so the interrupt should remain disabled when leaving those callbacks. + */ + void (*lock)(struct mem_cgroup *src, struct mem_cgroup *dst); + void (*relocate)(struct mem_cgroup *src, struct mem_cgroup *dst); + void (*unlock)(struct mem_cgroup *src, struct mem_cgroup *dst); +}; + +#define DEFINE_MEMCG_REPARENT_OPS(name) \ + const struct memcg_reparent_ops memcg_##name##_reparent_ops = { \ + .lock = name##_reparent_lock, \ + .relocate = name##_reparent_relocate, \ + .unlock = name##_reparent_unlock, \ + } + +#define DECLARE_MEMCG_REPARENT_OPS(name) \ + extern const struct memcg_reparent_ops memcg_##name##_reparent_ops + /* * size of first charge trial. "32" comes from vmscan.c's magic value. * TODO: maybe necessary to use big numbers in big irons. diff --git a/mm/memcontrol.c b/mm/memcontrol.c index dcaf6cf5dc74..7d62764c6380 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -337,24 +337,60 @@ static struct obj_cgroup *obj_cgroup_alloc(void) return objcg; } -static void memcg_reparent_objcgs(struct mem_cgroup *memcg) +static void objcg_reparent_lock(struct mem_cgroup *src, struct mem_cgroup *dst) +{ + spin_lock(&objcg_lock); +} + +static void objcg_reparent_relocate(struct mem_cgroup *src, struct mem_cgroup *dst) { struct obj_cgroup *objcg, *iter; - struct mem_cgroup *parent = parent_mem_cgroup(memcg); - objcg = rcu_replace_pointer(memcg->objcg, NULL, true); + objcg = rcu_replace_pointer(src->objcg, NULL, true); + /* 1) Ready to reparent active objcg. */ + list_add(&objcg->list, &src->objcg_list); + /* 2) Reparent active objcg and already reparented objcgs to dst. */ + list_for_each_entry(iter, &src->objcg_list, list) + WRITE_ONCE(iter->memcg, dst); + /* 3) Move already reparented objcgs to the dst's list */ + list_splice(&src->objcg_list, &dst->objcg_list); +} + +static void objcg_reparent_unlock(struct mem_cgroup *src, struct mem_cgroup *dst) +{ + spin_unlock(&objcg_lock); +} - spin_lock_irq(&objcg_lock); +static DEFINE_MEMCG_REPARENT_OPS(objcg); - /* 1) Ready to reparent active objcg. */ - list_add(&objcg->list, &memcg->objcg_list); - /* 2) Reparent active objcg and already reparented objcgs to parent. */ - list_for_each_entry(iter, &memcg->objcg_list, list) - WRITE_ONCE(iter->memcg, parent); - /* 3) Move already reparented objcgs to the parent's list */ - list_splice(&memcg->objcg_list, &parent->objcg_list); - - spin_unlock_irq(&objcg_lock); +static const struct memcg_reparent_ops *memcg_reparent_ops[] = { + &memcg_objcg_reparent_ops, +}; + +#define DEFINE_MEMCG_REPARENT_FUNC(phase) \ + static void memcg_reparent_##phase(struct mem_cgroup *src, \ + struct mem_cgroup *dst) \ + { \ + int i; \ + \ + for (i = 0; i < ARRAY_SIZE(memcg_reparent_ops); i++) \ + memcg_reparent_ops[i]->phase(src, dst); \ + } + +DEFINE_MEMCG_REPARENT_FUNC(lock) +DEFINE_MEMCG_REPARENT_FUNC(relocate) +DEFINE_MEMCG_REPARENT_FUNC(unlock) + +static void memcg_reparent_objcgs(struct mem_cgroup *src) +{ + struct mem_cgroup *dst = parent_mem_cgroup(src); + struct obj_cgroup *objcg = rcu_dereference_protected(src->objcg, true); + + local_irq_disable(); + memcg_reparent_lock(src, dst); + memcg_reparent_relocate(src, dst); + memcg_reparent_unlock(src, dst); + local_irq_enable(); percpu_ref_kill(&objcg->refcnt); } -- 2.11.0
WARNING: multiple messages have this Message-ID (diff)
From: Muchun Song <songmuchun-EC8Uxl6Npydl57MIdRCFDg@public.gmane.org> To: hannes-druUgvl0LCNAfugRpC6u6w@public.gmane.org, mhocko-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org, roman.gushchin-fxUVXftIFDnyG1zEObXtfA@public.gmane.org, shakeelb-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org Cc: cgroups-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, linux-mm-Bw31MaZKKs3YtjvyW6yDsg@public.gmane.org, linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, duanxiongchun-EC8Uxl6Npydl57MIdRCFDg@public.gmane.org, longman-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org, Muchun Song <songmuchun-EC8Uxl6Npydl57MIdRCFDg@public.gmane.org> Subject: [PATCH v4 08/11] mm: memcontrol: introduce memcg_reparent_ops Date: Tue, 24 May 2022 14:05:48 +0800 [thread overview] Message-ID: <20220524060551.80037-9-songmuchun@bytedance.com> (raw) In-Reply-To: <20220524060551.80037-1-songmuchun-EC8Uxl6Npydl57MIdRCFDg@public.gmane.org> In the previous patch, we know how to make the lruvec lock safe when LRU pages are reparented. We should do something like following. memcg_reparent_objcgs(memcg) 1) lock // lruvec belongs to memcg and lruvec_parent belongs to parent memcg. spin_lock(&lruvec->lru_lock); spin_lock(&lruvec_parent->lru_lock); 2) relocate from current memcg to its parent // Move all the pages from the lruvec list to the parent lruvec list. 3) unlock spin_unlock(&lruvec_parent->lru_lock); spin_unlock(&lruvec->lru_lock); Apart from the page lruvec lock, the deferred split queue lock (THP only) also needs to do something similar. So we extract the necessary three steps in the memcg_reparent_objcgs(). memcg_reparent_objcgs(memcg) 1) lock memcg_reparent_ops->lock(memcg, parent); 2) relocate memcg_reparent_ops->relocate(memcg, reparent); 3) unlock memcg_reparent_ops->unlock(memcg, reparent); Now there are two different locks (e.g. lruvec lock and deferred split queue lock) need to use this infrastructure. In the next patch, we will use those APIs to make those locks safe when the LRU pages reparented. Signed-off-by: Muchun Song <songmuchun-EC8Uxl6Npydl57MIdRCFDg@public.gmane.org> --- include/linux/memcontrol.h | 20 +++++++++++++++ mm/memcontrol.c | 62 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 3a0e2592434e..e806e743a1fc 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -347,6 +347,26 @@ struct mem_cgroup { struct mem_cgroup_per_node *nodeinfo[]; }; +struct memcg_reparent_ops { + /* + * Note that interrupt is disabled before calling those callbacks, + * so the interrupt should remain disabled when leaving those callbacks. + */ + void (*lock)(struct mem_cgroup *src, struct mem_cgroup *dst); + void (*relocate)(struct mem_cgroup *src, struct mem_cgroup *dst); + void (*unlock)(struct mem_cgroup *src, struct mem_cgroup *dst); +}; + +#define DEFINE_MEMCG_REPARENT_OPS(name) \ + const struct memcg_reparent_ops memcg_##name##_reparent_ops = { \ + .lock = name##_reparent_lock, \ + .relocate = name##_reparent_relocate, \ + .unlock = name##_reparent_unlock, \ + } + +#define DECLARE_MEMCG_REPARENT_OPS(name) \ + extern const struct memcg_reparent_ops memcg_##name##_reparent_ops + /* * size of first charge trial. "32" comes from vmscan.c's magic value. * TODO: maybe necessary to use big numbers in big irons. diff --git a/mm/memcontrol.c b/mm/memcontrol.c index dcaf6cf5dc74..7d62764c6380 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -337,24 +337,60 @@ static struct obj_cgroup *obj_cgroup_alloc(void) return objcg; } -static void memcg_reparent_objcgs(struct mem_cgroup *memcg) +static void objcg_reparent_lock(struct mem_cgroup *src, struct mem_cgroup *dst) +{ + spin_lock(&objcg_lock); +} + +static void objcg_reparent_relocate(struct mem_cgroup *src, struct mem_cgroup *dst) { struct obj_cgroup *objcg, *iter; - struct mem_cgroup *parent = parent_mem_cgroup(memcg); - objcg = rcu_replace_pointer(memcg->objcg, NULL, true); + objcg = rcu_replace_pointer(src->objcg, NULL, true); + /* 1) Ready to reparent active objcg. */ + list_add(&objcg->list, &src->objcg_list); + /* 2) Reparent active objcg and already reparented objcgs to dst. */ + list_for_each_entry(iter, &src->objcg_list, list) + WRITE_ONCE(iter->memcg, dst); + /* 3) Move already reparented objcgs to the dst's list */ + list_splice(&src->objcg_list, &dst->objcg_list); +} + +static void objcg_reparent_unlock(struct mem_cgroup *src, struct mem_cgroup *dst) +{ + spin_unlock(&objcg_lock); +} - spin_lock_irq(&objcg_lock); +static DEFINE_MEMCG_REPARENT_OPS(objcg); - /* 1) Ready to reparent active objcg. */ - list_add(&objcg->list, &memcg->objcg_list); - /* 2) Reparent active objcg and already reparented objcgs to parent. */ - list_for_each_entry(iter, &memcg->objcg_list, list) - WRITE_ONCE(iter->memcg, parent); - /* 3) Move already reparented objcgs to the parent's list */ - list_splice(&memcg->objcg_list, &parent->objcg_list); - - spin_unlock_irq(&objcg_lock); +static const struct memcg_reparent_ops *memcg_reparent_ops[] = { + &memcg_objcg_reparent_ops, +}; + +#define DEFINE_MEMCG_REPARENT_FUNC(phase) \ + static void memcg_reparent_##phase(struct mem_cgroup *src, \ + struct mem_cgroup *dst) \ + { \ + int i; \ + \ + for (i = 0; i < ARRAY_SIZE(memcg_reparent_ops); i++) \ + memcg_reparent_ops[i]->phase(src, dst); \ + } + +DEFINE_MEMCG_REPARENT_FUNC(lock) +DEFINE_MEMCG_REPARENT_FUNC(relocate) +DEFINE_MEMCG_REPARENT_FUNC(unlock) + +static void memcg_reparent_objcgs(struct mem_cgroup *src) +{ + struct mem_cgroup *dst = parent_mem_cgroup(src); + struct obj_cgroup *objcg = rcu_dereference_protected(src->objcg, true); + + local_irq_disable(); + memcg_reparent_lock(src, dst); + memcg_reparent_relocate(src, dst); + memcg_reparent_unlock(src, dst); + local_irq_enable(); percpu_ref_kill(&objcg->refcnt); } -- 2.11.0
next prev parent reply other threads:[~2022-05-24 6:28 UTC|newest] Thread overview: 93+ messages / expand[flat|nested] mbox.gz Atom feed top 2022-05-24 6:05 [PATCH v4 00/11] Use obj_cgroup APIs to charge the LRU pages Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-24 6:05 ` [PATCH v4 01/11] mm: memcontrol: prepare objcg API for non-kmem usage Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-24 19:01 ` Johannes Weiner 2022-05-24 19:01 ` Johannes Weiner 2022-05-25 8:46 ` Muchun Song 2022-05-25 8:46 ` Muchun Song 2022-05-25 2:36 ` Roman Gushchin 2022-05-25 2:36 ` Roman Gushchin 2022-05-25 7:57 ` Muchun Song 2022-05-25 7:57 ` Muchun Song 2022-05-25 12:37 ` Johannes Weiner 2022-05-25 12:37 ` Johannes Weiner 2022-05-25 13:08 ` Muchun Song 2022-05-25 13:08 ` Muchun Song 2022-05-24 6:05 ` [PATCH v4 02/11] mm: memcontrol: introduce compact_folio_lruvec_lock_irqsave Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-24 19:22 ` Johannes Weiner 2022-05-24 19:22 ` Johannes Weiner 2022-05-25 9:38 ` Muchun Song 2022-05-25 9:38 ` Muchun Song 2022-05-24 6:05 ` [PATCH v4 03/11] mm: memcontrol: make lruvec lock safe when LRU pages are reparented Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-24 19:23 ` Waiman Long 2022-05-25 10:20 ` Muchun Song 2022-05-25 10:20 ` Muchun Song 2022-05-25 14:59 ` Waiman Long 2022-05-25 14:59 ` Waiman Long 2022-05-24 19:27 ` Johannes Weiner 2022-05-24 19:27 ` Johannes Weiner 2022-05-25 9:53 ` Muchun Song 2022-05-25 9:53 ` Muchun Song 2022-05-25 12:30 ` Johannes Weiner 2022-05-25 12:30 ` Johannes Weiner 2022-05-25 13:03 ` Muchun Song 2022-05-25 13:03 ` Muchun Song 2022-05-25 14:48 ` Johannes Weiner 2022-05-25 14:48 ` Johannes Weiner 2022-05-25 15:38 ` Muchun Song 2022-05-25 15:38 ` Muchun Song 2022-05-26 20:17 ` Waiman Long 2022-05-26 20:17 ` Waiman Long 2022-05-27 2:55 ` Muchun Song 2022-05-27 2:55 ` Muchun Song 2022-05-24 6:05 ` [PATCH v4 04/11] mm: vmscan: rework move_pages_to_lru() Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-24 19:38 ` Johannes Weiner 2022-05-24 19:38 ` Johannes Weiner 2022-05-25 11:38 ` Muchun Song 2022-05-25 11:38 ` Muchun Song 2022-05-24 19:52 ` Waiman Long 2022-05-24 19:52 ` Waiman Long 2022-05-25 11:43 ` Muchun Song 2022-05-25 11:43 ` Muchun Song 2022-05-25 2:43 ` Roman Gushchin 2022-05-25 2:43 ` Roman Gushchin 2022-05-25 11:41 ` Muchun Song 2022-05-25 11:41 ` Muchun Song 2022-05-24 6:05 ` [PATCH v4 05/11] mm: thp: introduce folio_split_queue_lock{_irqsave}() Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-24 6:05 ` [PATCH v4 06/11] mm: thp: make split queue lock safe when LRU pages are reparented Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-25 2:54 ` Roman Gushchin 2022-05-25 2:54 ` Roman Gushchin 2022-05-25 11:44 ` Muchun Song 2022-05-25 11:44 ` Muchun Song 2022-05-24 6:05 ` [PATCH v4 07/11] mm: memcontrol: make all the callers of {folio,page}_memcg() safe Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-25 3:03 ` Roman Gushchin 2022-05-25 3:03 ` Roman Gushchin 2022-05-25 11:51 ` Muchun Song 2022-05-25 11:51 ` Muchun Song 2022-05-24 6:05 ` Muchun Song [this message] 2022-05-24 6:05 ` [PATCH v4 08/11] mm: memcontrol: introduce memcg_reparent_ops Muchun Song 2022-05-24 6:05 ` [PATCH v4 09/11] mm: memcontrol: use obj_cgroup APIs to charge the LRU pages Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-24 12:29 ` kernel test robot 2022-05-24 18:16 ` kernel test robot 2022-05-24 18:16 ` kernel test robot 2022-05-25 7:14 ` [mm] bec0ae1210: WARNING:possible_recursive_locking_detected kernel test robot 2022-05-25 7:14 ` kernel test robot 2022-05-24 6:05 ` [PATCH v4 10/11] mm: lru: add VM_BUG_ON_FOLIO to lru maintenance function Muchun Song 2022-05-24 6:05 ` Muchun Song 2022-05-24 19:44 ` Johannes Weiner 2022-05-24 19:44 ` Johannes Weiner 2022-05-25 11:59 ` Muchun Song 2022-05-25 11:59 ` Muchun Song 2022-05-25 2:40 ` Roman Gushchin 2022-05-25 2:40 ` Roman Gushchin 2022-05-25 11:58 ` Muchun Song 2022-05-24 6:05 ` [PATCH v4 11/11] mm: lru: use lruvec lock to serialize memcg changes Muchun Song 2022-05-24 6:05 ` Muchun Song
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20220524060551.80037-9-songmuchun@bytedance.com \ --to=songmuchun@bytedance.com \ --cc=cgroups@vger.kernel.org \ --cc=duanxiongchun@bytedance.com \ --cc=hannes@cmpxchg.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ --cc=longman@redhat.com \ --cc=mhocko@kernel.org \ --cc=roman.gushchin@linux.dev \ --cc=shakeelb@google.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.