From: Byungchul Park <byungchul.park@lge.com>
To: torvalds@linux-foundation.org
Cc: damien.lemoal@opensource.wdc.com, linux-ide@vger.kernel.org,
adilger.kernel@dilger.ca, linux-ext4@vger.kernel.org,
mingo@redhat.com, linux-kernel@vger.kernel.org,
peterz@infradead.org, will@kernel.org, tglx@linutronix.de,
rostedt@goodmis.org, joel@joelfernandes.org, sashal@kernel.org,
daniel.vetter@ffwll.ch, chris@chris-wilson.co.uk,
duyuyang@gmail.com, johannes.berg@intel.com, tj@kernel.org,
tytso@mit.edu, willy@infradead.org, david@fromorbit.com,
amir73il@gmail.com, bfields@fieldses.org,
gregkh@linuxfoundation.org, kernel-team@lge.com,
linux-mm@kvack.org, akpm@linux-foundation.org, mhocko@kernel.org,
minchan@kernel.org, hannes@cmpxchg.org, vdavydov.dev@gmail.com,
sj@kernel.org, jglisse@redhat.com, dennis@kernel.org,
cl@linux.com, penberg@kernel.org, rientjes@google.com,
vbabka@suse.cz, ngupta@vflare.org, linux-block@vger.kernel.org,
paolo.valente@linaro.org, josef@toxicpanda.com,
linux-fsdevel@vger.kernel.org, viro@zeniv.linux.org.uk,
jack@suse.cz, jack@suse.com, jlayton@kernel.org,
dan.j.williams@intel.com, hch@infradead.org, djwong@kernel.org,
dri-devel@lists.freedesktop.org, airlied@linux.ie,
rodrigosiqueiramelo@gmail.com, melissa.srw@gmail.com,
hamohammed.sa@gmail.com, 42.hyeyoo@gmail.com
Subject: [PATCH RFC v6 11/21] dept: Apply Dept to wait/event of PG_{locked,writeback}
Date: Wed, 4 May 2022 17:17:39 +0900 [thread overview]
Message-ID: <1651652269-15342-12-git-send-email-byungchul.park@lge.com> (raw)
In-Reply-To: <1651652269-15342-1-git-send-email-byungchul.park@lge.com>
Makes Dept able to track dependencies by PG_{locked,writeback}. For
instance, (un)lock_page() generates that type of dependency.
Signed-off-by: Byungchul Park <byungchul.park@lge.com>
---
include/linux/dept_page.h | 78 +++++++++++++++++++++++++++++++++++++++++
include/linux/page-flags.h | 45 ++++++++++++++++++++++--
include/linux/pagemap.h | 7 +++-
init/main.c | 2 ++
kernel/dependency/dept_object.h | 2 +-
lib/Kconfig.debug | 1 +
mm/filemap.c | 68 +++++++++++++++++++++++++++++++++++
mm/page_ext.c | 5 +++
8 files changed, 204 insertions(+), 4 deletions(-)
create mode 100644 include/linux/dept_page.h
diff --git a/include/linux/dept_page.h b/include/linux/dept_page.h
new file mode 100644
index 00000000..d2d093d
--- /dev/null
+++ b/include/linux/dept_page.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_DEPT_PAGE_H
+#define __LINUX_DEPT_PAGE_H
+
+#ifdef CONFIG_DEPT
+#include <linux/dept.h>
+
+extern struct page_ext_operations dept_pglocked_ops;
+extern struct page_ext_operations dept_pgwriteback_ops;
+extern struct dept_map_common pglocked_mc;
+extern struct dept_map_common pgwriteback_mc;
+
+extern void dept_page_init(void);
+extern struct dept_map_each *get_pglocked_me(struct page *page);
+extern struct dept_map_each *get_pgwriteback_me(struct page *page);
+
+#define dept_pglocked_wait(f) \
+do { \
+ struct dept_map_each *me = get_pglocked_me(&(f)->page); \
+ \
+ if (likely(me)) \
+ dept_wait_split_map(me, &pglocked_mc, _RET_IP_, \
+ __func__, 0); \
+} while (0)
+
+#define dept_pglocked_set_bit(f) \
+do { \
+ struct dept_map_each *me = get_pglocked_me(&(f)->page); \
+ \
+ if (likely(me)) \
+ dept_ask_event_split_map(me, &pglocked_mc); \
+} while (0)
+
+#define dept_pglocked_event(f) \
+do { \
+ struct dept_map_each *me = get_pglocked_me(&(f)->page); \
+ \
+ if (likely(me)) \
+ dept_event_split_map(me, &pglocked_mc, _RET_IP_,\
+ __func__); \
+} while (0)
+
+#define dept_pgwriteback_wait(f) \
+do { \
+ struct dept_map_each *me = get_pgwriteback_me(&(f)->page);\
+ \
+ if (likely(me)) \
+ dept_wait_split_map(me, &pgwriteback_mc, _RET_IP_,\
+ __func__, 0); \
+} while (0)
+
+#define dept_pgwriteback_set_bit(f) \
+do { \
+ struct dept_map_each *me = get_pgwriteback_me(&(f)->page);\
+ \
+ if (likely(me)) \
+ dept_ask_event_split_map(me, &pgwriteback_mc);\
+} while (0)
+
+#define dept_pgwriteback_event(f) \
+do { \
+ struct dept_map_each *me = get_pgwriteback_me(&(f)->page);\
+ \
+ if (likely(me)) \
+ dept_event_split_map(me, &pgwriteback_mc, _RET_IP_,\
+ __func__); \
+} while (0)
+#else
+#define dept_page_init() do { } while (0)
+#define dept_pglocked_wait(f) do { } while (0)
+#define dept_pglocked_set_bit(f) do { } while (0)
+#define dept_pglocked_event(f) do { } while (0)
+#define dept_pgwriteback_wait(f) do { } while (0)
+#define dept_pgwriteback_set_bit(f) do { } while (0)
+#define dept_pgwriteback_event(f) do { } while (0)
+#endif
+
+#endif /* __LINUX_DEPT_PAGE_H */
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 9d8eeaa..9fd9e39 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -480,7 +480,6 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n)
#define TESTSCFLAG_FALSE(uname, lname) \
TESTSETFLAG_FALSE(uname, lname) TESTCLEARFLAG_FALSE(uname, lname)
-__PAGEFLAG(Locked, locked, PF_NO_TAIL)
PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
PAGEFLAG(Error, error, PF_NO_TAIL) TESTCLEARFLAG(Error, error, PF_NO_TAIL)
PAGEFLAG(Referenced, referenced, PF_HEAD)
@@ -528,7 +527,6 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n)
* risky: they bypass page accounting.
*/
TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
- TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
/* PG_readahead is only used for reads; PG_reclaim is only for writes */
@@ -611,6 +609,49 @@ static __always_inline bool PageSwapCache(struct page *page)
PAGEFLAG_FALSE(SkipKASanPoison, skip_kasan_poison)
#endif
+#ifdef CONFIG_DEPT
+TESTPAGEFLAG(Locked, locked, PF_NO_TAIL)
+__CLEARPAGEFLAG(Locked, locked, PF_NO_TAIL)
+TESTCLEARFLAG(Writeback, writeback, PF_NO_TAIL)
+
+#include <linux/dept_page.h>
+
+static __always_inline
+void __folio_set_locked(struct folio *folio)
+{
+ dept_pglocked_set_bit(folio);
+ __set_bit(PG_locked, folio_flags(folio, FOLIO_PF_NO_TAIL));
+}
+
+static __always_inline void __SetPageLocked(struct page *page)
+{
+ dept_pglocked_set_bit(page_folio(page));
+ __set_bit(PG_locked, &PF_NO_TAIL(page, 1)->flags);
+}
+
+static __always_inline
+bool folio_test_set_writeback(struct folio *folio)
+{
+ bool ret = test_and_set_bit(PG_writeback, folio_flags(folio, FOLIO_PF_NO_TAIL));
+
+ if (!ret)
+ dept_pgwriteback_set_bit(folio);
+ return ret;
+}
+
+static __always_inline int TestSetPageWriteback(struct page *page)
+{
+ int ret = test_and_set_bit(PG_writeback, &PF_NO_TAIL(page, 1)->flags);
+
+ if (!ret)
+ dept_pgwriteback_set_bit(page_folio(page));
+ return ret;
+}
+#else
+__PAGEFLAG(Locked, locked, PF_NO_TAIL)
+TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
+#endif
+
/*
* PageReported() is used to track reported free pages within the Buddy
* allocator. We can use the non-atomic version of the test and set
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 993994c..49b211c 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -15,6 +15,7 @@
#include <linux/bitops.h>
#include <linux/hardirq.h> /* for in_interrupt() */
#include <linux/hugetlb_inline.h>
+#include <linux/dept_page.h>
struct folio_batch;
@@ -890,7 +891,11 @@ bool __folio_lock_or_retry(struct folio *folio, struct mm_struct *mm,
static inline bool folio_trylock(struct folio *folio)
{
- return likely(!test_and_set_bit_lock(PG_locked, folio_flags(folio, 0)));
+ int ret = test_and_set_bit_lock(PG_locked, folio_flags(folio, 0));
+
+ if (likely(!ret))
+ dept_pglocked_set_bit(folio);
+ return likely(!ret);
}
/*
diff --git a/init/main.c b/init/main.c
index deabdd5..7d3b905 100644
--- a/init/main.c
+++ b/init/main.c
@@ -101,6 +101,7 @@
#include <linux/init_syscalls.h>
#include <linux/stackdepot.h>
#include <net/net_namespace.h>
+#include <linux/pagemap.h>
#include <asm/io.h>
#include <asm/bugs.h>
@@ -1073,6 +1074,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
lockdep_init();
dept_init();
+ dept_page_init();
/*
* Need to run this when irqs are enabled, because it wants
diff --git a/kernel/dependency/dept_object.h b/kernel/dependency/dept_object.h
index 0b7eb16..75b4212 100644
--- a/kernel/dependency/dept_object.h
+++ b/kernel/dependency/dept_object.h
@@ -6,7 +6,7 @@
* nr: # of the object that should be kept in the pool.
*/
-OBJECT(dep, 1024 * 8)
+OBJECT(dep, 1024 * 16)
OBJECT(class, 1024 * 8)
OBJECT(stack, 1024 * 32)
OBJECT(ecxt, 1024 * 16)
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 3c17507..6bb1bf2 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1270,6 +1270,7 @@ config DEPT
select DEBUG_RWSEMS
select DEBUG_WW_MUTEX_SLOWPATH
select DEBUG_LOCK_ALLOC
+ select PAGE_EXTENSION
select TRACE_IRQFLAGS
select STACKTRACE
select FRAME_POINTER if !MIPS && !PPC && !ARM && !S390 && !MICROBLAZE && !ARC && !X86
diff --git a/mm/filemap.c b/mm/filemap.c
index 9a1eef6..eb20de95 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1157,6 +1157,11 @@ static void folio_wake_bit(struct folio *folio, int bit_nr)
unsigned long flags;
wait_queue_entry_t bookmark;
+ if (bit_nr == PG_locked)
+ dept_pglocked_event(folio);
+ else if (bit_nr == PG_writeback)
+ dept_pgwriteback_event(folio);
+
key.folio = folio;
key.bit_nr = bit_nr;
key.page_match = 0;
@@ -1229,6 +1234,10 @@ static inline bool folio_trylock_flag(struct folio *folio, int bit_nr,
if (wait->flags & WQ_FLAG_EXCLUSIVE) {
if (test_and_set_bit(bit_nr, &folio->flags))
return false;
+ else if (bit_nr == PG_locked)
+ dept_pglocked_set_bit(folio);
+ else if (bit_nr == PG_writeback)
+ dept_pgwriteback_set_bit(folio);
} else if (test_bit(bit_nr, &folio->flags))
return false;
@@ -1250,6 +1259,11 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
bool delayacct = false;
unsigned long pflags;
+ if (bit_nr == PG_locked)
+ dept_pglocked_wait(folio);
+ else if (bit_nr == PG_writeback)
+ dept_pgwriteback_wait(folio);
+
if (bit_nr == PG_locked &&
!folio_test_uptodate(folio) && folio_test_workingset(folio)) {
if (!folio_test_swapbacked(folio)) {
@@ -1342,6 +1356,11 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0))))
goto repeat;
+ if (bit_nr == PG_locked)
+ dept_pglocked_set_bit(folio);
+ else if (bit_nr == PG_writeback)
+ dept_pgwriteback_set_bit(folio);
+
wait->flags |= WQ_FLAG_DONE;
break;
}
@@ -3983,3 +4002,52 @@ bool filemap_release_folio(struct folio *folio, gfp_t gfp)
return try_to_free_buffers(&folio->page);
}
EXPORT_SYMBOL(filemap_release_folio);
+
+#ifdef CONFIG_DEPT
+static bool need_dept_pglocked(void)
+{
+ return true;
+}
+
+struct page_ext_operations dept_pglocked_ops = {
+ .size = sizeof(struct dept_map_each),
+ .need = need_dept_pglocked,
+};
+
+struct dept_map_each *get_pglocked_me(struct page *p)
+{
+ struct page_ext *e = lookup_page_ext(p);
+
+ return e ? (void *)e + dept_pglocked_ops.offset : NULL;
+}
+EXPORT_SYMBOL(get_pglocked_me);
+
+static bool need_dept_pgwriteback(void)
+{
+ return true;
+}
+
+struct page_ext_operations dept_pgwriteback_ops = {
+ .size = sizeof(struct dept_map_each),
+ .need = need_dept_pgwriteback,
+};
+
+struct dept_map_each *get_pgwriteback_me(struct page *p)
+{
+ struct page_ext *e = lookup_page_ext(p);
+
+ return e ? (void *)e + dept_pgwriteback_ops.offset : NULL;
+}
+EXPORT_SYMBOL(get_pgwriteback_me);
+
+struct dept_map_common pglocked_mc;
+EXPORT_SYMBOL(pglocked_mc);
+struct dept_map_common pgwriteback_mc;
+EXPORT_SYMBOL(pgwriteback_mc);
+
+void dept_page_init(void)
+{
+ dept_split_map_common_init(&pglocked_mc, NULL, "pglocked");
+ dept_split_map_common_init(&pgwriteback_mc, NULL, "pgwriteback");
+}
+#endif
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 2e66d93..b7f5b0d 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -9,6 +9,7 @@
#include <linux/page_owner.h>
#include <linux/page_idle.h>
#include <linux/page_table_check.h>
+#include <linux/dept_page.h>
/*
* struct page extension
@@ -79,6 +80,10 @@ static bool need_page_idle(void)
#ifdef CONFIG_PAGE_TABLE_CHECK
&page_table_check_ops,
#endif
+#ifdef CONFIG_DEPT
+ &dept_pglocked_ops,
+ &dept_pgwriteback_ops,
+#endif
};
unsigned long page_ext_size = sizeof(struct page_ext);
--
1.9.1
next prev parent reply other threads:[~2022-05-04 8:53 UTC|newest]
Thread overview: 52+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-05-04 8:17 [PATCH RFC v6 00/21] DEPT(Dependency Tracker) Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 01/21] llist: Move llist_{head,node} definition to types.h Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 02/21] dept: Implement Dept(Dependency Tracker) Byungchul Park
2022-05-21 3:24 ` Hyeonggon Yoo
2022-05-04 8:17 ` [PATCH RFC v6 03/21] dept: Apply Dept to spinlock Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 04/21] dept: Apply Dept to mutex families Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 05/21] dept: Apply Dept to rwlock Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 06/21] dept: Apply Dept to wait_for_completion()/complete() Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 07/21] dept: Apply Dept to seqlock Byungchul Park
2022-05-21 5:25 ` Hyeonggon Yoo
2022-05-24 6:00 ` Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 08/21] dept: Apply Dept to rwsem Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 09/21] dept: Add proc knobs to show stats and dependency graph Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 10/21] dept: Introduce split map concept and new APIs for them Byungchul Park
2022-05-04 8:17 ` Byungchul Park [this message]
2022-05-04 8:17 ` [PATCH RFC v6 12/21] dept: Apply SDT to swait Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 13/21] dept: Apply SDT to wait(waitqueue) Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 14/21] locking/lockdep, cpu/hotplus: Use a weaker annotation in AP thread Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 15/21] dept: Distinguish each syscall context from another Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 16/21] dept: Distinguish each work " Byungchul Park
2022-05-04 11:23 ` Sergey Shtylyov
2022-05-04 8:17 ` [PATCH RFC v6 17/21] dept: Disable Dept within the wait_bit layer by default Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 18/21] dept: Disable Dept on struct crypto_larval's completion for now Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 19/21] dept: Differentiate onstack maps from others of different tasks in class Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 20/21] dept: Do not add dependencies between events within scheduler and sleeps Byungchul Park
2022-05-04 8:17 ` [PATCH RFC v6 21/21] dept: Unstage wait when tagging a normal sleep wait Byungchul Park
2022-05-04 18:17 ` [PATCH RFC v6 00/21] DEPT(Dependency Tracker) Linus Torvalds
2022-05-06 0:11 ` Byungchul Park
2022-05-07 7:20 ` Hyeonggon Yoo
2022-05-09 0:16 ` Byungchul Park
2022-05-09 20:47 ` Steven Rostedt
2022-05-09 23:38 ` Byungchul Park
2022-05-10 14:12 ` Steven Rostedt
2022-05-10 23:26 ` Byungchul Park
2022-05-10 11:18 ` Hyeonggon Yoo
2022-05-10 23:39 ` Byungchul Park
2022-05-11 10:04 ` Hyeonggon Yoo
2022-05-19 10:11 ` Catalin Marinas
2022-05-23 2:43 ` Byungchul Park
2022-05-09 1:22 ` Byungchul Park
2022-05-09 21:05 ` Theodore Ts'o
2022-05-09 22:28 ` Theodore Ts'o
2022-05-10 0:32 ` Byungchul Park
2022-05-10 1:32 ` Theodore Ts'o
2022-05-10 5:37 ` Byungchul Park
2022-05-11 1:16 ` Byungchul Park
2022-05-12 5:25 ` [REPORT] syscall reboot + umh + firmware fallback Byungchul Park
2022-05-12 9:15 ` Tejun Heo
2022-05-12 11:18 ` Byungchul Park
2022-05-12 13:56 ` Theodore Ts'o
2022-05-23 1:10 ` Byungchul Park
2022-05-12 16:41 ` Tejun Heo
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1651652269-15342-12-git-send-email-byungchul.park@lge.com \
--to=byungchul.park@lge.com \
--cc=42.hyeyoo@gmail.com \
--cc=adilger.kernel@dilger.ca \
--cc=airlied@linux.ie \
--cc=akpm@linux-foundation.org \
--cc=amir73il@gmail.com \
--cc=bfields@fieldses.org \
--cc=chris@chris-wilson.co.uk \
--cc=cl@linux.com \
--cc=damien.lemoal@opensource.wdc.com \
--cc=dan.j.williams@intel.com \
--cc=daniel.vetter@ffwll.ch \
--cc=david@fromorbit.com \
--cc=dennis@kernel.org \
--cc=djwong@kernel.org \
--cc=dri-devel@lists.freedesktop.org \
--cc=duyuyang@gmail.com \
--cc=gregkh@linuxfoundation.org \
--cc=hamohammed.sa@gmail.com \
--cc=hannes@cmpxchg.org \
--cc=hch@infradead.org \
--cc=jack@suse.com \
--cc=jack@suse.cz \
--cc=jglisse@redhat.com \
--cc=jlayton@kernel.org \
--cc=joel@joelfernandes.org \
--cc=johannes.berg@intel.com \
--cc=josef@toxicpanda.com \
--cc=kernel-team@lge.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-ext4@vger.kernel.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-ide@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=melissa.srw@gmail.com \
--cc=mhocko@kernel.org \
--cc=minchan@kernel.org \
--cc=mingo@redhat.com \
--cc=ngupta@vflare.org \
--cc=paolo.valente@linaro.org \
--cc=penberg@kernel.org \
--cc=peterz@infradead.org \
--cc=rientjes@google.com \
--cc=rodrigosiqueiramelo@gmail.com \
--cc=rostedt@goodmis.org \
--cc=sashal@kernel.org \
--cc=sj@kernel.org \
--cc=tglx@linutronix.de \
--cc=tj@kernel.org \
--cc=torvalds@linux-foundation.org \
--cc=tytso@mit.edu \
--cc=vbabka@suse.cz \
--cc=vdavydov.dev@gmail.com \
--cc=viro@zeniv.linux.org.uk \
--cc=will@kernel.org \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).