From: Minchan Kim <minchan@kernel.org>
To: Andrew Morton <akpm@linux-foundation.org>
Cc: linux-mm <linux-mm@kvack.org>,
LKML <linux-kernel@vger.kernel.org>,
joaodias@google.com, surenb@google.com, cgoldswo@codeaurora.org,
willy@infradead.org, mhocko@suse.com, david@redhat.com,
vbabka@suse.cz, linux-fsdevel@vger.kernel.org,
Minchan Kim <minchan@kernel.org>
Subject: [PATCH v3 3/3] mm: fs: Invalidate BH LRU during page migration
Date: Wed, 10 Mar 2021 08:14:29 -0800 [thread overview]
Message-ID: <20210310161429.399432-3-minchan@kernel.org> (raw)
In-Reply-To: <20210310161429.399432-1-minchan@kernel.org>
Pages containing buffer_heads that are in one of the per-CPU
buffer_head LRU caches will be pinned and thus cannot be migrated.
This can prevent CMA allocations from succeeding, which are often used
on platforms with co-processors (such as a DSP) that can only use
physically contiguous memory. It can also prevent memory
hot-unplugging from succeeding, which involves migrating at least
MIN_MEMORY_BLOCK_SIZE bytes of memory, which ranges from 8 MiB to 1
GiB based on the architecture in use.
Correspondingly, invalidate the BH LRU caches before a migration
starts and stop any buffer_head from being cached in the LRU caches,
until migration has finished.
Signed-off-by: Chris Goldsworthy <cgoldswo@codeaurora.org>
Signed-off-by: Minchan Kim <minchan@kernel.org>
---
fs/buffer.c | 12 ++++++++++--
include/linux/buffer_head.h | 4 ++++
mm/swap.c | 5 ++++-
3 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/fs/buffer.c b/fs/buffer.c
index 0cb7ffd4977c..ca9dd736bcb8 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -1264,6 +1264,14 @@ static void bh_lru_install(struct buffer_head *bh)
int i;
check_irqs_on();
+ /*
+ * buffer_head in bh_lru could increase refcount of the page
+ * until it will be invalidated. It causes page migraion failure.
+ * Skip putting upcoming bh into bh_lru until migration is done.
+ */
+ if (lru_cache_disabled())
+ return;
+
bh_lru_lock();
b = this_cpu_ptr(&bh_lrus);
@@ -1409,7 +1417,7 @@ EXPORT_SYMBOL(__bread_gfp);
* This doesn't race because it runs in each cpu either in irq
* or with preempt disabled.
*/
-static void invalidate_bh_lru(void *arg)
+void invalidate_bh_lru(void *arg)
{
struct bh_lru *b = &get_cpu_var(bh_lrus);
int i;
@@ -1421,7 +1429,7 @@ static void invalidate_bh_lru(void *arg)
put_cpu_var(bh_lrus);
}
-static bool has_bh_in_lru(int cpu, void *dummy)
+bool has_bh_in_lru(int cpu, void *dummy)
{
struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
int i;
diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h
index 6b47f94378c5..05998b5947a2 100644
--- a/include/linux/buffer_head.h
+++ b/include/linux/buffer_head.h
@@ -194,6 +194,8 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,
struct buffer_head *__bread_gfp(struct block_device *,
sector_t block, unsigned size, gfp_t gfp);
void invalidate_bh_lrus(void);
+void invalidate_bh_lru(void *arg);
+bool has_bh_in_lru(int cpu, void *dummy);
struct buffer_head *alloc_buffer_head(gfp_t gfp_flags);
void free_buffer_head(struct buffer_head * bh);
void unlock_buffer(struct buffer_head *bh);
@@ -406,6 +408,8 @@ static inline int inode_has_buffers(struct inode *inode) { return 0; }
static inline void invalidate_inode_buffers(struct inode *inode) {}
static inline int remove_inode_buffers(struct inode *inode) { return 1; }
static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; }
+static inline void invalidate_bh_lru(void *arg) {}
+static inline bool has_bh_in_lru(int cpu, void *dummy) { return 0; }
#define buffer_heads_over_limit 0
#endif /* CONFIG_BLOCK */
diff --git a/mm/swap.c b/mm/swap.c
index fbdf6ac05aec..2a431959a45d 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -36,6 +36,7 @@
#include <linux/hugetlb.h>
#include <linux/page_idle.h>
#include <linux/local_lock.h>
+#include <linux/buffer_head.h>
#include "internal.h"
@@ -641,6 +642,7 @@ void lru_add_drain_cpu(int cpu)
pagevec_lru_move_fn(pvec, lru_lazyfree_fn);
activate_page_drain(cpu);
+ invalidate_bh_lru(NULL);
}
/**
@@ -828,7 +830,8 @@ static void __lru_add_drain_all(bool force_all_cpus)
pagevec_count(&per_cpu(lru_pvecs.lru_deactivate_file, cpu)) ||
pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) ||
pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) ||
- need_activate_page_drain(cpu)) {
+ need_activate_page_drain(cpu) ||
+ has_bh_in_lru(cpu, NULL)) {
INIT_WORK(work, lru_add_drain_per_cpu);
queue_work_on(cpu, mm_percpu_wq, work);
__cpumask_set_cpu(cpu, &has_work);
--
2.30.1.766.gb4fecdf3b7-goog
next prev parent reply other threads:[~2021-03-10 16:15 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-03-10 16:14 [PATCH v3 1/3] mm: replace migrate_prep with lru_add_drain_all Minchan Kim
2021-03-10 16:14 ` [PATCH v3 2/3] mm: disable LRU pagevec during the migration temporarily Minchan Kim
2021-03-11 22:41 ` Chris Goldsworthy
2021-03-14 5:10 ` Chris Goldsworthy
2021-03-12 8:21 ` Michal Hocko
2021-03-12 9:00 ` David Hildenbrand
2021-03-18 0:13 ` Andrew Morton
2021-03-18 1:13 ` Minchan Kim
2021-03-18 8:09 ` Michal Hocko
2021-03-10 16:14 ` Minchan Kim [this message]
2021-03-12 9:03 ` [PATCH v3 3/3] mm: fs: Invalidate BH LRU during page migration David Hildenbrand
2021-03-12 9:33 ` David Hildenbrand
2021-03-12 17:17 ` Minchan Kim
2021-03-16 18:26 ` Minchan Kim
2021-03-17 11:18 ` David Hildenbrand
2021-03-17 2:37 ` [mm] 8fd8d23ab1: WARNING:at_fs/buffer.c:#__brelse kernel test robot
2021-03-17 16:29 ` Minchan Kim
2021-03-19 14:05 ` Oliver Sang
2021-03-19 16:47 ` Minchan Kim
2021-03-12 8:19 ` [PATCH v3 1/3] mm: replace migrate_prep with lru_add_drain_all Michal Hocko
2021-03-12 8:53 ` David Hildenbrand
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210310161429.399432-3-minchan@kernel.org \
--to=minchan@kernel.org \
--cc=akpm@linux-foundation.org \
--cc=cgoldswo@codeaurora.org \
--cc=david@redhat.com \
--cc=joaodias@google.com \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=mhocko@suse.com \
--cc=surenb@google.com \
--cc=vbabka@suse.cz \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).