From: Rik van Riel <riel@redhat.com>
To: linux-mm@kvack.org
Cc: linux-kernel@vger.kernel.org, lee.shermerhorn@hp.com,
Lee Schermerhorn <lee.schermerhorn@hp.com>
Subject: [patch 19/20] handle mlocked pages during map/unmap and truncate
Date: Tue, 18 Dec 2007 16:15:58 -0500 [thread overview]
Message-ID: <20071218211550.392853742@redhat.com> (raw)
In-Reply-To: 20071218211539.250334036@redhat.com
[-- Attachment #1: noreclaim-04.2-move-mlocked-pages-off-the-LRU.patch --]
[-- Type: text/plain, Size: 6747 bytes --]
V2 -> V3:
+ rebase to 23-mm1 atop RvR's split lru series [no changes]
V1 -> V2:
+ modified mmap.c:mmap_region() to return error if mlock_vma_pages_range()
does. This can only occur if the vma gets removed/changed while
we're switching mmap_sem lock modes. Most callers don't care, but
sys_remap_file_pages() appears to.
Rework of Nick Piggins's "mm: move mlocked pages off the LRU" patch
-- part 2 0f 2.
Remove mlocked pages from the LRU using "NoReclaim infrastructure"
during mmap()/mremap(). Try to move back to normal LRU lists on
munmap() when last locked mapping removed. Removed PageMlocked()
status when page truncated from file.
Originally Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Signed-off-by: Rik van Riel <riel@redhat.com>
Index: Linux/mm/mmap.c
===================================================================
--- Linux.orig/mm/mmap.c 2007-11-12 16:21:48.000000000 -0500
+++ Linux/mm/mmap.c 2007-11-12 16:22:03.000000000 -0500
@@ -32,6 +32,8 @@
#include <asm/tlb.h>
#include <asm/mmu_context.h>
+#include "internal.h"
+
#ifndef arch_mmap_check
#define arch_mmap_check(addr, len, flags) (0)
#endif
@@ -1211,9 +1213,13 @@ out:
vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
if (vm_flags & VM_LOCKED) {
mm->locked_vm += len >> PAGE_SHIFT;
- make_pages_present(addr, addr + len);
- }
- if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
+ /*
+ * makes pages present; downgrades, drops, requires mmap_sem
+ */
+ error = mlock_vma_pages_range(vma, addr, addr + len);
+ if (error)
+ return error; /* vma gone! */
+ } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
make_pages_present(addr, addr + len);
return addr;
@@ -1892,6 +1898,19 @@ int do_munmap(struct mm_struct *mm, unsi
vma = prev? prev->vm_next: mm->mmap;
/*
+ * unlock any mlock()ed ranges before detaching vmas
+ */
+ if (mm->locked_vm) {
+ struct vm_area_struct *tmp = vma;
+ while (tmp && tmp->vm_start < end) {
+ if (tmp->vm_flags & VM_LOCKED)
+ munlock_vma_pages_range(tmp,
+ tmp->vm_start, tmp->vm_end);
+ tmp = tmp->vm_next;
+ }
+ }
+
+ /*
* Remove the vma's, and unmap the actual pages
*/
detach_vmas_to_be_unmapped(mm, vma, prev, end);
@@ -2024,7 +2043,7 @@ out:
mm->total_vm += len >> PAGE_SHIFT;
if (flags & VM_LOCKED) {
mm->locked_vm += len >> PAGE_SHIFT;
- make_pages_present(addr, addr + len);
+ mlock_vma_pages_range(vma, addr, addr + len);
}
return addr;
}
@@ -2035,13 +2054,26 @@ EXPORT_SYMBOL(do_brk);
void exit_mmap(struct mm_struct *mm)
{
struct mmu_gather *tlb;
- struct vm_area_struct *vma = mm->mmap;
+ struct vm_area_struct *vma;
unsigned long nr_accounted = 0;
unsigned long end;
/* mm's last user has gone, and its about to be pulled down */
arch_exit_mmap(mm);
+ if (mm->locked_vm) {
+ vma = mm->mmap;
+ while (vma) {
+ if (vma->vm_flags & VM_LOCKED)
+ munlock_vma_pages_range(vma,
+ vma->vm_start, vma->vm_end);
+ vma = vma->vm_next;
+ }
+ }
+
+ vma = mm->mmap;
+
+
lru_add_drain();
flush_cache_mm(mm);
tlb = tlb_gather_mmu(mm, 1);
Index: Linux/mm/mremap.c
===================================================================
--- Linux.orig/mm/mremap.c 2007-11-12 16:21:48.000000000 -0500
+++ Linux/mm/mremap.c 2007-11-12 16:22:03.000000000 -0500
@@ -23,6 +23,8 @@
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
+#include "internal.h"
+
static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
@@ -232,8 +234,8 @@ static unsigned long move_vma(struct vm_
if (vm_flags & VM_LOCKED) {
mm->locked_vm += new_len >> PAGE_SHIFT;
if (new_len > old_len)
- make_pages_present(new_addr + old_len,
- new_addr + new_len);
+ mlock_vma_pages_range(vma, new_addr + old_len,
+ new_addr + new_len);
}
return new_addr;
@@ -373,7 +375,7 @@ unsigned long do_mremap(unsigned long ad
vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
if (vma->vm_flags & VM_LOCKED) {
mm->locked_vm += pages;
- make_pages_present(addr + old_len,
+ mlock_vma_pages_range(vma, addr + old_len,
addr + new_len);
}
ret = addr;
Index: Linux/mm/vmscan.c
===================================================================
--- Linux.orig/mm/vmscan.c 2007-11-12 16:21:59.000000000 -0500
+++ Linux/mm/vmscan.c 2007-11-12 16:22:03.000000000 -0500
@@ -534,6 +534,10 @@ static unsigned long shrink_page_list(st
goto activate_locked;
case SWAP_AGAIN:
goto keep_locked;
+ case SWAP_MLOCK:
+ ClearPageActive(page);
+ SetPageNoreclaim(page);
+ goto keep_locked; /* to noreclaim list */
case SWAP_SUCCESS:
; /* try to free the page below */
}
Index: Linux/mm/filemap.c
===================================================================
--- Linux.orig/mm/filemap.c 2007-11-12 16:21:48.000000000 -0500
+++ Linux/mm/filemap.c 2007-11-12 16:22:03.000000000 -0500
@@ -2504,8 +2504,16 @@ generic_file_direct_IO(int rw, struct ki
if (rw == WRITE) {
write_len = iov_length(iov, nr_segs);
end = (offset + write_len - 1) >> PAGE_CACHE_SHIFT;
- if (mapping_mapped(mapping))
+ if (mapping_mapped(mapping)) {
+ /*
+ * Calling unmap_mapping_range like this is wrong,
+ * because it can lead to mlocked pages being
+ * discarded (this is true even before the Noreclaim
+ * mlock work). direct-IO vs pagecache is a load of
+ * junk anyway, so who cares.
+ */
unmap_mapping_range(mapping, offset, write_len, 0);
+ }
}
retval = filemap_write_and_wait(mapping);
Index: Linux/mm/truncate.c
===================================================================
--- Linux.orig/mm/truncate.c 2007-11-12 16:21:48.000000000 -0500
+++ Linux/mm/truncate.c 2007-11-12 16:22:03.000000000 -0500
@@ -18,6 +18,7 @@
#include <linux/task_io_accounting_ops.h>
#include <linux/buffer_head.h> /* grr. try_to_release_page,
do_invalidatepage */
+#include "internal.h"
/**
@@ -104,6 +105,7 @@ truncate_complete_page(struct address_sp
do_invalidatepage(page, 0);
remove_from_page_cache(page);
+ clear_page_mlock(page);
ClearPageUptodate(page);
ClearPageMappedToDisk(page);
page_cache_release(page); /* pagecache ref */
@@ -128,6 +130,7 @@ invalidate_complete_page(struct address_
if (PagePrivate(page) && !try_to_release_page(page, 0))
return 0;
+ clear_page_mlock(page);
ret = remove_mapping(mapping, page);
return ret;
@@ -354,6 +357,7 @@ invalidate_complete_page2(struct address
if (PageDirty(page))
goto failed;
+ clear_page_mlock(page);
BUG_ON(PagePrivate(page));
__remove_from_page_cache(page);
write_unlock_irq(&mapping->tree_lock);
--
All Rights Reversed
next prev parent reply other threads:[~2007-12-18 21:21 UTC|newest]
Thread overview: 59+ messages / expand[flat|nested] mbox.gz Atom feed top
2007-12-18 21:15 [patch 00/20] VM pageout scalability improvements Rik van Riel
2007-12-18 21:15 ` [patch 01/20] convert anon_vma list lock a read/write lock Rik van Riel
2007-12-20 7:07 ` Christoph Lameter
2007-12-18 21:15 ` [patch 02/20] make the inode i_mmap_lock a reader/writer lock Rik van Riel
2007-12-19 0:48 ` Nick Piggin
2007-12-19 4:09 ` KOSAKI Motohiro
2007-12-19 15:52 ` Lee Schermerhorn
2007-12-19 16:31 ` Rik van Riel
2007-12-19 16:53 ` Lee Schermerhorn
2007-12-19 19:28 ` Peter Zijlstra
2007-12-19 23:40 ` Nick Piggin
2007-12-20 7:04 ` Christoph Lameter
2007-12-20 7:59 ` Nick Piggin
2008-01-02 23:35 ` Mike Travis
2008-01-03 6:07 ` Nick Piggin
2008-01-03 8:55 ` Ingo Molnar
2008-01-07 9:01 ` Nick Piggin
2007-12-18 21:15 ` [patch 03/20] move isolate_lru_page() to vmscan.c Rik van Riel
2007-12-20 7:08 ` Christoph Lameter
2007-12-18 21:15 ` [patch 04/20] free swap space on swap-in/activation Rik van Riel
2007-12-18 21:15 ` [patch 05/20] define page_file_cache() function Rik van Riel
2007-12-18 21:15 ` [patch 06/20] debugging checks for page_file_cache() Rik van Riel
2007-12-18 21:15 ` [patch 07/20] Use an indexed array for LRU variables Rik van Riel
2007-12-18 21:15 ` [patch 08/20] split LRU lists into anon & file sets Rik van Riel
2007-12-18 21:15 ` [patch 09/20] split anon & file LRUs for memcontrol code Rik van Riel
2007-12-18 21:15 ` [patch 10/20] SEQ replacement for anonymous pages Rik van Riel
2007-12-19 5:17 ` KOSAKI Motohiro
2007-12-19 13:40 ` Rik van Riel
2007-12-20 2:04 ` KOSAKI Motohiro
2007-12-18 21:15 ` [patch 11/20] add newly swapped in pages to the inactive list Rik van Riel
2007-12-18 21:15 ` [patch 12/20] No Reclaim LRU Infrastructure Rik van Riel
2007-12-18 21:15 ` [patch 13/20] Non-reclaimable page statistics Rik van Riel
2007-12-18 21:15 ` [patch 14/20] Scan noreclaim list for reclaimable pages Rik van Riel
2007-12-18 21:15 ` [patch 15/20] ramfs pages are non-reclaimable Rik van Riel
2007-12-18 21:15 ` [patch 16/20] SHM_LOCKED pages are nonreclaimable Rik van Riel
2007-12-18 21:15 ` [patch 17/20] non-reclaimable mlocked pages Rik van Riel
2007-12-19 0:56 ` Nick Piggin
2007-12-19 13:45 ` Rik van Riel
2007-12-19 14:24 ` Peter Zijlstra
2007-12-19 14:53 ` Rik van Riel
2007-12-19 16:08 ` Lee Schermerhorn
2007-12-19 16:04 ` Lee Schermerhorn
2007-12-20 20:56 ` Rik van Riel
2007-12-21 10:52 ` Nick Piggin
2007-12-21 14:17 ` Rik van Riel
2007-12-23 12:22 ` Nick Piggin
2007-12-24 1:00 ` Rik van Riel
2007-12-19 23:34 ` Nick Piggin
2007-12-20 7:19 ` Christoph Lameter
2007-12-20 15:33 ` Rik van Riel
2007-12-21 17:13 ` Lee Schermerhorn
2007-12-18 21:15 ` [patch 18/20] mlock vma pages under mmap_sem held for read Rik van Riel
2007-12-18 21:15 ` Rik van Riel [this message]
2007-12-18 21:15 ` [patch 20/20] account mlocked pages Rik van Riel
2007-12-22 20:27 ` [patch 00/20] VM pageout scalability improvements Balbir Singh
2007-12-23 0:21 ` Rik van Riel
2007-12-23 22:59 ` Balbir Singh
2007-12-24 1:11 ` Rik van Riel
2007-12-28 3:20 ` Matt Mackall
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20071218211550.392853742@redhat.com \
--to=riel@redhat.com \
--cc=lee.schermerhorn@hp.com \
--cc=lee.shermerhorn@hp.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).