From: Christoph Lameter <clameter@sgi.com>
To: Linus Torvalds <torvalds@osdl.org>
Cc: Hugh Dickins <hugh@veritas.com>, Andrew Morton <akpm@osdl.org>,
"David S. Miller" <davem@davemloft.net>,
linux-ia64@vger.kernel.org, linux-mm@kvack.org,
Linux Kernel Development <linux-kernel@vger.kernel.org>
Subject: Prezeroing V4 [1/4]: Arch specific page zeroing during page fault
Date: Mon, 10 Jan 2005 15:54:08 -0800 (PST) [thread overview]
Message-ID: <Pine.LNX.4.58.0501101553140.25654@schroedinger.engr.sgi.com> (raw)
In-Reply-To: <Pine.LNX.4.58.0501101552100.25654@schroedinger.engr.sgi.com>
This patch fixes the __GFP_ZERO related code by adding a new function
alloc_zeroed_user_highpage that is then used in the anonymous page fault
handler and in the COW code to allocate pages. The function can be defined
per arch to setup special processing for user pages by defining
__HAVE_ARCH_ALLOC_ZEROED_USER_PAGE.
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Index: linux-2.6.10/include/asm-ia64/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-ia64/page.h 2004-12-24 13:34:00.000000000 -0800
+++ linux-2.6.10/include/asm-ia64/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -75,6 +75,16 @@
flush_dcache_page(page); \
} while (0)
+
+#define alloc_zeroed_user_highpage(vma, vaddr) \
+({ \
+ struct page *page = alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr); \
+ flush_dcache_page(page); \
+ page; \
+})
+
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+
#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
#ifdef CONFIG_VIRTUAL_MEM_MAP
Index: linux-2.6.10/include/asm-h8300/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-h8300/page.h 2004-12-24 13:35:25.000000000 -0800
+++ linux-2.6.10/include/asm-h8300/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -30,6 +30,9 @@
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
+#define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr)
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+
/*
* These are used to make use of C type-checking..
*/
Index: linux-2.6.10/mm/memory.c
===================================================================
--- linux-2.6.10.orig/mm/memory.c 2005-01-10 13:48:11.000000000 -0800
+++ linux-2.6.10/mm/memory.c 2005-01-10 13:54:30.000000000 -0800
@@ -84,20 +84,6 @@
EXPORT_SYMBOL(vmalloc_earlyreserve);
/*
- * We special-case the C-O-W ZERO_PAGE, because it's such
- * a common occurrence (no need to read the page to know
- * that it's zero - better for the cache and memory subsystem).
- */
-static inline void copy_cow_page(struct page * from, struct page * to, unsigned long address)
-{
- if (from == ZERO_PAGE(address)) {
- clear_user_highpage(to, address);
- return;
- }
- copy_user_highpage(to, from, address);
-}
-
-/*
* Note: this doesn't free the actual pages themselves. That
* has been handled earlier when unmapping all the memory regions.
*/
@@ -1329,11 +1315,16 @@
if (unlikely(anon_vma_prepare(vma)))
goto no_new_page;
- new_page = alloc_page_vma(GFP_HIGHUSER, vma, address);
- if (!new_page)
- goto no_new_page;
- copy_cow_page(old_page,new_page,address);
-
+ if (old_page == ZERO_PAGE(address)) {
+ new_page = alloc_zeroed_user_highpage(vma, address);
+ if (!new_page)
+ goto no_new_page;
+ } else {
+ new_page = alloc_page_vma(GFP_HIGHUSER, vma, address);
+ if (!new_page)
+ goto no_new_page;
+ copy_user_highpage(new_page, old_page, address);
+ }
/*
* Re-check the pte - we dropped the lock
*/
@@ -1795,7 +1786,7 @@
if (unlikely(anon_vma_prepare(vma)))
goto no_mem;
- page = alloc_page_vma(GFP_HIGHZERO, vma, addr);
+ page = alloc_zeroed_user_highpage(vma, addr);
if (!page)
goto no_mem;
Index: linux-2.6.10/include/asm-m32r/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-m32r/page.h 2004-12-24 13:34:29.000000000 -0800
+++ linux-2.6.10/include/asm-m32r/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -17,6 +17,9 @@
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
+#define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr)
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+
/*
* These are used to make use of C type-checking..
*/
Index: linux-2.6.10/include/asm-alpha/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-alpha/page.h 2004-12-24 13:35:24.000000000 -0800
+++ linux-2.6.10/include/asm-alpha/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -18,6 +18,9 @@
extern void clear_page(void *page);
#define clear_user_page(page, vaddr, pg) clear_page(page)
+#define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vmaddr)
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+
extern void copy_page(void * _to, void * _from);
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
Index: linux-2.6.10/include/asm-m68knommu/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-m68knommu/page.h 2005-01-10 13:48:11.000000000 -0800
+++ linux-2.6.10/include/asm-m68knommu/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -30,6 +30,9 @@
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
+#define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr)
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+
/*
* These are used to make use of C type-checking..
*/
Index: linux-2.6.10/include/asm-cris/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-cris/page.h 2004-12-24 13:34:30.000000000 -0800
+++ linux-2.6.10/include/asm-cris/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -21,6 +21,9 @@
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
+#define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr)
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+
/*
* These are used to make use of C type-checking..
*/
Index: linux-2.6.10/include/linux/highmem.h
===================================================================
--- linux-2.6.10.orig/include/linux/highmem.h 2005-01-10 13:48:11.000000000 -0800
+++ linux-2.6.10/include/linux/highmem.h 2005-01-10 13:53:59.000000000 -0800
@@ -42,6 +42,17 @@
smp_wmb();
}
+#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+static inline struct page* alloc_zeroed_user_highpage(struct vm_area_struct *vma,
+ unsigned long vaddr)
+{
+ struct page *page = alloc_page_vma(GFP_HIGHUSER, vma, vaddr);
+
+ clear_user_highpage(page, vaddr);
+ return page;
+}
+#endif
+
static inline void clear_highpage(struct page *page)
{
void *kaddr = kmap_atomic(page, KM_USER0);
Index: linux-2.6.10/include/asm-i386/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-i386/page.h 2005-01-10 13:48:11.000000000 -0800
+++ linux-2.6.10/include/asm-i386/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -36,6 +36,9 @@
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
+#define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr)
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+
/*
* These are used to make use of C type-checking..
*/
Index: linux-2.6.10/include/asm-x86_64/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-x86_64/page.h 2005-01-10 13:48:11.000000000 -0800
+++ linux-2.6.10/include/asm-x86_64/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -38,6 +38,8 @@
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
+#define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr)
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
/*
* These are used to make use of C type-checking..
*/
Index: linux-2.6.10/include/asm-s390/page.h
===================================================================
--- linux-2.6.10.orig/include/asm-s390/page.h 2004-12-24 13:34:01.000000000 -0800
+++ linux-2.6.10/include/asm-s390/page.h 2005-01-10 13:53:59.000000000 -0800
@@ -106,6 +106,9 @@
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
+#define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr)
+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
+
/* Pure 2^n version of get_order */
extern __inline__ int get_order(unsigned long size)
{
next prev parent reply other threads:[~2005-01-10 23:57 UTC|newest]
Thread overview: 87+ messages / expand[flat|nested] mbox.gz Atom feed top
[not found] <B8E391BBE9FE384DAA4C5C003888BE6F02900FBD@scsmsx401.amr.corp.intel.com>
[not found] ` <41C20E3E.3070209@yahoo.com.au>
2004-12-21 19:55 ` Increase page fault rate by prezeroing V1 [0/3]: Overview Christoph Lameter
2004-12-21 19:56 ` Increase page fault rate by prezeroing V1 [1/3]: Introduce __GFP_ZERO Christoph Lameter
2004-12-21 19:57 ` Increase page fault rate by prezeroing V1 [2/3]: zeroing and scrubd Christoph Lameter
2005-01-01 2:22 ` Nick Piggin
2005-01-01 2:55 ` pmarques
2004-12-21 19:57 ` Increase page fault rate by prezeroing V1 [3/3]: Altix SN2 BTE Zeroing Christoph Lameter
2004-12-22 12:46 ` Robin Holt
2004-12-22 19:56 ` Christoph Lameter
2004-12-23 19:29 ` Prezeroing V2 [0/3]: Why and When it works Christoph Lameter
2004-12-23 19:33 ` Prezeroing V2 [1/4]: __GFP_ZERO / clear_page() removal Christoph Lameter
2004-12-23 19:33 ` Prezeroing V2 [2/4]: add second parameter to clear_page() for all arches Christoph Lameter
2004-12-24 8:33 ` Pavel Machek
2004-12-24 16:18 ` Christoph Lameter
2004-12-24 16:27 ` Pavel Machek
2004-12-24 17:02 ` David S. Miller
2004-12-24 17:05 ` David S. Miller
2004-12-27 22:48 ` David S. Miller
2005-01-03 17:52 ` Christoph Lameter
2005-01-01 10:24 ` Geert Uytterhoeven
2005-01-04 23:12 ` Prezeroing V3 [0/4]: Discussion and i386 performance tests Christoph Lameter
2005-01-04 23:13 ` Prezeroing V3 [1/4]: Allow request for zeroed memory Christoph Lameter
2005-01-04 23:45 ` Dave Hansen
2005-01-05 1:16 ` Christoph Lameter
2005-01-05 1:26 ` Linus Torvalds
2005-01-05 23:11 ` Christoph Lameter
2005-01-05 0:34 ` Linus Torvalds
2005-01-05 0:47 ` Andrew Morton
2005-01-05 1:15 ` Christoph Lameter
2005-01-08 21:12 ` Hugh Dickins
2005-01-08 21:56 ` David S. Miller
2005-01-21 20:09 ` alloc_zeroed_user_highpage to fix the clear_user_highpage issue Christoph Lameter
2005-02-09 9:58 ` [Patch] Fix oops in alloc_zeroed_user_highpage() when page is NULL Michael Ellerman
2005-02-10 0:38 ` Christoph Lameter
2005-01-21 20:12 ` Extend clear_page by an order parameter Christoph Lameter
2005-01-21 22:29 ` Paul Mackerras
2005-01-21 23:48 ` Christoph Lameter
2005-01-22 0:35 ` Paul Mackerras
2005-01-22 0:43 ` Andrew Morton
2005-01-22 1:08 ` Paul Mackerras
2005-01-22 1:20 ` Roman Zippel
2005-01-22 1:25 ` Paul Mackerras
2005-01-22 1:54 ` Christoph Lameter
2005-01-22 2:53 ` Paul Mackerras
2005-01-23 7:45 ` Andrew Morton
2005-01-24 16:37 ` Christoph Lameter
2005-01-24 20:23 ` David S. Miller
2005-01-24 20:33 ` Christoph Lameter
2005-01-21 20:15 ` A scrub daemon (prezeroing) Christoph Lameter
2005-01-10 17:16 ` Prezeroing V3 [1/4]: Allow request for zeroed memory Christoph Lameter
2005-01-10 18:13 ` Linus Torvalds
2005-01-10 20:17 ` Christoph Lameter
2005-01-10 23:53 ` Prezeroing V4 [0/4]: Overview Christoph Lameter
2005-01-10 23:54 ` Christoph Lameter [this message]
2005-01-11 0:41 ` Prezeroing V4 [1/4]: Arch specific page zeroing during page fault Chris Wright
2005-01-11 0:46 ` Christoph Lameter
2005-01-11 0:49 ` Chris Wright
2005-01-10 23:55 ` Prezeroing V4 [2/4]: Zeroing implementation Christoph Lameter
2005-01-10 23:55 ` Prezeroing V4 [3/4]: Altix SN2 BTE zero driver Christoph Lameter
2005-01-10 23:56 ` Prezeroing V4 [4/4]: Extend clear_page to take an order parameter Christoph Lameter
2005-01-04 23:14 ` Prezeroing V3 [2/4]: Extension of " Christoph Lameter
2005-01-05 23:25 ` Christoph Lameter
2005-01-06 13:52 ` Andi Kleen
2005-01-06 17:47 ` Christoph Lameter
2005-01-04 23:15 ` Prezeroing V3 [3/4]: Page zeroing through kscrubd Christoph Lameter
2005-01-04 23:16 ` Prezeroing V3 [4/4]: Driver for hardware zeroing on Altix Christoph Lameter
2005-01-05 2:16 ` Andi Kleen
2005-01-05 16:24 ` Christoph Lameter
2004-12-23 19:34 ` Prezeroing V2 [3/4]: Add support for ZEROED and NOT_ZEROED free maps Christoph Lameter
2004-12-23 19:35 ` Prezeroing V2 [4/4]: Hardware Zeroing through SGI BTE Christoph Lameter
2004-12-23 20:08 ` Prezeroing V2 [1/4]: __GFP_ZERO / clear_page() removal Brian Gerst
2004-12-24 16:24 ` Christoph Lameter
2004-12-23 19:49 ` Prezeroing V2 [0/3]: Why and When it works Arjan van de Ven
2004-12-23 20:57 ` Matt Mackall
2004-12-23 21:01 ` Paul Mackerras
2004-12-23 21:11 ` Paul Mackerras
2004-12-23 21:37 ` Andrew Morton
2004-12-23 23:00 ` Paul Mackerras
2004-12-23 21:48 ` Linus Torvalds
2004-12-23 22:34 ` Zwane Mwaikambo
2004-12-24 9:14 ` Arjan van de Ven
2004-12-24 18:21 ` Linus Torvalds
2004-12-24 18:57 ` Arjan van de Ven
2004-12-27 22:50 ` David S. Miller
2004-12-28 11:53 ` Marcelo Tosatti
2004-12-24 16:17 ` Christoph Lameter
2004-12-24 18:31 ` Increase page fault rate by prezeroing V1 [0/3]: Overview Andrea Arcangeli
2005-01-03 17:54 ` Christoph Lameter
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=Pine.LNX.4.58.0501101553140.25654@schroedinger.engr.sgi.com \
--to=clameter@sgi.com \
--cc=akpm@osdl.org \
--cc=davem@davemloft.net \
--cc=hugh@veritas.com \
--cc=linux-ia64@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=torvalds@osdl.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).