From mboxrd@z Thu Jan 1 00:00:00 1970 From: "Jan Beulich" Subject: [PATCH] xmalloc: make close-to-PAGE_SIZE allocations more efficient Date: Mon, 18 Feb 2013 12:45:33 +0000 Message-ID: <5122307D02000078000BF20C@nat28.tlf.novell.com> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="=__Part44743A7D.0__=" Return-path: List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: xen-devel List-Id: xen-devel@lists.xenproject.org This is a MIME message. If you are reading this text, you may want to consider changing to a mail reader or gateway that understands how to properly handle MIME multipart messages. --=__Part44743A7D.0__= Content-Type: text/plain; charset=US-ASCII Content-Transfer-Encoding: quoted-printable Content-Disposition: inline Rather than bumping their sizes to slightly above (a multiple of) PAGE_SIZE (in order to store tracking information), thus requiring a non-order-0 allocation even when no more than a page is being requested, return the result of alloc_xenheap_pages() directly, and use the struct page_info field underlying PFN_ORDER() to store the actual size (needed for freeing the memory). This leverages the fact that sub-allocation of memory obtained from the page allocator can only ever result in non-page-aligned memory chunks (with the exception of zero size allocations with sufficiently high alignment being requested, which is why zero-size allocations now get special cased). Use the new property to simplify allocation of the trap info array for PV guests on x86. Signed-off-by: Jan Beulich --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -369,13 +369,6 @@ int switch_compat(struct domain *d) return -ENOMEM; } =20 -static inline bool_t standalone_trap_ctxt(struct vcpu *v) -{ - BUILD_BUG_ON(NR_VECTORS * sizeof(*v->arch.pv_vcpu.trap_ctxt) > = PAGE_SIZE); - return NR_VECTORS * sizeof(*v->arch.pv_vcpu.trap_ctxt) + sizeof(*v) - > PAGE_SIZE; -} - int vcpu_initialise(struct vcpu *v) { struct domain *d =3D v->domain; @@ -427,19 +420,15 @@ int vcpu_initialise(struct vcpu *v) =20 if ( !is_idle_domain(d) ) { - if ( standalone_trap_ctxt(v) ) + BUILD_BUG_ON(NR_VECTORS * sizeof(*v->arch.pv_vcpu.trap_ctxt) > + PAGE_SIZE); + v->arch.pv_vcpu.trap_ctxt =3D xzalloc_array(struct trap_info, + NR_VECTORS); + if ( !v->arch.pv_vcpu.trap_ctxt ) { - v->arch.pv_vcpu.trap_ctxt =3D alloc_xenheap_page(); - if ( !v->arch.pv_vcpu.trap_ctxt ) - { - rc =3D -ENOMEM; - goto done; - } - clear_page(v->arch.pv_vcpu.trap_ctxt); + rc =3D -ENOMEM; + goto done; } - else - v->arch.pv_vcpu.trap_ctxt =3D (void *)v + PAGE_SIZE - - NR_VECTORS * sizeof(*v->arch.pv_vcpu.trap_ctxt); =20 /* PV guests by default have a 100Hz ticker. */ v->periodic_period =3D MILLISECS(10); @@ -467,8 +456,8 @@ int vcpu_initialise(struct vcpu *v) { vcpu_destroy_fpu(v); =20 - if ( !is_hvm_domain(d) && standalone_trap_ctxt(v) ) - free_xenheap_page(v->arch.pv_vcpu.trap_ctxt); + if ( !is_hvm_domain(d) ) + xfree(v->arch.pv_vcpu.trap_ctxt); } =20 return rc; @@ -483,8 +472,8 @@ void vcpu_destroy(struct vcpu *v) =20 if ( is_hvm_vcpu(v) ) hvm_vcpu_destroy(v); - else if ( standalone_trap_ctxt(v) ) - free_xenheap_page(v->arch.pv_vcpu.trap_ctxt); + else + xfree(v->arch.pv_vcpu.trap_ctxt); } =20 int arch_domain_create(struct domain *d, unsigned int domcr_flags) --- a/xen/common/xmalloc_tlsf.c +++ b/xen/common/xmalloc_tlsf.c @@ -26,6 +26,7 @@ #include #include #include +#include #include =20 #define MAX_POOL_NAME_LEN 16 @@ -524,25 +525,30 @@ static void xmalloc_pool_put(void *p) free_xenheap_page(p); } =20 -static void *xmalloc_whole_pages(unsigned long size) +static void *xmalloc_whole_pages(unsigned long size, unsigned long align) { - struct bhdr *b; - unsigned int i, pageorder =3D get_order_from_bytes(size + BHDR_OVERHEA= D); - char *p; + unsigned int i, order =3D get_order_from_bytes(size); + void *res, *p; + + if ( align > size ) + get_order_from_bytes(align); =20 - b =3D alloc_xenheap_pages(pageorder, 0); - if ( b =3D=3D NULL ) + res =3D alloc_xenheap_pages(order, 0); + if ( res =3D=3D NULL ) return NULL; =20 - b->size =3D PAGE_ALIGN(size + BHDR_OVERHEAD); - for ( p =3D (char *)b + b->size, i =3D 0; i < pageorder; ++i ) + for ( p =3D res + PAGE_ALIGN(size), i =3D 0; i < order; ++i ) if ( (unsigned long)p & (PAGE_SIZE << i) ) { free_xenheap_pages(p, i); p +=3D PAGE_SIZE << i; } =20 - return (void *)b->ptr.buffer; + PFN_ORDER(virt_to_page(res)) =3D PFN_UP(size); + /* Check that there was no truncation: */ + ASSERT(PFN_ORDER(virt_to_page(res)) =3D=3D PFN_UP(size)); + + return res; } =20 static void tlsf_init(void) @@ -559,6 +565,11 @@ static void tlsf_init(void) * xmalloc() */ =20 +#ifndef ZERO_BLOCK_PTR +/* Return value for zero-size allocation, distinguished from NULL. */ +#define ZERO_BLOCK_PTR ((void *)-1L) +#endif + void *_xmalloc(unsigned long size, unsigned long align) { void *p =3D NULL; @@ -566,6 +577,9 @@ void *_xmalloc(unsigned long size, unsig =20 ASSERT(!in_irq()); =20 + if ( !size ) + return ZERO_BLOCK_PTR; + ASSERT((align & (align - 1)) =3D=3D 0); if ( align < MEM_ALIGN ) align =3D MEM_ALIGN; @@ -577,7 +591,7 @@ void *_xmalloc(unsigned long size, unsig if ( size < PAGE_SIZE ) p =3D xmem_pool_alloc(size, xenpool); if ( p =3D=3D NULL ) - p =3D xmalloc_whole_pages(size); + return xmalloc_whole_pages(size - align + MEM_ALIGN, align); =20 /* Add alignment padding. */ if ( (pad =3D -(long)p & (align - 1)) !=3D 0 ) @@ -604,11 +618,28 @@ void xfree(void *p) { struct bhdr *b; =20 - if ( p =3D=3D NULL ) + if ( p =3D=3D NULL || p =3D=3D ZERO_BLOCK_PTR ) return; =20 ASSERT(!in_irq()); =20 + if ( !((unsigned long)p & (PAGE_SIZE - 1)) ) + { + unsigned long size =3D PFN_ORDER(virt_to_page(p)); + unsigned int i, order =3D get_order_from_pages(size); + + BUG_ON((unsigned long)p & ((PAGE_SIZE << order) - 1)); + for ( i =3D 0; ; ++i ) + { + if ( !(size & (1 << i)) ) + continue; + size -=3D 1 << i; + free_xenheap_pages(p + (size << PAGE_SHIFT), i); + if ( i + 1 >=3D order ) + return; + } + } + /* Strip alignment padding. */ b =3D (struct bhdr *)((char *) p - BHDR_OVERHEAD); if ( b->size & 1 ) @@ -618,21 +649,5 @@ void xfree(void *p) ASSERT(!(b->size & 1)); } =20 - if ( b->size >=3D PAGE_SIZE ) - { - unsigned int i, order =3D get_order_from_bytes(b->size); - - BUG_ON((unsigned long)b & ((PAGE_SIZE << order) - 1)); - for ( i =3D 0; ; ++i ) - { - if ( !(b->size & (PAGE_SIZE << i)) ) - continue; - b->size -=3D PAGE_SIZE << i; - free_xenheap_pages((void *)b + b->size, i); - if ( i + 1 >=3D order ) - break; - } - } - else - xmem_pool_free(p, xenpool); + xmem_pool_free(p, xenpool); } --- a/xen/include/asm-x86/config.h +++ b/xen/include/asm-x86/config.h @@ -91,6 +91,9 @@ /* Primary stack is restricted to 8kB by guard pages. */ #define PRIMARY_STACK_SIZE 8192 =20 +/* Return value for zero-size _xmalloc(), distinguished from NULL. */ +#define ZERO_BLOCK_PTR ((void *)0xBAD0BAD0BAD0BAD0UL) + #ifndef __ASSEMBLY__ extern unsigned long trampoline_phys; #define bootsym_phys(sym) \ --=__Part44743A7D.0__= Content-Type: text/plain; name="xmalloc-exact-pages.patch" Content-Transfer-Encoding: quoted-printable Content-Disposition: attachment; filename="xmalloc-exact-pages.patch" xmalloc: make close-to-PAGE_SIZE allocations more efficient=0A=0ARather = than bumping their sizes to slightly above (a multiple of)=0APAGE_SIZE (in = order to store tracking information), thus requiring=0Aa non-order-0 = allocation even when no more than a page is being=0Arequested, return the = result of alloc_xenheap_pages() directly, and use=0Athe struct page_info = field underlying PFN_ORDER() to store the actual=0Asize (needed for = freeing the memory).=0A=0AThis leverages the fact that sub-allocation of = memory obtained from the=0Apage allocator can only ever result in = non-page-aligned memory chunks=0A(with the exception of zero size = allocations with sufficiently high=0Aalignment being requested, which is = why zero-size allocations now get=0Aspecial cased).=0A=0AUse the new = property to simplify allocation of the trap info array for=0APV guests on = x86.=0A=0ASigned-off-by: Jan Beulich =0A=0A--- = a/xen/arch/x86/domain.c=0A+++ b/xen/arch/x86/domain.c=0A@@ -369,13 +369,6 = @@ int switch_compat(struct domain *d)=0A return -ENOMEM;=0A }=0A = =0A-static inline bool_t standalone_trap_ctxt(struct vcpu *v)=0A-{=0A- = BUILD_BUG_ON(NR_VECTORS * sizeof(*v->arch.pv_vcpu.trap_ctxt) > PAGE_SIZE);= =0A- return NR_VECTORS * sizeof(*v->arch.pv_vcpu.trap_ctxt) + sizeof(*v)= =0A- > PAGE_SIZE;=0A-}=0A-=0A int vcpu_initialise(struct vcpu = *v)=0A {=0A struct domain *d =3D v->domain;=0A@@ -427,19 +420,15 @@ = int vcpu_initialise(struct vcpu *v)=0A =0A if ( !is_idle_domain(d) = )=0A {=0A- if ( standalone_trap_ctxt(v) )=0A+ BUILD_BUG_O= N(NR_VECTORS * sizeof(*v->arch.pv_vcpu.trap_ctxt) >=0A+ = PAGE_SIZE);=0A+ v->arch.pv_vcpu.trap_ctxt =3D xzalloc_array(struct = trap_info,=0A+ NR_VECTORS)= ;=0A+ if ( !v->arch.pv_vcpu.trap_ctxt )=0A {=0A- = v->arch.pv_vcpu.trap_ctxt =3D alloc_xenheap_page();=0A- if ( = !v->arch.pv_vcpu.trap_ctxt )=0A- {=0A- rc =3D = -ENOMEM;=0A- goto done;=0A- }=0A- = clear_page(v->arch.pv_vcpu.trap_ctxt);=0A+ rc =3D -ENOMEM;=0A+ = goto done;=0A }=0A- else=0A- v->arch.pv= _vcpu.trap_ctxt =3D (void *)v + PAGE_SIZE -=0A- NR_VECTORS = * sizeof(*v->arch.pv_vcpu.trap_ctxt);=0A =0A /* PV guests by = default have a 100Hz ticker. */=0A v->periodic_period =3D = MILLISECS(10);=0A@@ -467,8 +456,8 @@ int vcpu_initialise(struct vcpu = *v)=0A {=0A vcpu_destroy_fpu(v);=0A =0A- if ( !is_hvm_do= main(d) && standalone_trap_ctxt(v) )=0A- free_xenheap_page(v->ar= ch.pv_vcpu.trap_ctxt);=0A+ if ( !is_hvm_domain(d) )=0A+ = xfree(v->arch.pv_vcpu.trap_ctxt);=0A }=0A =0A return rc;=0A@@ = -483,8 +472,8 @@ void vcpu_destroy(struct vcpu *v)=0A =0A if ( = is_hvm_vcpu(v) )=0A hvm_vcpu_destroy(v);=0A- else if ( = standalone_trap_ctxt(v) )=0A- free_xenheap_page(v->arch.pv_vcpu.trap= _ctxt);=0A+ else=0A+ xfree(v->arch.pv_vcpu.trap_ctxt);=0A }=0A = =0A int arch_domain_create(struct domain *d, unsigned int domcr_flags)=0A--= - a/xen/common/xmalloc_tlsf.c=0A+++ b/xen/common/xmalloc_tlsf.c=0A@@ -26,6 = +26,7 @@=0A #include =0A #include =0A #include = =0A+#include =0A #include =0A =0A #define = MAX_POOL_NAME_LEN 16=0A@@ -524,25 +525,30 @@ static void xmalloc_pool= _put(void *p)=0A free_xenheap_page(p);=0A }=0A =0A-static void = *xmalloc_whole_pages(unsigned long size)=0A+static void *xmalloc_whole_page= s(unsigned long size, unsigned long align)=0A {=0A- struct bhdr *b;=0A- = unsigned int i, pageorder =3D get_order_from_bytes(size + BHDR_OVERHEAD)= ;=0A- char *p;=0A+ unsigned int i, order =3D get_order_from_bytes(siz= e);=0A+ void *res, *p;=0A+=0A+ if ( align > size )=0A+ = get_order_from_bytes(align);=0A =0A- b =3D alloc_xenheap_pages(pageorder= , 0);=0A- if ( b =3D=3D NULL )=0A+ res =3D alloc_xenheap_pages(order,= 0);=0A+ if ( res =3D=3D NULL )=0A return NULL;=0A =0A- = b->size =3D PAGE_ALIGN(size + BHDR_OVERHEAD);=0A- for ( p =3D (char *)b = + b->size, i =3D 0; i < pageorder; ++i )=0A+ for ( p =3D res + = PAGE_ALIGN(size), i =3D 0; i < order; ++i )=0A if ( (unsigned = long)p & (PAGE_SIZE << i) )=0A {=0A free_xenheap_pages(= p, i);=0A p +=3D PAGE_SIZE << i;=0A }=0A =0A- = return (void *)b->ptr.buffer;=0A+ PFN_ORDER(virt_to_page(res)) =3D = PFN_UP(size);=0A+ /* Check that there was no truncation: */=0A+ = ASSERT(PFN_ORDER(virt_to_page(res)) =3D=3D PFN_UP(size));=0A+=0A+ = return res;=0A }=0A =0A static void tlsf_init(void)=0A@@ -559,6 +565,11 @@ = static void tlsf_init(void)=0A * xmalloc()=0A */=0A =0A+#ifndef = ZERO_BLOCK_PTR=0A+/* Return value for zero-size allocation, distinguished = from NULL. */=0A+#define ZERO_BLOCK_PTR ((void *)-1L)=0A+#endif=0A+=0A = void *_xmalloc(unsigned long size, unsigned long align)=0A {=0A void = *p =3D NULL;=0A@@ -566,6 +577,9 @@ void *_xmalloc(unsigned long size, = unsig=0A =0A ASSERT(!in_irq());=0A =0A+ if ( !size )=0A+ = return ZERO_BLOCK_PTR;=0A+=0A ASSERT((align & (align - 1)) =3D=3D = 0);=0A if ( align < MEM_ALIGN )=0A align =3D MEM_ALIGN;=0A@@ = -577,7 +591,7 @@ void *_xmalloc(unsigned long size, unsig=0A if ( size = < PAGE_SIZE )=0A p =3D xmem_pool_alloc(size, xenpool);=0A if ( = p =3D=3D NULL )=0A- p =3D xmalloc_whole_pages(size);=0A+ = return xmalloc_whole_pages(size - align + MEM_ALIGN, align);=0A =0A /* = Add alignment padding. */=0A if ( (pad =3D -(long)p & (align - 1)) = !=3D 0 )=0A@@ -604,11 +618,28 @@ void xfree(void *p)=0A {=0A struct = bhdr *b;=0A =0A- if ( p =3D=3D NULL )=0A+ if ( p =3D=3D NULL || p = =3D=3D ZERO_BLOCK_PTR )=0A return;=0A =0A ASSERT(!in_irq());=0A= =0A+ if ( !((unsigned long)p & (PAGE_SIZE - 1)) )=0A+ {=0A+ = unsigned long size =3D PFN_ORDER(virt_to_page(p));=0A+ unsigned int = i, order =3D get_order_from_pages(size);=0A+=0A+ BUG_ON((unsigned = long)p & ((PAGE_SIZE << order) - 1));=0A+ for ( i =3D 0; ; ++i = )=0A+ {=0A+ if ( !(size & (1 << i)) )=0A+ = continue;=0A+ size -=3D 1 << i;=0A+ free_xenheap_page= s(p + (size << PAGE_SHIFT), i);=0A+ if ( i + 1 >=3D order )=0A+ = return;=0A+ }=0A+ }=0A+=0A /* Strip alignment = padding. */=0A b =3D (struct bhdr *)((char *) p - BHDR_OVERHEAD);=0A = if ( b->size & 1 )=0A@@ -618,21 +649,5 @@ void xfree(void *p)=0A = ASSERT(!(b->size & 1));=0A }=0A =0A- if ( b->size >=3D PAGE_SIZE = )=0A- {=0A- unsigned int i, order =3D get_order_from_bytes(b->siz= e);=0A-=0A- BUG_ON((unsigned long)b & ((PAGE_SIZE << order) - = 1));=0A- for ( i =3D 0; ; ++i )=0A- {=0A- if ( = !(b->size & (PAGE_SIZE << i)) )=0A- continue;=0A- = b->size -=3D PAGE_SIZE << i;=0A- free_xenheap_pages((void *)b = + b->size, i);=0A- if ( i + 1 >=3D order )=0A- = break;=0A- }=0A- }=0A- else=0A- xmem_pool_free(p, = xenpool);=0A+ xmem_pool_free(p, xenpool);=0A }=0A--- a/xen/include/asm-x= 86/config.h=0A+++ b/xen/include/asm-x86/config.h=0A@@ -91,6 +91,9 @@=0A /* = Primary stack is restricted to 8kB by guard pages. */=0A #define PRIMARY_ST= ACK_SIZE 8192=0A =0A+/* Return value for zero-size _xmalloc(), distinguishe= d from NULL. */=0A+#define ZERO_BLOCK_PTR ((void *)0xBAD0BAD0BAD0BAD0UL)=0A= +=0A #ifndef __ASSEMBLY__=0A extern unsigned long trampoline_phys;=0A = #define bootsym_phys(sym) \=0A --=__Part44743A7D.0__= Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: inline _______________________________________________ Xen-devel mailing list Xen-devel@lists.xen.org http://lists.xen.org/xen-devel --=__Part44743A7D.0__=--