linux: try harder to balloon up under memory pressure.

linux: try harder to balloon up under memory pressure.

[Ian Campbell]

Currently if the balloon driver is unable to increase the guest's
reservation it assumes the failure was due to reaching its full
allocation, gives up on the ballooning operation and records the limit
it reached as the "hard limit". The driver will not try again until
the target is set again (even to the same value).

However it is possible that ballooning has in fact failed due to
memory pressure in the host and therefore it is desirable to keep
attempting to reach the target in case memory becomes available. The
most likely scenario is that some guests are ballooning down while
others are ballooning up and therefore there is temporary memory
pressure while things stabilise. You would not expect a well behaved
toolstack to ask a domain to balloon to more than its allocation nor
would you expect it to deliberately over-commit memory by setting
balloon targets which exceed the total host memory.

This patch drops the concept of a hard limit and causes the balloon
driver to retry increasing the reservation on a timer in the same
manner as when decreasing the reservation.

Also if we partially succeed in increasing the reservation
(i.e. receive less pages than we asked for) then we may as well keep
those pages rather than returning them to Xen.

This patch is for the 2.6.18-xen tree.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>

diff -r fd9c565657b8 drivers/xen/balloon/balloon.c
--- a/drivers/xen/balloon/balloon.c	Thu Jun 04 16:26:49 2009 +0100
+++ b/drivers/xen/balloon/balloon.c	Fri Jun 05 11:56:22 2009 +0100
@@ -188,7 +188,7 @@
 
 static unsigned long current_target(void)
 {
-	unsigned long target = min(bs.target_pages, bs.hard_limit);
+	unsigned long target = bs.target_pages;
 	if (target > (bs.current_pages + bs.balloon_low + bs.balloon_high))
 		target = bs.current_pages + bs.balloon_low + bs.balloon_high;
 	return target;
@@ -255,26 +255,12 @@
 	}
 
 	set_xen_guest_handle(reservation.extent_start, frame_list);
-	reservation.nr_extents   = nr_pages;
-	rc = HYPERVISOR_memory_op(
-		XENMEM_populate_physmap, &reservation);
-	if (rc < nr_pages) {
-		if (rc > 0) {
-			int ret;
+	reservation.nr_extents = nr_pages;
+	rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation);
+	if (rc < 0)
+		goto out;
 
-			/* We hit the Xen hard limit: reprobe. */
-			reservation.nr_extents = rc;
-			ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
-					&reservation);
-			BUG_ON(ret != rc);
-		}
-		if (rc >= 0)
-			bs.hard_limit = (bs.current_pages + rc -
-					 bs.driver_pages);
-		goto out;
-	}
-
-	for (i = 0; i < nr_pages; i++) {
+	for (i = 0; i < rc; i++) {
 		page = balloon_retrieve();
 		BUG_ON(page == NULL);
 
@@ -302,13 +288,13 @@
 		balloon_free_page(page);
 	}
 
-	bs.current_pages += nr_pages;
+	bs.current_pages += rc;
 	totalram_pages = bs.current_pages;
 
  out:
 	balloon_unlock(flags);
 
-	return 0;
+	return rc < 0 ? rc : rc != nr_pages;
 }
 
 static int decrease_reservation(unsigned long nr_pages)
@@ -420,7 +406,6 @@
 void balloon_set_new_target(unsigned long target)
 {
 	/* No need for lock. Not read-modify-write updates. */
-	bs.hard_limit   = ~0UL;
 	bs.target_pages = max(target, minimum_target());
 	schedule_work(&balloon_worker);
 }
@@ -498,17 +483,11 @@
 		"Requested target:   %8lu kB\n"
 		"Low-mem balloon:    %8lu kB\n"
 		"High-mem balloon:   %8lu kB\n"
-		"Driver pages:       %8lu kB\n"
-		"Xen hard limit:     ",
+		"Driver pages:       %8lu kB\n",
 		PAGES2KB(bs.current_pages), PAGES2KB(bs.target_pages), 
 		PAGES2KB(bs.balloon_low), PAGES2KB(bs.balloon_high),
 		PAGES2KB(bs.driver_pages));
 
-	if (bs.hard_limit != ~0UL)
-		len += sprintf(page + len, "%8lu kB\n",
-			       PAGES2KB(bs.hard_limit));
-	else
-		len += sprintf(page + len, "     ??? kB\n");
 
 	*eof = 1;
 	return len;
@@ -539,7 +518,6 @@
 	bs.balloon_low   = 0;
 	bs.balloon_high  = 0;
 	bs.driver_pages  = 0UL;
-	bs.hard_limit    = ~0UL;
 
 	init_timer(&balloon_timer);
 	balloon_timer.data = 0;
diff -r fd9c565657b8 drivers/xen/balloon/common.h
--- a/drivers/xen/balloon/common.h	Thu Jun 04 16:26:49 2009 +0100
+++ b/drivers/xen/balloon/common.h	Fri Jun 05 11:56:22 2009 +0100
@@ -35,8 +35,6 @@
 	/* We aim for 'current allocation' == 'target allocation'. */
 	unsigned long current_pages;
 	unsigned long target_pages;
-	/* We may hit the hard limit in Xen. If we do then we remember it. */
-	unsigned long hard_limit;
 	/*
 	 * Drivers may alter the memory reservation independently, but they
 	 * must inform the balloon driver so we avoid hitting the hard limit.
diff -r fd9c565657b8 drivers/xen/balloon/sysfs.c
--- a/drivers/xen/balloon/sysfs.c	Thu Jun 04 16:26:49 2009 +0100
+++ b/drivers/xen/balloon/sysfs.c	Fri Jun 05 11:56:22 2009 +0100
@@ -53,9 +53,6 @@
 BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(bs.current_pages));
 BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(bs.balloon_low));
 BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(bs.balloon_high));
-BALLOON_SHOW(hard_limit_kb,
-	     (bs.hard_limit!=~0UL) ? "%lu\n" : "???\n",
-	     (bs.hard_limit!=~0UL) ? PAGES2KB(bs.hard_limit) : 0);
 BALLOON_SHOW(driver_kb, "%lu\n", PAGES2KB(bs.driver_pages));
 
 static ssize_t show_target_kb(struct sys_device *dev, char *buf)
@@ -96,7 +93,6 @@
 	&attr_current_kb.attr,
 	&attr_low_kb.attr,
 	&attr_high_kb.attr,
-	&attr_hard_limit_kb.attr,
 	&attr_driver_kb.attr,
 	NULL
 };

Re: linux: try harder to balloon up under memory pressure.

[Ian Campbell]

On Fri, 2009-06-05 at 06:57 -0400, Ian Campbell wrote:
> 
> This patch is for the 2.6.18-xen tree.

Same patch for pvops kernel

Subject: xen: try harder to balloon up under memory pressure.

Currently if the balloon driver is unable to increase the guest's
reservation it assumes the failure was due to reaching its full
allocation, gives up on the ballooning operation and records the limit
it reached as the "hard limit". The driver will not try again until
the target is set again (even to the same value).

However it is possible that ballooning has in fact failed due to
memory pressure in the host and therefore it is desirable to keep
attempting to reach the target in case memory becomes available. The
most likely scenario is that some guests are ballooning down while
others are ballooning up and therefore there is temporary memory
pressure while things stabilise. You would not expect a well behaved
toolstack to ask a domain to balloon to more than its allocation nor
would you expect it to deliberately over-commit memory by setting
balloon targets which exceed the total host memory.

This patch drops the concept of a hard limit and causes the balloon
driver to retry increasing the reservation on a timer in the same
manner as when decreasing the reservation.

Also if we partially succeed in increasing the reservation
(i.e. receive less pages than we asked for) then we may as well keep
those pages rather than returning them to Xen.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
 drivers/xen/balloon.c |   31 +++++--------------------------
 1 files changed, 5 insertions(+), 26 deletions(-)

diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index c83da03..5287d09 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -66,8 +66,6 @@ struct balloon_stats {
 	/* We aim for 'current allocation' == 'target allocation'. */
 	unsigned long current_pages;
 	unsigned long target_pages;
-	/* We may hit the hard limit in Xen. If we do then we remember it. */
-	unsigned long hard_limit;
 	/*
 	 * Drivers may alter the memory reservation independently, but they
 	 * must inform the balloon driver so we avoid hitting the hard limit.
@@ -178,7 +176,7 @@ static void balloon_alarm(unsigned long unused)
 
 static unsigned long current_target(void)
 {
-	unsigned long target = min(balloon_stats.target_pages, balloon_stats.hard_limit);
+	unsigned long target = balloon_stats.target_pages;
 
 	target = min(target,
 		     balloon_stats.current_pages +
@@ -214,23 +212,10 @@ static int increase_reservation(unsigned long nr_pages)
 	set_xen_guest_handle(reservation.extent_start, frame_list);
 	reservation.nr_extents = nr_pages;
 	rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation);
-	if (rc < nr_pages) {
-		if (rc > 0) {
-			int ret;
-
-			/* We hit the Xen hard limit: reprobe. */
-			reservation.nr_extents = rc;
-			ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
-						   &reservation);
-			BUG_ON(ret != rc);
-		}
-		if (rc >= 0)
-			balloon_stats.hard_limit = (balloon_stats.current_pages + rc -
-						    balloon_stats.driver_pages);
+	if (rc < 0)
 		goto out;
-	}
 
-	for (i = 0; i < nr_pages; i++) {
+	for (i = 0; i < rc; i++) {
 		page = balloon_retrieve();
 		BUG_ON(page == NULL);
 
@@ -256,13 +241,13 @@ static int increase_reservation(unsigned long nr_pages)
 		__free_page(page);
 	}
 
-	balloon_stats.current_pages += nr_pages;
+	balloon_stats.current_pages += rc;
 	totalram_pages = balloon_stats.current_pages;
 
  out:
 	spin_unlock_irqrestore(&xen_reservation_lock, flags);
 
-	return 0;
+	return rc < 0 ? rc : rc != nr_pages;
 }
 
 static int decrease_reservation(unsigned long nr_pages)
@@ -364,7 +349,6 @@ static void balloon_process(struct work_struct *work)
 static void balloon_set_new_target(unsigned long target)
 {
 	/* No need for lock. Not read-modify-write updates. */
-	balloon_stats.hard_limit   = ~0UL;
 	balloon_stats.target_pages = target;
 	schedule_work(&balloon_worker);
 }
@@ -424,7 +408,6 @@ static int __init balloon_init(void)
 	balloon_stats.balloon_low   = 0;
 	balloon_stats.balloon_high  = 0;
 	balloon_stats.driver_pages  = 0UL;
-	balloon_stats.hard_limit    = ~0UL;
 
 	init_timer(&balloon_timer);
 	balloon_timer.data = 0;
@@ -564,9 +547,6 @@ EXPORT_SYMBOL_GPL(free_empty_pages_and_pagevec);
 BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(balloon_stats.current_pages));
 BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_low));
 BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_high));
-BALLOON_SHOW(hard_limit_kb,
-	     (balloon_stats.hard_limit!=~0UL) ? "%lu\n" : "???\n",
-	     (balloon_stats.hard_limit!=~0UL) ? PAGES2KB(balloon_stats.hard_limit) : 0);
 BALLOON_SHOW(driver_kb, "%lu\n", PAGES2KB(balloon_stats.driver_pages));
 
 static ssize_t show_target_kb(struct sys_device *dev, struct sysdev_attribute *attr,
@@ -635,7 +615,6 @@ static struct attribute *balloon_info_attrs[] = {
 	&attr_current_kb.attr,
 	&attr_low_kb.attr,
 	&attr_high_kb.attr,
-	&attr_hard_limit_kb.attr,
 	&attr_driver_kb.attr,
 	NULL
 };
-- 
1.5.6.5

RE: linux: try harder to balloon up under memory pressure.

[Dan Magenheimer]

Excellent!  I think this explains a weirdness I saw a year
ago when working on self-ballooning, and also a problem
I saw under certain conditions for tmem, that I had
communicated offline to Jeremy last month (because I was
looking at the pvops driver at the time) but never got
around to understanding in detail.

Suggestion:  Perhaps the balloon_timer interval should
be configurable via sysfs?

> -----Original Message-----
> From: Ian Campbell [mailto:Ian.Campbell@eu.citrix.com]
> Sent: Friday, June 05, 2009 4:57 AM
> To: xen-devel
> Cc: Keir Fraser
> Subject: [Xen-devel] linux: try harder to balloon up under memory
> pressure.
> 
> 
> Currently if the balloon driver is unable to increase the guest's
> reservation it assumes the failure was due to reaching its full
> allocation, gives up on the ballooning operation and records the limit
> it reached as the "hard limit". The driver will not try again until
> the target is set again (even to the same value).
> 
> However it is possible that ballooning has in fact failed due to
> memory pressure in the host and therefore it is desirable to keep
> attempting to reach the target in case memory becomes available. The
> most likely scenario is that some guests are ballooning down while
> others are ballooning up and therefore there is temporary memory
> pressure while things stabilise. You would not expect a well behaved
> toolstack to ask a domain to balloon to more than its allocation nor
> would you expect it to deliberately over-commit memory by setting
> balloon targets which exceed the total host memory.
> 
> This patch drops the concept of a hard limit and causes the balloon
> driver to retry increasing the reservation on a timer in the same
> manner as when decreasing the reservation.
> 
> Also if we partially succeed in increasing the reservation
> (i.e. receive less pages than we asked for) then we may as well keep
> those pages rather than returning them to Xen.
> 
> This patch is for the 2.6.18-xen tree.
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> 
> diff -r fd9c565657b8 drivers/xen/balloon/balloon.c
> --- a/drivers/xen/balloon/balloon.c	Thu Jun 04 16:26:49 2009 +0100
> +++ b/drivers/xen/balloon/balloon.c	Fri Jun 05 11:56:22 2009 +0100
> @@ -188,7 +188,7 @@
>  
>  static unsigned long current_target(void)
>  {
> -	unsigned long target = min(bs.target_pages, bs.hard_limit);
> +	unsigned long target = bs.target_pages;
>  	if (target > (bs.current_pages + bs.balloon_low + 
> bs.balloon_high))
>  		target = bs.current_pages + bs.balloon_low + 
> bs.balloon_high;
>  	return target;
> @@ -255,26 +255,12 @@
>  	}
>  
>  	set_xen_guest_handle(reservation.extent_start, frame_list);
> -	reservation.nr_extents   = nr_pages;
> -	rc = HYPERVISOR_memory_op(
> -		XENMEM_populate_physmap, &reservation);
> -	if (rc < nr_pages) {
> -		if (rc > 0) {
> -			int ret;
> +	reservation.nr_extents = nr_pages;
> +	rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, 
> &reservation);
> +	if (rc < 0)
> +		goto out;
>  
> -			/* We hit the Xen hard limit: reprobe. */
> -			reservation.nr_extents = rc;
> -			ret = 
> HYPERVISOR_memory_op(XENMEM_decrease_reservation,
> -					&reservation);
> -			BUG_ON(ret != rc);
> -		}
> -		if (rc >= 0)
> -			bs.hard_limit = (bs.current_pages + rc -
> -					 bs.driver_pages);
> -		goto out;
> -	}
> -
> -	for (i = 0; i < nr_pages; i++) {
> +	for (i = 0; i < rc; i++) {
>  		page = balloon_retrieve();
>  		BUG_ON(page == NULL);
>  
> @@ -302,13 +288,13 @@
>  		balloon_free_page(page);
>  	}
>  
> -	bs.current_pages += nr_pages;
> +	bs.current_pages += rc;
>  	totalram_pages = bs.current_pages;
>  
>   out:
>  	balloon_unlock(flags);
>  
> -	return 0;
> +	return rc < 0 ? rc : rc != nr_pages;
>  }
>  
>  static int decrease_reservation(unsigned long nr_pages)
> @@ -420,7 +406,6 @@
>  void balloon_set_new_target(unsigned long target)
>  {
>  	/* No need for lock. Not read-modify-write updates. */
> -	bs.hard_limit   = ~0UL;
>  	bs.target_pages = max(target, minimum_target());
>  	schedule_work(&balloon_worker);
>  }
> @@ -498,17 +483,11 @@
>  		"Requested target:   %8lu kB\n"
>  		"Low-mem balloon:    %8lu kB\n"
>  		"High-mem balloon:   %8lu kB\n"
> -		"Driver pages:       %8lu kB\n"
> -		"Xen hard limit:     ",
> +		"Driver pages:       %8lu kB\n",
>  		PAGES2KB(bs.current_pages), PAGES2KB(bs.target_pages), 
>  		PAGES2KB(bs.balloon_low), PAGES2KB(bs.balloon_high),
>  		PAGES2KB(bs.driver_pages));
>  
> -	if (bs.hard_limit != ~0UL)
> -		len += sprintf(page + len, "%8lu kB\n",
> -			       PAGES2KB(bs.hard_limit));
> -	else
> -		len += sprintf(page + len, "     ??? kB\n");
>  
>  	*eof = 1;
>  	return len;
> @@ -539,7 +518,6 @@
>  	bs.balloon_low   = 0;
>  	bs.balloon_high  = 0;
>  	bs.driver_pages  = 0UL;
> -	bs.hard_limit    = ~0UL;
>  
>  	init_timer(&balloon_timer);
>  	balloon_timer.data = 0;
> diff -r fd9c565657b8 drivers/xen/balloon/common.h
> --- a/drivers/xen/balloon/common.h	Thu Jun 04 16:26:49 2009 +0100
> +++ b/drivers/xen/balloon/common.h	Fri Jun 05 11:56:22 2009 +0100
> @@ -35,8 +35,6 @@
>  	/* We aim for 'current allocation' == 'target allocation'. */
>  	unsigned long current_pages;
>  	unsigned long target_pages;
> -	/* We may hit the hard limit in Xen. If we do then we 
> remember it. */
> -	unsigned long hard_limit;
>  	/*
>  	 * Drivers may alter the memory reservation 
> independently, but they
>  	 * must inform the balloon driver so we avoid hitting 
> the hard limit.
> diff -r fd9c565657b8 drivers/xen/balloon/sysfs.c
> --- a/drivers/xen/balloon/sysfs.c	Thu Jun 04 16:26:49 2009 +0100
> +++ b/drivers/xen/balloon/sysfs.c	Fri Jun 05 11:56:22 2009 +0100
> @@ -53,9 +53,6 @@
>  BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(bs.current_pages));
>  BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(bs.balloon_low));
>  BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(bs.balloon_high));
> -BALLOON_SHOW(hard_limit_kb,
> -	     (bs.hard_limit!=~0UL) ? "%lu\n" : "???\n",
> -	     (bs.hard_limit!=~0UL) ? PAGES2KB(bs.hard_limit) : 0);
>  BALLOON_SHOW(driver_kb, "%lu\n", PAGES2KB(bs.driver_pages));
>  
>  static ssize_t show_target_kb(struct sys_device *dev, char *buf)
> @@ -96,7 +93,6 @@
>  	&attr_current_kb.attr,
>  	&attr_low_kb.attr,
>  	&attr_high_kb.attr,
> -	&attr_hard_limit_kb.attr,
>  	&attr_driver_kb.attr,
>  	NULL
>  };
> 
> 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel
>

RE: linux: try harder to balloon up under memory pressure.

[Ian Campbell]

On Fri, 2009-06-05 at 10:30 -0400, Dan Magenheimer wrote:
> Suggestion:  Perhaps the balloon_timer interval should
> be configurable via sysfs?

Perhaps. It'd be preferable to have a default which Just Works though.
I'd be inclined to wait until someone has a concrete problem with the
interval and then decide whether the default simply needs adjusting or a
tuning nob is required.

I also thought about making it back-off on each successive failure but
decided that was getting ahead of myself too.

Ian.

> 
> > -----Original Message-----
> > From: Ian Campbell [mailto:Ian.Campbell@eu.citrix.com]
> > Sent: Friday, June 05, 2009 4:57 AM
> > To: xen-devel
> > Cc: Keir Fraser
> > Subject: [Xen-devel] linux: try harder to balloon up under memory
> > pressure.
> > 
> > 
> > Currently if the balloon driver is unable to increase the guest's
> > reservation it assumes the failure was due to reaching its full
> > allocation, gives up on the ballooning operation and records the limit
> > it reached as the "hard limit". The driver will not try again until
> > the target is set again (even to the same value).
> > 
> > However it is possible that ballooning has in fact failed due to
> > memory pressure in the host and therefore it is desirable to keep
> > attempting to reach the target in case memory becomes available. The
> > most likely scenario is that some guests are ballooning down while
> > others are ballooning up and therefore there is temporary memory
> > pressure while things stabilise. You would not expect a well behaved
> > toolstack to ask a domain to balloon to more than its allocation nor
> > would you expect it to deliberately over-commit memory by setting
> > balloon targets which exceed the total host memory.
> > 
> > This patch drops the concept of a hard limit and causes the balloon
> > driver to retry increasing the reservation on a timer in the same
> > manner as when decreasing the reservation.
> > 
> > Also if we partially succeed in increasing the reservation
> > (i.e. receive less pages than we asked for) then we may as well keep
> > those pages rather than returning them to Xen.
> > 
> > This patch is for the 2.6.18-xen tree.
> > 
> > Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> > 
> > diff -r fd9c565657b8 drivers/xen/balloon/balloon.c
> > --- a/drivers/xen/balloon/balloon.c	Thu Jun 04 16:26:49 2009 +0100
> > +++ b/drivers/xen/balloon/balloon.c	Fri Jun 05 11:56:22 2009 +0100
> > @@ -188,7 +188,7 @@
> >  
> >  static unsigned long current_target(void)
> >  {
> > -	unsigned long target = min(bs.target_pages, bs.hard_limit);
> > +	unsigned long target = bs.target_pages;
> >  	if (target > (bs.current_pages + bs.balloon_low + 
> > bs.balloon_high))
> >  		target = bs.current_pages + bs.balloon_low + 
> > bs.balloon_high;
> >  	return target;
> > @@ -255,26 +255,12 @@
> >  	}
> >  
> >  	set_xen_guest_handle(reservation.extent_start, frame_list);
> > -	reservation.nr_extents   = nr_pages;
> > -	rc = HYPERVISOR_memory_op(
> > -		XENMEM_populate_physmap, &reservation);
> > -	if (rc < nr_pages) {
> > -		if (rc > 0) {
> > -			int ret;
> > +	reservation.nr_extents = nr_pages;
> > +	rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, 
> > &reservation);
> > +	if (rc < 0)
> > +		goto out;
> >  
> > -			/* We hit the Xen hard limit: reprobe. */
> > -			reservation.nr_extents = rc;
> > -			ret = 
> > HYPERVISOR_memory_op(XENMEM_decrease_reservation,
> > -					&reservation);
> > -			BUG_ON(ret != rc);
> > -		}
> > -		if (rc >= 0)
> > -			bs.hard_limit = (bs.current_pages + rc -
> > -					 bs.driver_pages);
> > -		goto out;
> > -	}
> > -
> > -	for (i = 0; i < nr_pages; i++) {
> > +	for (i = 0; i < rc; i++) {
> >  		page = balloon_retrieve();
> >  		BUG_ON(page == NULL);
> >  
> > @@ -302,13 +288,13 @@
> >  		balloon_free_page(page);
> >  	}
> >  
> > -	bs.current_pages += nr_pages;
> > +	bs.current_pages += rc;
> >  	totalram_pages = bs.current_pages;
> >  
> >   out:
> >  	balloon_unlock(flags);
> >  
> > -	return 0;
> > +	return rc < 0 ? rc : rc != nr_pages;
> >  }
> >  
> >  static int decrease_reservation(unsigned long nr_pages)
> > @@ -420,7 +406,6 @@
> >  void balloon_set_new_target(unsigned long target)
> >  {
> >  	/* No need for lock. Not read-modify-write updates. */
> > -	bs.hard_limit   = ~0UL;
> >  	bs.target_pages = max(target, minimum_target());
> >  	schedule_work(&balloon_worker);
> >  }
> > @@ -498,17 +483,11 @@
> >  		"Requested target:   %8lu kB\n"
> >  		"Low-mem balloon:    %8lu kB\n"
> >  		"High-mem balloon:   %8lu kB\n"
> > -		"Driver pages:       %8lu kB\n"
> > -		"Xen hard limit:     ",
> > +		"Driver pages:       %8lu kB\n",
> >  		PAGES2KB(bs.current_pages), PAGES2KB(bs.target_pages), 
> >  		PAGES2KB(bs.balloon_low), PAGES2KB(bs.balloon_high),
> >  		PAGES2KB(bs.driver_pages));
> >  
> > -	if (bs.hard_limit != ~0UL)
> > -		len += sprintf(page + len, "%8lu kB\n",
> > -			       PAGES2KB(bs.hard_limit));
> > -	else
> > -		len += sprintf(page + len, "     ??? kB\n");
> >  
> >  	*eof = 1;
> >  	return len;
> > @@ -539,7 +518,6 @@
> >  	bs.balloon_low   = 0;
> >  	bs.balloon_high  = 0;
> >  	bs.driver_pages  = 0UL;
> > -	bs.hard_limit    = ~0UL;
> >  
> >  	init_timer(&balloon_timer);
> >  	balloon_timer.data = 0;
> > diff -r fd9c565657b8 drivers/xen/balloon/common.h
> > --- a/drivers/xen/balloon/common.h	Thu Jun 04 16:26:49 2009 +0100
> > +++ b/drivers/xen/balloon/common.h	Fri Jun 05 11:56:22 2009 +0100
> > @@ -35,8 +35,6 @@
> >  	/* We aim for 'current allocation' == 'target allocation'. */
> >  	unsigned long current_pages;
> >  	unsigned long target_pages;
> > -	/* We may hit the hard limit in Xen. If we do then we 
> > remember it. */
> > -	unsigned long hard_limit;
> >  	/*
> >  	 * Drivers may alter the memory reservation 
> > independently, but they
> >  	 * must inform the balloon driver so we avoid hitting 
> > the hard limit.
> > diff -r fd9c565657b8 drivers/xen/balloon/sysfs.c
> > --- a/drivers/xen/balloon/sysfs.c	Thu Jun 04 16:26:49 2009 +0100
> > +++ b/drivers/xen/balloon/sysfs.c	Fri Jun 05 11:56:22 2009 +0100
> > @@ -53,9 +53,6 @@
> >  BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(bs.current_pages));
> >  BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(bs.balloon_low));
> >  BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(bs.balloon_high));
> > -BALLOON_SHOW(hard_limit_kb,
> > -	     (bs.hard_limit!=~0UL) ? "%lu\n" : "???\n",
> > -	     (bs.hard_limit!=~0UL) ? PAGES2KB(bs.hard_limit) : 0);
> >  BALLOON_SHOW(driver_kb, "%lu\n", PAGES2KB(bs.driver_pages));
> >  
> >  static ssize_t show_target_kb(struct sys_device *dev, char *buf)
> > @@ -96,7 +93,6 @@
> >  	&attr_current_kb.attr,
> >  	&attr_low_kb.attr,
> >  	&attr_high_kb.attr,
> > -	&attr_hard_limit_kb.attr,
> >  	&attr_driver_kb.attr,
> >  	NULL
> >  };
> > 
> > 
> > 
> > _______________________________________________
> > Xen-devel mailing list
> > Xen-devel@lists.xensource.com
> > http://lists.xensource.com/xen-devel
> >

RE: linux: try harder to balloon up under memory pressure.

[Dan Magenheimer]

> I also thought about making it back-off on each successive failure but
> decided that was getting ahead of myself too.

Yes, and then the next step is to have it optionally adapt even
more intelligently -- based on kernel-internal memory metrics.
My first attempt at self-ballooning was in the
balloon driver, but I found that the key metric I wanted,
while available from /proc/meminfo, wasn't exported
within the kernel itself.  (Go figure.)  But I decided
putting the policy in user-space was better anyway.

However, for balloon_timer, the default of once per second
(correct?) seems too infrequent when the guest needs
memory NOW, given that the alternative is swapping or OOMs.
OTOH, when most of the time repeated requests will
fail, no sense invoking the timer more frequently.
So some flexibility may be warranted.

> -----Original Message-----
> From: Ian Campbell [mailto:Ian.Campbell@eu.citrix.com]
> Sent: Friday, June 05, 2009 8:39 AM
> To: Dan Magenheimer
> Cc: xen-devel; Keir Fraser; Jeremy Fitzhardinge
> Subject: RE: [Xen-devel] linux: try harder to balloon up under memory
> pressure.
> 
> 
> On Fri, 2009-06-05 at 10:30 -0400, Dan Magenheimer wrote:
> > Suggestion:  Perhaps the balloon_timer interval should
> > be configurable via sysfs?
> 
> Perhaps. It'd be preferable to have a default which Just Works though.
> I'd be inclined to wait until someone has a concrete problem with the
> interval and then decide whether the default simply needs 
> adjusting or a
> tuning nob is required.
> 
> I also thought about making it back-off on each successive failure but
> decided that was getting ahead of myself too.
> 
> Ian.
> 
> > 
> > > -----Original Message-----
> > > From: Ian Campbell [mailto:Ian.Campbell@eu.citrix.com]
> > > Sent: Friday, June 05, 2009 4:57 AM
> > > To: xen-devel
> > > Cc: Keir Fraser
> > > Subject: [Xen-devel] linux: try harder to balloon up under memory
> > > pressure.
> > > 
> > > 
> > > Currently if the balloon driver is unable to increase the guest's
> > > reservation it assumes the failure was due to reaching its full
> > > allocation, gives up on the ballooning operation and 
> records the limit
> > > it reached as the "hard limit". The driver will not try 
> again until
> > > the target is set again (even to the same value).
> > > 
> > > However it is possible that ballooning has in fact failed due to
> > > memory pressure in the host and therefore it is desirable to keep
> > > attempting to reach the target in case memory becomes 
> available. The
> > > most likely scenario is that some guests are ballooning down while
> > > others are ballooning up and therefore there is temporary memory
> > > pressure while things stabilise. You would not expect a 
> well behaved
> > > toolstack to ask a domain to balloon to more than its 
> allocation nor
> > > would you expect it to deliberately over-commit memory by setting
> > > balloon targets which exceed the total host memory.
> > > 
> > > This patch drops the concept of a hard limit and causes 
> the balloon
> > > driver to retry increasing the reservation on a timer in the same
> > > manner as when decreasing the reservation.
> > > 
> > > Also if we partially succeed in increasing the reservation
> > > (i.e. receive less pages than we asked for) then we may 
> as well keep
> > > those pages rather than returning them to Xen.
> > > 
> > > This patch is for the 2.6.18-xen tree.
> > > 
> > > Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> > > 
> > > diff -r fd9c565657b8 drivers/xen/balloon/balloon.c
> > > --- a/drivers/xen/balloon/balloon.c	Thu Jun 04 
> 16:26:49 2009 +0100
> > > +++ b/drivers/xen/balloon/balloon.c	Fri Jun 05 
> 11:56:22 2009 +0100
> > > @@ -188,7 +188,7 @@
> > >  
> > >  static unsigned long current_target(void)
> > >  {
> > > -	unsigned long target = min(bs.target_pages, bs.hard_limit);
> > > +	unsigned long target = bs.target_pages;
> > >  	if (target > (bs.current_pages + bs.balloon_low + 
> > > bs.balloon_high))
> > >  		target = bs.current_pages + bs.balloon_low + 
> > > bs.balloon_high;
> > >  	return target;
> > > @@ -255,26 +255,12 @@
> > >  	}
> > >  
> > >  	set_xen_guest_handle(reservation.extent_start, frame_list);
> > > -	reservation.nr_extents   = nr_pages;
> > > -	rc = HYPERVISOR_memory_op(
> > > -		XENMEM_populate_physmap, &reservation);
> > > -	if (rc < nr_pages) {
> > > -		if (rc > 0) {
> > > -			int ret;
> > > +	reservation.nr_extents = nr_pages;
> > > +	rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, 
> > > &reservation);
> > > +	if (rc < 0)
> > > +		goto out;
> > >  
> > > -			/* We hit the Xen hard limit: reprobe. */
> > > -			reservation.nr_extents = rc;
> > > -			ret = 
> > > HYPERVISOR_memory_op(XENMEM_decrease_reservation,
> > > -					&reservation);
> > > -			BUG_ON(ret != rc);
> > > -		}
> > > -		if (rc >= 0)
> > > -			bs.hard_limit = (bs.current_pages + rc -
> > > -					 bs.driver_pages);
> > > -		goto out;
> > > -	}
> > > -
> > > -	for (i = 0; i < nr_pages; i++) {
> > > +	for (i = 0; i < rc; i++) {
> > >  		page = balloon_retrieve();
> > >  		BUG_ON(page == NULL);
> > >  
> > > @@ -302,13 +288,13 @@
> > >  		balloon_free_page(page);
> > >  	}
> > >  
> > > -	bs.current_pages += nr_pages;
> > > +	bs.current_pages += rc;
> > >  	totalram_pages = bs.current_pages;
> > >  
> > >   out:
> > >  	balloon_unlock(flags);
> > >  
> > > -	return 0;
> > > +	return rc < 0 ? rc : rc != nr_pages;
> > >  }
> > >  
> > >  static int decrease_reservation(unsigned long nr_pages)
> > > @@ -420,7 +406,6 @@
> > >  void balloon_set_new_target(unsigned long target)
> > >  {
> > >  	/* No need for lock. Not read-modify-write updates. */
> > > -	bs.hard_limit   = ~0UL;
> > >  	bs.target_pages = max(target, minimum_target());
> > >  	schedule_work(&balloon_worker);
> > >  }
> > > @@ -498,17 +483,11 @@
> > >  		"Requested target:   %8lu kB\n"
> > >  		"Low-mem balloon:    %8lu kB\n"
> > >  		"High-mem balloon:   %8lu kB\n"
> > > -		"Driver pages:       %8lu kB\n"
> > > -		"Xen hard limit:     ",
> > > +		"Driver pages:       %8lu kB\n",
> > >  		PAGES2KB(bs.current_pages), PAGES2KB(bs.target_pages), 
> > >  		PAGES2KB(bs.balloon_low), PAGES2KB(bs.balloon_high),
> > >  		PAGES2KB(bs.driver_pages));
> > >  
> > > -	if (bs.hard_limit != ~0UL)
> > > -		len += sprintf(page + len, "%8lu kB\n",
> > > -			       PAGES2KB(bs.hard_limit));
> > > -	else
> > > -		len += sprintf(page + len, "     ??? kB\n");
> > >  
> > >  	*eof = 1;
> > >  	return len;
> > > @@ -539,7 +518,6 @@
> > >  	bs.balloon_low   = 0;
> > >  	bs.balloon_high  = 0;
> > >  	bs.driver_pages  = 0UL;
> > > -	bs.hard_limit    = ~0UL;
> > >  
> > >  	init_timer(&balloon_timer);
> > >  	balloon_timer.data = 0;
> > > diff -r fd9c565657b8 drivers/xen/balloon/common.h
> > > --- a/drivers/xen/balloon/common.h	Thu Jun 04 
> 16:26:49 2009 +0100
> > > +++ b/drivers/xen/balloon/common.h	Fri Jun 05 
> 11:56:22 2009 +0100
> > > @@ -35,8 +35,6 @@
> > >  	/* We aim for 'current allocation' == 'target allocation'. */
> > >  	unsigned long current_pages;
> > >  	unsigned long target_pages;
> > > -	/* We may hit the hard limit in Xen. If we do then we 
> > > remember it. */
> > > -	unsigned long hard_limit;
> > >  	/*
> > >  	 * Drivers may alter the memory reservation 
> > > independently, but they
> > >  	 * must inform the balloon driver so we avoid hitting 
> > > the hard limit.
> > > diff -r fd9c565657b8 drivers/xen/balloon/sysfs.c
> > > --- a/drivers/xen/balloon/sysfs.c	Thu Jun 04 16:26:49 2009 +0100
> > > +++ b/drivers/xen/balloon/sysfs.c	Fri Jun 05 11:56:22 2009 +0100
> > > @@ -53,9 +53,6 @@
> > >  BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(bs.current_pages));
> > >  BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(bs.balloon_low));
> > >  BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(bs.balloon_high));
> > > -BALLOON_SHOW(hard_limit_kb,
> > > -	     (bs.hard_limit!=~0UL) ? "%lu\n" : "???\n",
> > > -	     (bs.hard_limit!=~0UL) ? PAGES2KB(bs.hard_limit) : 0);
> > >  BALLOON_SHOW(driver_kb, "%lu\n", PAGES2KB(bs.driver_pages));
> > >  
> > >  static ssize_t show_target_kb(struct sys_device *dev, char *buf)
> > > @@ -96,7 +93,6 @@
> > >  	&attr_current_kb.attr,
> > >  	&attr_low_kb.attr,
> > >  	&attr_high_kb.attr,
> > > -	&attr_hard_limit_kb.attr,
> > >  	&attr_driver_kb.attr,
> > >  	NULL
> > >  };
> > > 
> > > 
> > > 
> > > _______________________________________________
> > > Xen-devel mailing list
> > > Xen-devel@lists.xensource.com
> > > http://lists.xensource.com/xen-devel
> > >
> 
>