* RE: [PATCH][TOOLS] Reducingimpactofdomainsave/restore/dump on Dom0
@ 2007-02-24 0:50 Graham, Simon
0 siblings, 0 replies; 6+ messages in thread
From: Graham, Simon @ 2007-02-24 0:50 UTC (permalink / raw)
To: Keir Fraser, Keir Fraser, xen-devel
> > This wont quite work -- xc_private.c isn't part of libxenguest which
> > includes save/restore which needs the function... that's why I made
> it
> > inline in xc_private.h which is included everywhere...
>
> Libxenguest links against libxenctrl: it uses plenty of functions from
> libxenctrl already (xc_domain_getinfo() to name just one example).
Well, then something else isn't right -- when I first did this, I put
the routine in xc_private.c and got unresolved externals making
libxenguest.so... that's the _only_ reason I put it inline in
xc_private.h...
Let me try again...
^ permalink raw reply [flat|nested] 6+ messages in thread
* RE: [PATCH][TOOLS]Reducingimpactofdomainsave/restore/dump on Dom0
@ 2007-02-24 14:06 Graham, Simon
0 siblings, 0 replies; 6+ messages in thread
From: Graham, Simon @ 2007-02-24 14:06 UTC (permalink / raw)
To: Graham, Simon, Keir Fraser, Keir Fraser, xen-devel
[-- Attachment #1: Type: text/plain, Size: 1076 bytes --]
> > Libxenguest links against libxenctrl: it uses plenty of functions
> from
> > libxenctrl already (xc_domain_getinfo() to name just one example).
>
> Well, then something else isn't right -- when I first did this, I put
> the routine in xc_private.c and got unresolved externals making
> libxenguest.so... that's the _only_ reason I put it inline in
> xc_private.h...
>
Well.. now it works -- beats me why it didn't before!
Anyway -- attached is take - um - 4 I think; implementation moved into
xc_linux.c and xc_solaris.c (I chose not to go with the default
implementation using the 'weak' attribute in favour of an explicitly
null version on Solaris). No changes in tools/xcutils. I also removed
the xc_ prefix from the routine to make it clear that it's internal...
/simgr
-----------------------------------
Reduce impact of saving/restoring/dumping large domains on Dom0 memory
usage by means of fadvise64() to tell the OS to discard the cache pages
used for the save/dump file.
Signed-off-by: Simon Graham <Simon.Graham@stratus.com>
[-- Attachment #2: xen-dom0-cache.patch --]
[-- Type: application/octet-stream, Size: 7648 bytes --]
Index: trunk/tools/libxc/xc_solaris.c
===================================================================
--- trunk/tools/libxc/xc_solaris.c (revision 10784)
+++ trunk/tools/libxc/xc_solaris.c (working copy)
@@ -242,3 +242,10 @@
{
return dorw(xce_handle, (char *)&port, sizeof(port), 1);
}
+
+/* Optionally flush file to disk and discard page cache */
+int discard_file_cache(int fd, int flush)
+{
+ // TODO: Implement for Solaris!
+ return 0;
+}
Index: trunk/tools/libxc/xc_linux_save.c
===================================================================
--- trunk/tools/libxc/xc_linux_save.c (revision 10784)
+++ trunk/tools/libxc/xc_linux_save.c (working copy)
@@ -171,7 +171,29 @@
(new->tv_usec - old->tv_usec);
}
+static int noncached_write(int fd, int live, void *buffer, int len)
+{
+ static int write_count = 0;
+ int rc = write(fd,buffer,len);
+
+ if (!live) {
+ write_count += len;
+
+ if (write_count >= MAX_PAGECACHE_USAGE*PAGE_SIZE) {
+ int serrno = errno;
+
+ /* Time to discard cache - dont care if this fails */
+ discard_file_cache(fd, 0 /* no flush */);
+
+ write_count = 0;
+
+ errno = serrno;
+ }
+ }
+ return rc;
+}
+
#ifdef ADAPTIVE_SAVE
@@ -204,7 +226,7 @@
}
-static int ratewrite(int io_fd, void *buf, int n)
+static int ratewrite(int io_fd, int live, void *buf, int n)
{
static int budget = 0;
static int burst_time_us = -1;
@@ -214,7 +236,7 @@
long long delta;
if (START_MBIT_RATE == 0)
- return write(io_fd, buf, n);
+ return noncached_write(io_fd, live, buf, n);
budget -= n;
if (budget < 0) {
@@ -250,13 +272,13 @@
}
}
}
- return write(io_fd, buf, n);
+ return noncached_write(io_fd, live, buf, n);
}
#else /* ! ADAPTIVE SAVE */
#define RATE_IS_MAX() (0)
-#define ratewrite(_io_fd, _buf, _n) write((_io_fd), (_buf), (_n))
+#define ratewrite(_io_fd, _live, _buf, _n) noncached_write((_io_fd), (_live), (_buf), (_n))
#define initialize_mbit_rate()
#endif
@@ -1069,7 +1091,7 @@
if(race && !live)
goto out;
- if (ratewrite(io_fd, page, PAGE_SIZE) != PAGE_SIZE) {
+ if (ratewrite(io_fd, live, page, PAGE_SIZE) != PAGE_SIZE) {
ERROR("Error when writing to state file (4)"
" (errno %d)", errno);
goto out;
@@ -1078,7 +1100,7 @@
} else {
/* We have a normal page: just write it directly. */
- if (ratewrite(io_fd, spage, PAGE_SIZE) != PAGE_SIZE) {
+ if (ratewrite(io_fd, live, spage, PAGE_SIZE) != PAGE_SIZE) {
ERROR("Error when writing to state file (5)"
" (errno %d)", errno);
goto out;
@@ -1248,6 +1270,10 @@
DPRINTF("Warning - couldn't disable shadow mode");
}
}
+ else {
+ // flush last write and discard cache for file
+ discard_file_cache(io_fd, 1 /* flush */);
+ }
if (live_shinfo)
munmap(live_shinfo, PAGE_SIZE);
Index: trunk/tools/libxc/xc_linux.c
===================================================================
--- trunk/tools/libxc/xc_linux.c (revision 10784)
+++ trunk/tools/libxc/xc_linux.c (working copy)
@@ -328,6 +328,41 @@
return dorw(xce_handle, (char *)&port, sizeof(port), 1);
}
+/* Optionally flush file to disk and discard page cache */
+int discard_file_cache(int fd, int flush)
+{
+ off_t cur;
+
+ if (flush && fsync(fd) < 0) {
+ PERROR("Failed to flush file: %s", strerror(errno));
+ return -errno;
+ }
+
+ /*
+ * Calculate last page boundary of amount written so far
+ * unless we are flushing in which case entire cache
+ * is discarded
+ */
+ if (flush)
+ cur = 0;
+ else {
+ cur = lseek(fd, 0, SEEK_CUR);
+
+ if (cur != (off_t)-1)
+ cur &= ~(PAGE_SIZE-1);
+ else
+ cur = 0;
+ }
+
+ /* and throw out of cache */
+ if (posix_fadvise64(fd, 0, cur, POSIX_FADV_DONTNEED) < 0) {
+ PERROR("Failed to discard cache: %s", strerror(errno));
+ return -errno;
+ }
+
+ return 0;
+}
+
/*
* Local variables:
* mode: C
Index: trunk/tools/libxc/Makefile
===================================================================
--- trunk/tools/libxc/Makefile (revision 10784)
+++ trunk/tools/libxc/Makefile (working copy)
@@ -57,6 +57,8 @@
CFLAGS += -fno-strict-aliasing
CFLAGS += $(INCLUDES) -I.
+CFLAGS-$(CONFIG_Linux) += -D_GNU_SOURCE
+
# Define this to make it possible to run valgrind on code linked with these
# libraries.
#CFLAGS += -DVALGRIND -O0 -ggdb3
Index: trunk/tools/libxc/xc_private.h
===================================================================
--- trunk/tools/libxc/xc_private.h (revision 10784)
+++ trunk/tools/libxc/xc_private.h (working copy)
@@ -41,6 +41,13 @@
#define INFO 1
#define PROGRESS 0
+/*
+** Define max dirty page cache to permit during save/restore -- need to balance
+** keeping cache usage down with CPU impact of invalidating too often.
+** (Currently 16MB)
+*/
+#define MAX_PAGECACHE_USAGE (4*1024)
+
#if INFO
#define IPRINTF(_f, _a...) printf(_f , ## _a)
#else
@@ -158,4 +165,7 @@
void bitmap_64_to_byte(uint8_t *bp, const uint64_t *lp, int nbits);
void bitmap_byte_to_64(uint64_t *lp, const uint8_t *bp, int nbits);
+/* Optionally flush file to disk and discard page cache */
+int discard_file_cache(int fd, int flush);
+
#endif /* __XC_PRIVATE_H__ */
Index: trunk/tools/libxc/xc_core.c
===================================================================
--- trunk/tools/libxc/xc_core.c (revision 10784)
+++ trunk/tools/libxc/xc_core.c (working copy)
@@ -145,6 +145,12 @@
}
}
+ if (length >= DUMP_INCREMENT*PAGE_SIZE) {
+ // Now dumping pages -- make sure we discard clean pages from
+ // the cache after each write
+ discard_file_cache(da->fd, 0 /* no flush */);
+ }
+
return 0;
}
@@ -165,6 +171,9 @@
sts = xc_domain_dumpcore_via_callback(
xc_handle, domid, &da, &local_file_dump);
+ /* flush and discard any remaining portion of the file from cache */
+ discard_file_cache(da.fd, 1/* flush first*/);
+
close(da.fd);
return sts;
Index: trunk/tools/libxc/xc_linux_restore.c
===================================================================
--- trunk/tools/libxc/xc_linux_restore.c (revision 10784)
+++ trunk/tools/libxc/xc_linux_restore.c (working copy)
@@ -143,7 +143,7 @@
unsigned int console_evtchn, unsigned long *console_mfn)
{
DECLARE_DOMCTL;
- int rc = 1, i, n, pae_extended_cr3 = 0;
+ int rc = 1, i, n, m, pae_extended_cr3 = 0;
unsigned long mfn, pfn;
unsigned int prev_pc, this_pc;
int verify = 0;
@@ -330,7 +330,7 @@
*/
prev_pc = 0;
- n = 0;
+ n = m = 0;
while (1) {
int j, nr_mfns = 0;
@@ -529,6 +529,14 @@
munmap(region_base, j*PAGE_SIZE);
n+= j; /* crude stats */
+
+ /* discard cache for portion of file read so far up to last page boundary
+ every 16MB (4k pages) or so */
+ m += j;
+ if (m > MAX_PAGECACHE_USAGE) {
+ discard_file_cache(io_fd, 0 /* no flush */);
+ m = 0;
+ }
}
/*
@@ -863,6 +871,9 @@
free(p2m);
free(pfn_type);
+ /* discard cache for save file */
+ discard_file_cache(io_fd, 1 /*flush*/);
+
DPRINTF("Restore exit with rc=%d\n", rc);
return rc;
[-- Attachment #3: Type: text/plain, Size: 138 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH][TOOLS] Reducingimpactofdomainsave/restore/dump on Dom0
2007-02-23 18:48 [PATCH][TOOLS] Reducingimpactofdomainsave/restore/dump " Graham, Simon
@ 2007-02-23 23:04 ` Keir Fraser
0 siblings, 0 replies; 6+ messages in thread
From: Keir Fraser @ 2007-02-23 23:04 UTC (permalink / raw)
To: Graham, Simon, Keir Fraser, xen-devel
On 23/2/07 18:48, "Graham, Simon" <Simon.Graham@stratus.com> wrote:
>> Put the Linux definition in xc_linux.c. Put a default dummy
>> implementation
>> in xc_private.c with __attribute__ ((weak)). If you have only a
>> function
>> prototype in xc_private.h then you won't need -D_GNU_SOURCE for
>> xcutil/Makefile.
>
> This wont quite work -- xc_private.c isn't part of libxenguest which
> includes save/restore which needs the function... that's why I made it
> inline in xc_private.h which is included everywhere...
Libxenguest links against libxenctrl: it uses plenty of functions from
libxenctrl already (xc_domain_getinfo() to name just one example).
-- Keir
^ permalink raw reply [flat|nested] 6+ messages in thread
* RE: [PATCH][TOOLS] Reducingimpactofdomainsave/restore/dump on Dom0
@ 2007-02-23 18:48 Graham, Simon
2007-02-23 23:04 ` Keir Fraser
0 siblings, 1 reply; 6+ messages in thread
From: Graham, Simon @ 2007-02-23 18:48 UTC (permalink / raw)
To: Keir Fraser, xen-devel
> Put the Linux definition in xc_linux.c. Put a default dummy
> implementation
> in xc_private.c with __attribute__ ((weak)). If you have only a
> function
> prototype in xc_private.h then you won't need -D_GNU_SOURCE for
> xcutil/Makefile.
This wont quite work -- xc_private.c isn't part of libxenguest which
includes save/restore which needs the function... that's why I made it
inline in xc_private.h which is included everywhere...
/simgr
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH][TOOLS] Reducingimpactofdomainsave/restore/dump on Dom0
2007-02-23 18:32 Graham, Simon
@ 2007-02-23 18:43 ` Keir Fraser
0 siblings, 0 replies; 6+ messages in thread
From: Keir Fraser @ 2007-02-23 18:43 UTC (permalink / raw)
To: Graham, Simon, Keir Fraser, xen-devel
On 23/2/07 18:32, "Graham, Simon" <Simon.Graham@stratus.com> wrote:
>> 2. You need it in xcutil because this directory has code that includes
>> xc_private.h -- this seems wrong/bad to me but I wasn't prepared
>> to change it!
>>
>
> Actually, it's xcutils/readnotes.c that #include's xg_private.h (which
> in turn includes xc_private.h) -- since the function that does the
> advise call is inline in xc_private.h, this wont compile unless
> _GNU_SOURCE is defined...
Put the Linux definition in xc_linux.c. Put a default dummy implementation
in xc_private.c with __attribute__ ((weak)). If you have only a function
prototype in xc_private.h then you won't need -D_GNU_SOURCE for
xcutil/Makefile.
-- Keir
^ permalink raw reply [flat|nested] 6+ messages in thread
* RE: [PATCH][TOOLS] Reducingimpactofdomainsave/restore/dump on Dom0
@ 2007-02-23 18:32 Graham, Simon
2007-02-23 18:43 ` Keir Fraser
0 siblings, 1 reply; 6+ messages in thread
From: Graham, Simon @ 2007-02-23 18:32 UTC (permalink / raw)
To: Graham, Simon, Keir Fraser, xen-devel
> 2. You need it in xcutil because this directory has code that includes
> xc_private.h -- this seems wrong/bad to me but I wasn't prepared
> to change it!
>
Actually, it's xcutils/readnotes.c that #include's xg_private.h (which
in turn includes xc_private.h) -- since the function that does the
advise call is inline in xc_private.h, this wont compile unless
_GNU_SOURCE is defined...
Simon
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2007-02-24 14:06 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2007-02-24 0:50 [PATCH][TOOLS] Reducingimpactofdomainsave/restore/dump on Dom0 Graham, Simon
-- strict thread matches above, loose matches on Subject: below --
2007-02-24 14:06 [PATCH][TOOLS]Reducingimpactofdomainsave/restore/dump " Graham, Simon
2007-02-23 18:48 [PATCH][TOOLS] Reducingimpactofdomainsave/restore/dump " Graham, Simon
2007-02-23 23:04 ` Keir Fraser
2007-02-23 18:32 Graham, Simon
2007-02-23 18:43 ` Keir Fraser
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.