[PATCH XEN v8 11/29] tools: Refactor foreign memory mapping into libxenforeignmemory

[Ian Campbell <ian.campbell@citrix.com>]

libxenforeignmemory will provide a stable API and ABI for mapping
foreign domain memory (subject to appropriate privileges).

The new library exposes an interface equivalent to
xc_map_foreign_memory_bulk, which all the other
xc_map_foreign_memory_* functions (which remain in libxc) are
implemented in terms of.

Upon request (via #define XC_WANT_COMPAT_MAP_FOREIGN_API) libxenctrl
will provide a compat API for the old names. This is used by qemu-xen
and qemu-trad as well as various in tree things (which required
de-dupping various #includes in some too to get the #define before the
first).

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
Must be applied with:
  - "qemu-xen-traditional: qemu-xen-traditional: Add
    libxenforeignmemory to rpath-link" and a corresponding QEMU_TAG
    update folded here.
  - "mini-os: mini-os: Include libxenforeignmemory with libxc" and a
    corresponding bump to MINIOS_UPSTREAM_REVISION folded in here.

v5: Allow close(NULL)

v6: Define XC_WANT_COMPAT_MAP_FOREIGN_API for xen-mceinj.c and memshrtool.c
    Fix typo in type name in compat.c, and get the functions names to
    match.
    Use an osdep layer for _map, to allow for future common code.

v7: Correct comment in compat.h #error due to switch to osdep layer
    for map in v6
    Added mk-headers-$(XEN_TARGET_ARCH) build dependency instead of
    open coding the recursion.

v8: Remove *.so on clean, add distclean target.
---
 .gitignore                                         |   2 +
 stubdom/Makefile                                   |  17 +++-
 tools/Makefile                                     |   2 +
 tools/Rules.mk                                     |  11 ++-
 tools/console/daemon/utils.c                       |   1 -
 tools/console/daemon/utils.h                       |   1 +
 tools/libs/Makefile                                |   1 +
 tools/libs/foreignmemory/Makefile                  |  71 ++++++++++++++
 tools/libs/foreignmemory/compat.c                  |  72 ++++++++++++++
 tools/libs/foreignmemory/core.c                    |  84 +++++++++++++++++
 .../foreignmemory/freebsd.c}                       |  33 +++----
 .../libs/foreignmemory/include/xenforeignmemory.h  |  71 ++++++++++++++
 tools/libs/foreignmemory/libxenforeignmemory.map   |   7 ++
 .../foreignmemory/linux.c}                         |  70 ++++++--------
 tools/libs/foreignmemory/minios.c                  |  62 ++++++++++++
 tools/libs/foreignmemory/netbsd.c                  | 105 +++++++++++++++++++++
 tools/libs/foreignmemory/private.h                 |  53 +++++++++++
 tools/libs/foreignmemory/solaris.c                 |  93 ++++++++++++++++++
 tools/libxc/Makefile                               |   8 +-
 tools/libxc/include/xenctrl.h                      |  26 -----
 tools/libxc/include/xenctrl_compat.h               |  36 +++++++
 tools/libxc/xc_foreign_memory.c                    |  49 +---------
 tools/libxc/xc_minios.c                            |  29 ------
 tools/libxc/xc_netbsd.c                            |  73 --------------
 tools/libxc/xc_private.c                           |  13 +--
 tools/libxc/xc_private.h                           |  11 ++-
 tools/libxc/xc_solaris.c                           |  73 --------------
 tools/libxc/xc_sr_restore.c                        |   4 +-
 tools/libxc/xc_sr_save.c                           |   4 +-
 tools/libxc/xg_private.h                           |   3 +-
 tools/libxl/libxl_internal.h                       |   1 +
 tools/misc/xen-mfndump.c                           |   1 +
 tools/ocaml/libs/xc/xenctrl_stubs.c                |   1 +
 tools/python/xen/lowlevel/xc/xc.c                  |   2 +
 tools/tests/mce-test/tools/xen-mceinj.c            |   1 +
 tools/tests/mem-sharing/memshrtool.c               |   1 +
 tools/xenmon/xenbaked.c                            |   1 +
 tools/xenpaging/pagein.c                           |   1 -
 tools/xenpaging/xenpaging.c                        |   1 -
 tools/xenpaging/xenpaging.h                        |   2 +
 tools/xenstore/xenstored_core.c                    |   1 -
 tools/xenstore/xenstored_core.h                    |   1 +
 tools/xentrace/xenctx.c                            |   3 +-
 tools/xentrace/xentrace.c                          |   1 +
 44 files changed, 770 insertions(+), 333 deletions(-)
 create mode 100644 tools/libs/foreignmemory/Makefile
 create mode 100644 tools/libs/foreignmemory/compat.c
 create mode 100644 tools/libs/foreignmemory/core.c
 rename tools/{libxc/xc_freebsd_osdep.c => libs/foreignmemory/freebsd.c} (76%)
 create mode 100644 tools/libs/foreignmemory/include/xenforeignmemory.h
 create mode 100644 tools/libs/foreignmemory/libxenforeignmemory.map
 rename tools/{libxc/xc_linux_osdep.c => libs/foreignmemory/linux.c} (83%)
 create mode 100644 tools/libs/foreignmemory/minios.c
 create mode 100644 tools/libs/foreignmemory/netbsd.c
 create mode 100644 tools/libs/foreignmemory/private.h
 create mode 100644 tools/libs/foreignmemory/solaris.c

diff --git a/.gitignore b/.gitignore
index 55ecfb7..9eb2d30 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,6 +64,7 @@ stubdom/libs-*
 stubdom/libxc-*
 stubdom/libxencall-*
 stubdom/libxenevtchn-*
+stubdom/libxenforeignmemory-*
 stubdom/libxengnttab-*
 stubdom/libxentoollog-*
 stubdom/lwip-*
@@ -93,6 +94,7 @@ tools/libs/toollog/headers.chk
 tools/libs/evtchn/headers.chk
 tools/libs/gnttab/headers.chk
 tools/libs/call/headers.chk
+tools/libs/foreignmemory/headers.chk
 tools/blktap2/daemon/blktapctrl
 tools/blktap2/drivers/img2qcow
 tools/blktap2/drivers/lock-util
diff --git a/stubdom/Makefile b/stubdom/Makefile
index 5bf4ed6..6f68e98 100644
--- a/stubdom/Makefile
+++ b/stubdom/Makefile
@@ -337,6 +337,12 @@ mk-headers-$(XEN_TARGET_ARCH): $(IOEMU_LINKFARM_TARGET)
 	  ln -sf $(XEN_ROOT)/tools/libs/call/include/*.h include/ && \
 	  ln -sf $(XEN_ROOT)/tools/libs/call/*.c . && \
 	  ln -sf $(XEN_ROOT)/tools/libs/call/Makefile . )
+	mkdir -p libs-$(XEN_TARGET_ARCH)/foreignmemory/include
+	[ -h libs-$(XEN_TARGET_ARCH)/foreignmemory/Makefile ] || ( cd libs-$(XEN_TARGET_ARCH)/foreignmemory && \
+	  ln -sf $(XEN_ROOT)/tools/libs/foreignmemory/*.h . && \
+	  ln -sf $(XEN_ROOT)/tools/libs/foreignmemory/include/*.h include/ && \
+	  ln -sf $(XEN_ROOT)/tools/libs/foreignmemory/*.c . && \
+	  ln -sf $(XEN_ROOT)/tools/libs/foreignmemory/Makefile . )
 	mkdir -p libxc-$(XEN_TARGET_ARCH)
 	[ -h libxc-$(XEN_TARGET_ARCH)/Makefile ] || ( cd libxc-$(XEN_TARGET_ARCH) && \
 	  ln -sf $(XEN_ROOT)/tools/libxc/*.h . && \
@@ -396,12 +402,21 @@ libs-$(XEN_TARGET_ARCH)/call/libxencall.a: mk-headers-$(XEN_TARGET_ARCH) $(NEWLI
 	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= -C libs-$(XEN_TARGET_ARCH)/call
 
 #######
+# libxenforeignmemory
+#######
+
+.PHONY: libxenforeignmemory
+libxenforeignmemory: libs-$(XEN_TARGET_ARCH)/foreignmemory/libxenforeignmemory.a
+libs-$(XEN_TARGET_ARCH)/foreignmemory/libxenforeignmemory.a: mk-headers-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE)
+	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= -C libs-$(XEN_TARGET_ARCH)/foreignmemory
+
+#######
 # libxc
 #######
 
 .PHONY: libxc
 libxc: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a libxc-$(XEN_TARGET_ARCH)/libxenguest.a
-libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: mk-headers-$(XEN_TARGET_ARCH) libxentoollog libxenevtchn libxengnttab libxencall cross-zlib
+libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: mk-headers-$(XEN_TARGET_ARCH) libxentoollog libxenevtchn libxengnttab libxencall libxenforeignmemory cross-zlib
 	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= CONFIG_LIBXC_MINIOS=y -C libxc-$(XEN_TARGET_ARCH)
 
  libxc-$(XEN_TARGET_ARCH)/libxenguest.a: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a
diff --git a/tools/Makefile b/tools/Makefile
index e5bbc98..c03f6db 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -250,6 +250,7 @@ subdir-all-qemu-xen-dir: qemu-xen-dir-find
 		--source-path=$$source \
 		--extra-cflags="-DXC_WANT_COMPAT_EVTCHN_API=1 \
 		-DXC_WANT_COMPAT_GNTTAB_API=1 \
+		-DXC_WANT_COMPAT_MAP_FOREIGN_API=1 \
 		-I$(XEN_ROOT)/tools/include \
 		-I$(XEN_ROOT)/tools/libs/toollog/include \
 		-I$(XEN_ROOT)/tools/libs/evtchn/include \
@@ -264,6 +265,7 @@ subdir-all-qemu-xen-dir: qemu-xen-dir-find
 		-Wl,-rpath-link=$(XEN_ROOT)/tools/libs/evtchn \
 		-Wl,-rpath-link=$(XEN_ROOT)/tools/libs/gnttab \
 		-Wl,-rpath-link=$(XEN_ROOT)/tools/libs/call \
+		-Wl,-rpath-link=$(XEN_ROOT)/tools/libs/foreignmemory \
 		$(QEMU_UPSTREAM_RPATH)" \
 		--bindir=$(LIBEXEC_BIN) \
 		--datadir=$(SHAREDIR)/qemu-xen \
diff --git a/tools/Rules.mk b/tools/Rules.mk
index 18e3688..d47fa45 100644
--- a/tools/Rules.mk
+++ b/tools/Rules.mk
@@ -14,6 +14,7 @@ XEN_LIBXENTOOLLOG  = $(XEN_ROOT)/tools/libs/toollog
 XEN_LIBXENEVTCHN   = $(XEN_ROOT)/tools/libs/evtchn
 XEN_LIBXENGNTTAB   = $(XEN_ROOT)/tools/libs/gnttab
 XEN_LIBXENCALL     = $(XEN_ROOT)/tools/libs/call
+XEN_LIBXENFOREIGNMEMORY = $(XEN_ROOT)/tools/libs/foreignmemory
 XEN_LIBXC          = $(XEN_ROOT)/tools/libxc
 XEN_XENLIGHT       = $(XEN_ROOT)/tools/libxl
 XEN_XENSTORE       = $(XEN_ROOT)/tools/xenstore
@@ -103,12 +104,16 @@ CFLAGS_libxencall = -I$(XEN_LIBXENCALL)/include $(CFLAGS_xeninclude)
 LDLIBS_libxencall = $(XEN_LIBXENCALL)/libxencall$(libextension)
 SHLIB_libxencall  = -Wl,-rpath-link=$(XEN_LIBXENCALL)
 
-CFLAGS_libxenctrl = -I$(XEN_LIBXC)/include $(CFLAGS_libxentoollog) $(CFLAGS_xeninclude)
-SHDEPS_libxenctrl = $(SHLIB_libxentoollog) $(SHLIB_libxenevtchn) $(SHLIB_libxengnttab) $(SHLIB_libxengntshr) $(SHLIB_libxencall)
+CFLAGS_libxenforeignmemory = -I$(XEN_LIBXENFOREIGNMEMORY)/include $(CFLAGS_xeninclude)
+LDLIBS_libxenforeignmemory = $(XEN_LIBXENFOREIGNMEMORY)/libxenforeignmemory$(libextension)
+SHLIB_libxenforeignmemory  = -Wl,-rpath-link=$(XEN_LIBXENFOREIGNMEMORY)
+
+CFLAGS_libxenctrl = -I$(XEN_LIBXC)/include $(CFLAGS_libxentoollog) $(CFLAGS_libxenforeignmemory) $(CFLAGS_xeninclude)
+SHDEPS_libxenctrl = $(SHLIB_libxentoollog) $(SHLIB_libxenevtchn) $(SHLIB_libxengnttab) $(SHLIB_libxengntshr) $(SHLIB_libxencall) $(SHLIB_libxenforeignmemory)
 LDLIBS_libxenctrl = $(SHDEPS_libxenctrl) $(XEN_LIBXC)/libxenctrl$(libextension)
 SHLIB_libxenctrl  = $(SHDEPS_libxenctrl) -Wl,-rpath-link=$(XEN_LIBXC)
 
-CFLAGS_libxenguest = -I$(XEN_LIBXC)/include $(CFLAGS_libxenevtchn) $(CFLAGS_xeninclude)
+CFLAGS_libxenguest = -I$(XEN_LIBXC)/include $(CFLAGS_libxenevtchn) $(CFLAGS_libxenforeignmemory) $(CFLAGS_xeninclude)
 SHDEPS_libxenguest = $(SHLIB_libxenevtchn)
 LDLIBS_libxenguest = $(SHDEPS_libxenguest) $(XEN_LIBXC)/libxenguest$(libextension)
 SHLIB_libxenguest  = $(SHDEPS_libxenguest) -Wl,-rpath-link=$(XEN_LIBXC)
diff --git a/tools/console/daemon/utils.c b/tools/console/daemon/utils.c
index 644f6af..97d7798 100644
--- a/tools/console/daemon/utils.c
+++ b/tools/console/daemon/utils.c
@@ -33,7 +33,6 @@
 #include <string.h>
 #include <signal.h>
 
-#include "xenctrl.h"
 #include "utils.h"
 
 struct xs_handle *xs;
diff --git a/tools/console/daemon/utils.h b/tools/console/daemon/utils.h
index 1295822..a010ac4 100644
--- a/tools/console/daemon/utils.h
+++ b/tools/console/daemon/utils.h
@@ -23,6 +23,7 @@
 #include <stdbool.h>
 #include <syslog.h>
 #include <stdio.h>
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 
 #include <xenstore.h>
diff --git a/tools/libs/Makefile b/tools/libs/Makefile
index f4f5d57..dc5726d 100644
--- a/tools/libs/Makefile
+++ b/tools/libs/Makefile
@@ -6,5 +6,6 @@ SUBDIRS-y += toollog
 SUBDIRS-y += evtchn
 SUBDIRS-y += gnttab
 SUBDIRS-y += call
+SUBDIRS-y += foreignmemory
 
 all clean install distclean: %: subdirs-%
diff --git a/tools/libs/foreignmemory/Makefile b/tools/libs/foreignmemory/Makefile
new file mode 100644
index 0000000..f062f45
--- /dev/null
+++ b/tools/libs/foreignmemory/Makefile
@@ -0,0 +1,71 @@
+XEN_ROOT = $(CURDIR)/../../..
+include $(XEN_ROOT)/tools/Rules.mk
+
+MAJOR    = 1
+MINOR    = 0
+SHLIB_LDFLAGS += -Wl,--version-script=libxenforeignmemory.map
+
+CFLAGS   += -Werror -Wmissing-prototypes
+CFLAGS   += -I./include $(CFLAGS_xeninclude)
+CFLAGS   += $(CFLAGS_libxentoollog)
+
+SRCS-y                 += core.c
+SRCS-$(CONFIG_Linux)   += linux.c
+SRCS-$(CONFIG_FreeBSD) += freebsd.c
+SRCS-$(CONFIG_SunOS)   += compat.c solaris.c
+SRCS-$(CONFIG_NetBSD)  += compat.c netbsd.c
+SRCS-$(CONFIG_MiniOS)  += minios.c
+
+LIB_OBJS := $(patsubst %.c,%.o,$(SRCS-y))
+PIC_OBJS := $(patsubst %.c,%.opic,$(SRCS-y))
+
+LIB := libxenforeignmemory.a
+ifneq ($(nosharedlibs),y)
+LIB += libxenforeignmemory.so
+endif
+
+.PHONY: all
+all: build
+
+.PHONY: build
+build:
+	$(MAKE) libs
+
+.PHONY: libs
+libs: headers.chk $(LIB)
+
+headers.chk: $(wildcard include/*.h)
+
+libxenforeignmemory.a: $(LIB_OBJS)
+	$(AR) rc $@ $^
+
+libxenforeignmemory.so: libxenforeignmemory.so.$(MAJOR)
+	$(SYMLINK_SHLIB) $< $@
+libxenforeignmemory.so.$(MAJOR): libxenforeignmemory.so.$(MAJOR).$(MINOR)
+	$(SYMLINK_SHLIB) $< $@
+
+libxenforeignmemory.so.$(MAJOR).$(MINOR): $(PIC_OBJS) libxenforeignmemory.map
+	$(CC) $(LDFLAGS) $(PTHREAD_LDFLAGS) -Wl,$(SONAME_LDFLAG) -Wl,libxenforeignmemory.so.$(MAJOR) $(SHLIB_LDFLAGS) -o $@ $(PIC_OBJS) $(LDLIBS_libxentoollog) $(APPEND_LDFLAGS)
+
+.PHONY: install
+install: build
+	$(INSTALL_DIR) $(DESTDIR)$(libdir)
+	$(INSTALL_DIR) $(DESTDIR)$(includedir)
+	$(INSTALL_SHLIB) libxenforeignmemory.so.$(MAJOR).$(MINOR) $(DESTDIR)$(libdir)
+	$(INSTALL_DATA) libxenforeignmemory.a $(DESTDIR)$(libdir)
+	$(SYMLINK_SHLIB) libxenforeignmemory.so.$(MAJOR).$(MINOR) $(DESTDIR)$(libdir)/libxenforeignmemory.so.$(MAJOR)
+	$(SYMLINK_SHLIB) libxenforeignmemory.so.$(MAJOR) $(DESTDIR)$(libdir)/libxenforeignmemory.so
+	$(INSTALL_DATA) include/xenforeignmemory.h $(DESTDIR)$(includedir)
+
+.PHONY: TAGS
+TAGS:
+	etags -t *.c *.h
+
+.PHONY: clean
+clean:
+	rm -rf *.rpm $(LIB) *~ $(DEPS) $(LIB_OBJS) $(PIC_OBJS)
+	rm -f libxenforeignmemory.so.$(MAJOR).$(MINOR) libxenforeignmemory.so.$(MAJOR)
+	rm -f headers.chk
+
+.PHONY: distclean
+distclean: clean
diff --git a/tools/libs/foreignmemory/compat.c b/tools/libs/foreignmemory/compat.c
new file mode 100644
index 0000000..b8c6fc6
--- /dev/null
+++ b/tools/libs/foreignmemory/compat.c
@@ -0,0 +1,72 @@
+/* Compatibility functions for mapping foreign domain's memory.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Split out from xc_foreign_memory.c
+ */
+#if !defined(__NetBSD__) && !defined(__sun__)
+#error Please implement osdep_xenforeignmemory_map directly for new ports.
+#endif
+
+#include "private.h"
+
+void *osdep_xenforeignmemory_map(xenforeignmemory_handle *fmem,
+                                 uint32_t dom, int prot,
+                                 const xen_pfn_t *arr, int *err, unsigned int num)
+{
+    xen_pfn_t *pfn;
+    unsigned int i;
+    void *ret;
+
+    if ((int)num <= 0) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    pfn = malloc(num * sizeof(*pfn));
+    if (!pfn) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    memcpy(pfn, arr, num * sizeof(*arr));
+    ret = osdep_map_foreign_batch(fmem, dom, prot, pfn, num);
+
+    if (ret) {
+        for (i = 0; i < num; ++i)
+            switch (pfn[i] ^ arr[i]) {
+            case 0:
+                err[i] = 0;
+                break;
+            default:
+                err[i] = -EINVAL;
+                break;
+            }
+    } else
+        memset(err, 0, num * sizeof(*err));
+
+    free(pfn);
+
+    return ret;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libs/foreignmemory/core.c b/tools/libs/foreignmemory/core.c
new file mode 100644
index 0000000..21dc7ee
--- /dev/null
+++ b/tools/libs/foreignmemory/core.c
@@ -0,0 +1,84 @@
+/*
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdlib.h>
+
+#include "private.h"
+
+xenforeignmemory_handle *xenforeignmemory_open(xentoollog_logger *logger,
+                                               unsigned open_flags)
+{
+    xenforeignmemory_handle *fmem = malloc(sizeof(*fmem));
+    int rc;
+
+    if (!fmem) return NULL;
+
+    fmem->logger = logger;
+    fmem->logger_tofree = NULL;
+
+    if (!fmem->logger) {
+        fmem->logger = fmem->logger_tofree =
+            (xentoollog_logger*)
+            xtl_createlogger_stdiostream(stderr, XTL_PROGRESS, 0);
+        if (!fmem->logger) goto err;
+    }
+
+    rc = osdep_xenforeignmemory_open(fmem);
+    if ( rc  < 0 ) goto err;
+
+    return fmem;
+
+err:
+    osdep_xenforeignmemory_close(fmem);
+    xtl_logger_destroy(fmem->logger_tofree);
+    free(fmem);
+    return NULL;
+}
+
+int xenforeignmemory_close(xenforeignmemory_handle *fmem)
+{
+    int rc;
+
+    if ( !fmem )
+        return 0;
+
+    rc = osdep_xenforeignmemory_close(fmem);
+    xtl_logger_destroy(fmem->logger_tofree);
+    free(fmem);
+    return rc;
+}
+
+void *xenforeignmemory_map(xenforeignmemory_handle *fmem,
+                           uint32_t dom, int prot,
+                           const xen_pfn_t *arr, int *err, size_t num)
+{
+    return osdep_xenforeignmemory_map(fmem, dom, prot, arr, err, num);
+}
+
+int xenforeignmemory_unmap(xenforeignmemory_handle *fmem,
+                           void *addr, size_t num)
+{
+    return osdep_xenforeignmemory_unmap(fmem, addr, num);
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxc/xc_freebsd_osdep.c b/tools/libs/foreignmemory/freebsd.c
similarity index 76%
rename from tools/libxc/xc_freebsd_osdep.c
rename to tools/libs/foreignmemory/freebsd.c
index 7745d28..4b2815d 100644
--- a/tools/libxc/xc_freebsd_osdep.c
+++ b/tools/libs/foreignmemory/freebsd.c
@@ -3,9 +3,6 @@
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * xc_gnttab functions:
- * Copyright (c) 2007-2008, D G Murray <Derek.Murray@cl.cam.ac.uk>
- *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation;
@@ -30,14 +27,11 @@
 #include <sys/mman.h>
 #include <sys/ioctl.h>
 
-#include <xen/memory.h>
-
-#include "xc_private.h"
+#include "private.h"
 
 #define PRIVCMD_DEV     "/dev/xen/privcmd"
 
-/*------------------------- Privcmd device interface -------------------------*/
-int osdep_privcmd_open(xc_interface *xch)
+int osdep_xenforeignmemory_open(xenforeignmemory_handle *fmem)
 {
     int flags, saved_errno;
     int fd = open(PRIVCMD_DEV, O_RDWR);
@@ -46,7 +40,7 @@ int osdep_privcmd_open(xc_interface *xch)
     {
         PERROR("Could not obtain handle on privileged command interface "
                PRIVCMD_DEV);
-        return -1
+        return -1;
     }
 
     /*
@@ -69,7 +63,7 @@ int osdep_privcmd_open(xc_interface *xch)
         goto error;
     }
 
-    xch->privcmdfd = fd;
+    fmem->fd = fd;
     return 0;
 
  error:
@@ -80,26 +74,25 @@ int osdep_privcmd_open(xc_interface *xch)
     return -1;
 }
 
-int osdep_privcmd_close(xc_interface *xch)
+int osdep_xenforeignmemory_close(xenforeignmemory_handle *fmem)
 {
-    int fd = xch->privcmdfd;
+    int fd = fmem->fd;
     if ( fd == -1 )
         return 0;
     return close(fd);
 }
 
-/*----------------------- Privcmd foreign map interface ----------------------*/
-void *xc_map_foreign_bulk(xc_interface *xch,
-                          uint32_t dom, int prot,
-                          const xen_pfn_t *arr, int *err,
-                          unsigned int num)
+void *osdep_xenforeignmemory_map(xenforeignmemory_handle *fmem,
+                                 uint32_t dom, int prot,
+                                 const xen_pfn_t *arr, int *err,
+                                 unsigned int num)
 {
-    int fd = xch->privcmdfd;
+    int fd = fmem->fd;
     privcmd_mmapbatch_t ioctlx;
     void *addr;
     int rc;
 
-    addr = mmap(NULL, num << XC_PAGE_SHIFT, prot, MAP_SHARED, fd, 0);
+    addr = mmap(NULL, num << PAGE_SHIFT, prot, MAP_SHARED, fd, 0);
     if ( addr == MAP_FAILED )
     {
         PERROR("xc_map_foreign_bulk: mmap failed");
@@ -117,7 +110,7 @@ void *xc_map_foreign_bulk(xc_interface *xch,
     {
         int saved_errno = errno;
         PERROR("xc_map_foreign_bulk: ioctl failed");
-        (void)munmap(addr, num << XC_PAGE_SHIFT);
+        (void)munmap(addr, num << PAGE_SHIFT);
         errno = saved_errno;
         return NULL;
     }
diff --git a/tools/libs/foreignmemory/include/xenforeignmemory.h b/tools/libs/foreignmemory/include/xenforeignmemory.h
new file mode 100644
index 0000000..5b3fa7a
--- /dev/null
+++ b/tools/libs/foreignmemory/include/xenforeignmemory.h
@@ -0,0 +1,71 @@
+/*
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef XENFOREIGNMEMORY_H
+#define XENFOREIGNMEMORY_H
+
+/*
+ * This library allows you to map foreign domain memory, subject to
+ * permissions for both the process and the domain in which the
+ * process runs.
+ */
+
+#include <stdint.h>
+#include <stddef.h>
+
+#include <xen/xen.h>
+
+/* Callers who don't care don't need to #include <xentoollog.h> */
+typedef struct xentoollog_logger xentoollog_logger;
+
+typedef struct xenforeignmemory_handle xenforeignmemory_handle;
+
+/*
+ * Return a handle onto the hypercall driver.  Logs errors.
+ */
+xenforeignmemory_handle *xenforeignmemory_open(xentoollog_logger *logger,
+                                               unsigned open_flags);
+
+/*
+ * Close a handle previously allocated with xenforeignmemory_open().
+ */
+int xenforeignmemory_close(xenforeignmemory_handle *fmem);
+
+/*
+ * Maps a range within one domain to a local address range.  Mappings
+ * should be unmapped with munmap and should follow the same rules as mmap
+ * regarding page alignment.
+ *
+ * prot is as for mmap(2).
+ *
+ * Can partially succeed. When a page cannot be mapped, its respective
+ * field in @err is set to the corresponding errno value.
+ *
+ * Returns NULL if no pages can be mapped.
+ */
+void *xenforeignmemory_map(xenforeignmemory_handle *fmem, uint32_t dom,
+                           int prot, const xen_pfn_t *arr, int *err,
+                           unsigned int num);
+
+#endif
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libs/foreignmemory/libxenforeignmemory.map b/tools/libs/foreignmemory/libxenforeignmemory.map
new file mode 100644
index 0000000..11f0d2b
--- /dev/null
+++ b/tools/libs/foreignmemory/libxenforeignmemory.map
@@ -0,0 +1,7 @@
+VERS_1.0 {
+	global:
+		xenforeignmemory_open;
+		xenforeignmemory_close;
+		xenforeignmemory_map;
+	local: *; /* Do not expose anything by default */
+};
diff --git a/tools/libxc/xc_linux_osdep.c b/tools/libs/foreignmemory/linux.c
similarity index 83%
rename from tools/libxc/xc_linux_osdep.c
rename to tools/libs/foreignmemory/linux.c
index acf44ce..b403ca7 100644
--- a/tools/libxc/xc_linux_osdep.c
+++ b/tools/libs/foreignmemory/linux.c
@@ -1,11 +1,4 @@
- /******************************************************************************
- *
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
- *
- * xc_gnttab functions:
- * Copyright (c) 2007-2008, D G Murray <Derek.Murray@cl.cam.ac.uk>
- *
+/*
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation;
@@ -18,27 +11,26 @@
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Split out from xc_linus_osdep.c:
+ *
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  */
 
+#include <alloca.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
 #include <unistd.h>
+#include <string.h>
 
 #include <sys/mman.h>
 #include <sys/ioctl.h>
 
-#include <xen/memory.h>
-
-#include "xenctrl.h"
-
-#include "xc_private.h"
+#include "private.h"
 
 #define ROUNDUP(_x,_w) (((unsigned long)(_x)+(1UL<<(_w))-1) & ~((1UL<<(_w))-1))
 
-int osdep_privcmd_open(xc_interface *xch)
+int osdep_xenforeignmemory_open(xenforeignmemory_handle *fmem)
 {
     int flags, saved_errno;
     int fd = open("/dev/xen/privcmd", O_RDWR); /* prefer this newer interface */
@@ -73,7 +65,7 @@ int osdep_privcmd_open(xc_interface *xch)
         goto error;
     }
 
-    xch->privcmdfd = fd;
+    fmem->fd = fd;
     return 0;
 
  error:
@@ -83,9 +75,9 @@ int osdep_privcmd_open(xc_interface *xch)
     return -1;
 }
 
-int osdep_privcmd_close(xc_interface *xch)
+int osdep_xenforeignmemory_close(xenforeignmemory_handle *fmem)
 {
-    int fd = xch->privcmdfd;
+    int fd = fmem->fd;
     if (fd == -1)
         return 0;
     return close(fd);
@@ -127,7 +119,7 @@ static int retry_paged(int fd, uint32_t dom, void *addr,
 {
     privcmd_mmapbatch_v2_t ioctlx;
     int rc, paged = 0, i = 0;
-    
+
     do
     {
         /* Skip gfns not in paging state */
@@ -142,10 +134,10 @@ static int retry_paged(int fd, uint32_t dom, void *addr,
         /* At least one gfn is still in paging state */
         ioctlx.num = 1;
         ioctlx.dom = dom;
-        ioctlx.addr = (unsigned long)addr + ((unsigned long)i<<XC_PAGE_SHIFT);
+        ioctlx.addr = (unsigned long)addr + ((unsigned long)i<<PAGE_SHIFT);
         ioctlx.arr = arr + i;
         ioctlx.err = err + i;
-        
+
         /* Assemble a batch of requests */
         while ( ++i < num )
         {
@@ -153,34 +145,34 @@ static int retry_paged(int fd, uint32_t dom, void *addr,
                 break;
             ioctlx.num++;
         }
-        
+
         /* Send request and abort on fatal error */
         rc = ioctl(fd, IOCTL_PRIVCMD_MMAPBATCH_V2, &ioctlx);
         if ( rc < 0 && errno != ENOENT )
             goto out;
 
     } while ( i < num );
-    
+
     rc = paged;
 out:
     return rc;
 }
 
-void *xc_map_foreign_bulk(xc_interface *xch,
-                          uint32_t dom, int prot,
-                          const xen_pfn_t *arr, int *err, unsigned int num)
+void *osdep_xenforeignmemory_map(xenforeignmemory_handle *fmem,
+                                 uint32_t dom, int prot,
+                                 const xen_pfn_t *arr, int *err, unsigned int num)
 {
-    int fd = xch->privcmdfd;
+    int fd = fmem->fd;
     privcmd_mmapbatch_v2_t ioctlx;
     void *addr;
     unsigned int i;
     int rc;
 
-    addr = mmap(NULL, (unsigned long)num << XC_PAGE_SHIFT, prot, MAP_SHARED,
+    addr = mmap(NULL, (unsigned long)num << PAGE_SHIFT, prot, MAP_SHARED,
                 fd, 0);
     if ( addr == MAP_FAILED )
     {
-        PERROR("xc_map_foreign_bulk: mmap failed");
+        PERROR("mmap failed");
         return NULL;
     }
 
@@ -209,9 +201,9 @@ void *xc_map_foreign_bulk(xc_interface *xch,
          */
         privcmd_mmapbatch_t ioctlx;
         xen_pfn_t *pfn;
-        unsigned int pfn_arr_size = ROUNDUP((num * sizeof(*pfn)), XC_PAGE_SHIFT);
+        unsigned int pfn_arr_size = ROUNDUP((num * sizeof(*pfn)), PAGE_SHIFT);
 
-        if ( pfn_arr_size <= XC_PAGE_SIZE )
+        if ( pfn_arr_size <= PAGE_SIZE )
             pfn = alloca(num * sizeof(*pfn));
         else
         {
@@ -219,8 +211,8 @@ void *xc_map_foreign_bulk(xc_interface *xch,
                        MAP_PRIVATE | MAP_ANON | MAP_POPULATE, -1, 0);
             if ( pfn == MAP_FAILED )
             {
-                PERROR("xc_map_foreign_bulk: mmap of pfn array failed");
-                (void)munmap(addr, (unsigned long)num << XC_PAGE_SHIFT);
+                PERROR("mmap of pfn array failed");
+                (void)munmap(addr, (unsigned long)num << PAGE_SHIFT);
                 return NULL;
             }
         }
@@ -253,7 +245,7 @@ void *xc_map_foreign_bulk(xc_interface *xch,
                     continue;
                 }
                 rc = map_foreign_batch_single(fd, dom, pfn + i,
-                        (unsigned long)addr + ((unsigned long)i<<XC_PAGE_SHIFT));
+                        (unsigned long)addr + ((unsigned long)i<<PAGE_SHIFT));
                 if ( rc < 0 )
                 {
                     rc = -errno;
@@ -265,7 +257,7 @@ void *xc_map_foreign_bulk(xc_interface *xch,
             break;
         }
 
-        if ( pfn_arr_size > XC_PAGE_SIZE )
+        if ( pfn_arr_size > PAGE_SIZE )
             munmap(pfn, pfn_arr_size);
 
         if ( rc == -ENOENT && i == num )
@@ -281,8 +273,8 @@ void *xc_map_foreign_bulk(xc_interface *xch,
     {
         int saved_errno = errno;
 
-        PERROR("xc_map_foreign_bulk: ioctl failed");
-        (void)munmap(addr, (unsigned long)num << XC_PAGE_SHIFT);
+        PERROR("ioctl failed");
+        (void)munmap(addr, (unsigned long)num << PAGE_SHIFT);
         errno = saved_errno;
         return NULL;
     }
diff --git a/tools/libs/foreignmemory/minios.c b/tools/libs/foreignmemory/minios.c
new file mode 100644
index 0000000..a542e3d
--- /dev/null
+++ b/tools/libs/foreignmemory/minios.c
@@ -0,0 +1,62 @@
+/*
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Split out from xc_minios.c
+ *
+ * Copyright 2007-2008 Samuel Thibault <samuel.thibault@eu.citrix.com>.
+ */
+
+#include <mini-os/types.h>
+#include <mini-os/os.h>
+#include <mini-os/mm.h>
+#include <mini-os/lib.h>
+
+#include <errno.h>
+
+#include <sys/mman.h>
+
+#include "private.h"
+
+int osdep_xenforeignmemory_open(xenforeignmemory_handle *fmem)
+{
+    /* No fd required */
+    return 0;
+}
+
+int osdep_xenforeignmemory_close(xenforeignmemory_handle *fmem)
+{
+    return 0;
+}
+
+void *osdep_xenforeignmemory_map(xenforeignmemory_handle *fmem,
+                                 uint32_t dom, int prot,
+                                 const xen_pfn_t *arr, int *err, unsigned int num)
+{
+    unsigned long pt_prot = 0;
+    if (prot & PROT_READ)
+	pt_prot = L1_PROT_RO;
+    if (prot & PROT_WRITE)
+	pt_prot = L1_PROT;
+    return map_frames_ex(arr, num, 1, 0, 1, dom, err, pt_prot);
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libs/foreignmemory/netbsd.c b/tools/libs/foreignmemory/netbsd.c
new file mode 100644
index 0000000..704a096
--- /dev/null
+++ b/tools/libs/foreignmemory/netbsd.c
@@ -0,0 +1,105 @@
+/******************************************************************************
+ *
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+
+#include "private.h"
+
+int osdep_xenforeignmemory_open(xenforeignmemory_handle *fmem)
+{
+    int flags, saved_errno;
+    int fd = open("/kern/xen/privcmd", O_RDWR);
+
+    if ( fd == -1 )
+    {
+        PERROR("Could not obtain handle on privileged command interface");
+        return -1;
+    }
+
+    /* Although we return the file handle as the 'xc handle' the API
+       does not specify / guarentee that this integer is in fact
+       a file handle. Thus we must take responsiblity to ensure
+       it doesn't propagate (ie leak) outside the process */
+    if ( (flags = fcntl(fd, F_GETFD)) < 0 )
+    {
+        PERROR("Could not get file handle flags");
+        goto error;
+    }
+    flags |= FD_CLOEXEC;
+    if ( fcntl(fd, F_SETFD, flags) < 0 )
+    {
+        PERROR("Could not set file handle flags");
+        goto error;
+    }
+
+    fmem->fd = fd;
+    return 0;
+
+ error:
+    saved_errno = errno;
+    close(fd);
+    errno = saved_errno;
+    return -1;
+}
+
+int osdep_xenforeignmemory_close(xenforeignmemory_handle *fmem)
+{
+    int fd = fmem->fd;
+    return close(fd);
+}
+
+void *osdep_map_foreign_batch(xenforeignmem_handle *fmem, uint32_t dom,
+                              int prot, xen_pfn_t *arr, int num)
+{
+    int fd = fmem->fd;
+    privcmd_mmapbatch_t ioctlx;
+    void *addr;
+    addr = mmap(NULL, num*XC_PAGE_SIZE, prot, MAP_ANON | MAP_SHARED, -1, 0);
+    if ( addr == MAP_FAILED ) {
+        PERROR("osdep_map_foreign_batch: mmap failed");
+        return NULL;
+    }
+
+    ioctlx.num=num;
+    ioctlx.dom=dom;
+    ioctlx.addr=(unsigned long)addr;
+    ioctlx.arr=arr;
+    if ( ioctl(fd, IOCTL_PRIVCMD_MMAPBATCH, &ioctlx) < 0 )
+    {
+        int saved_errno = errno;
+        PERROR("osdep_map_foreign_batch: ioctl failed");
+        (void)munmap(addr, num*XC_PAGE_SIZE);
+        errno = saved_errno;
+        return NULL;
+    }
+    return addr;
+
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libs/foreignmemory/private.h b/tools/libs/foreignmemory/private.h
new file mode 100644
index 0000000..36a71ec
--- /dev/null
+++ b/tools/libs/foreignmemory/private.h
@@ -0,0 +1,53 @@
+#ifndef XENFOREIGNMEMORY_PRIVATE_H
+#define XENFOREIGNMEMORY_PRIVATE_H
+
+#include <xentoollog.h>
+
+#include <xenforeignmemory.h>
+
+#include <xen/xen.h>
+#include <xen/sys/privcmd.h>
+
+#ifndef PAGE_SHIFT /* Mini-os, Yukk */
+#define PAGE_SHIFT           12
+#endif
+#ifndef __MINIOS__ /* Yukk */
+#define PAGE_SIZE            (1UL << PAGE_SHIFT)
+#define PAGE_MASK            (~(PAGE_SIZE-1))
+#endif
+
+struct xenforeignmemory_handle {
+    xentoollog_logger *logger, *logger_tofree;
+    unsigned flags;
+    int fd;
+};
+
+int osdep_xenforeignmemory_open(xenforeignmemory_handle *fmem);
+int osdep_xenforeignmemory_close(xenforeignmemory_handle *fmem);
+
+void *osdep_xenforeignmemory_map(xenforeignmemory_handle *fmem,
+                                 uint32_t dom, int prot,
+                                 const xen_pfn_t *arr, int *err, size_t num);
+int osdep_xenforeignmemory_unmap(xenforeignmemory_handle *fmem,
+                                 void *addr, size_t num);
+
+#if defined(__NetBSD__) || defined(__sun__)
+/* Strictly compat for those two only only */
+void *compat_mapforeign_batch(xenforeignmem_handle *fmem, uint32_t dom,
+                              int prot, xen_pfn_t *arr, int num);
+#endif
+
+#define PERROR(_f...) \
+	xtl_log(fmem->logger, XTL_ERROR, errno, "xenforeignmemory", _f)
+
+#endif
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libs/foreignmemory/solaris.c b/tools/libs/foreignmemory/solaris.c
new file mode 100644
index 0000000..f4dd634
--- /dev/null
+++ b/tools/libs/foreignmemory/solaris.c
@@ -0,0 +1,93 @@
+/******************************************************************************
+ *
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "xc_private.h"
+
+#include <xen/memory.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <malloc.h>
+
+int osdep_xenforeignmemory_open(xenforeignmemory_handle *fmem)
+{
+    int flags, saved_errno;
+    int fd = open("/dev/xen/privcmd", O_RDWR);
+
+    if ( fd == -1 )
+    {
+        PERROR("Could not obtain handle on privileged command interface");
+        return -1;
+    }
+
+    /* Although we return the file handle as the 'xc handle' the API
+       does not specify / guarentee that this integer is in fact
+       a file handle. Thus we must take responsiblity to ensure
+       it doesn't propagate (ie leak) outside the process */
+    if ( (flags = fcntl(fd, F_GETFD)) < 0 )
+    {
+        PERROR("Could not get file handle flags");
+        goto error;
+    }
+    flags |= FD_CLOEXEC;
+    if ( fcntl(fd, F_SETFD, flags) < 0 )
+    {
+        PERROR("Could not set file handle flags");
+        goto error;
+    }
+
+    fmem->fd = fd;
+    return 0;
+
+ error:
+    saved_errno = errno;
+    close(fd);
+    errno = saved_errno;
+    return -1;
+}
+
+int osdep_xenforeignmemory_close(xenforeignmemory_handle *fmem)
+{
+    int fd = fmem->fd;
+    return close(fd);
+}
+
+void *osdep_map_foreign_batch(xenforeignmem_handle *fmem, uint32_t dom,
+                              int prot, xen_pfn_t *arr, int num)
+{
+    int fd = fmem->fd;
+    privcmd_mmapbatch_t ioctlx;
+    void *addr;
+    addr = mmap(NULL, num*XC_PAGE_SIZE, prot, MAP_SHARED, fd, 0);
+    if ( addr == MAP_FAILED )
+        return NULL;
+
+    ioctlx.num=num;
+    ioctlx.dom=dom;
+    ioctlx.addr=(unsigned long)addr;
+    ioctlx.arr=arr;
+    if ( ioctl(fd, IOCTL_PRIVCMD_MMAPBATCH, &ioctlx) < 0 )
+    {
+        int saved_errno = errno;
+        PERROR("XXXXXXXX");
+        (void)munmap(addr, num*XC_PAGE_SIZE);
+        errno = saved_errno;
+        return NULL;
+    }
+    return addr;
+}
diff --git a/tools/libxc/Makefile b/tools/libxc/Makefile
index a122f73..ca8bab7 100644
--- a/tools/libxc/Makefile
+++ b/tools/libxc/Makefile
@@ -42,8 +42,8 @@ CTRL_SRCS-y       += xc_kexec.c
 CTRL_SRCS-y       += xc_resource.c
 CTRL_SRCS-$(CONFIG_X86) += xc_psr.c
 CTRL_SRCS-$(CONFIG_X86) += xc_pagetab.c
-CTRL_SRCS-$(CONFIG_Linux) += xc_linux.c xc_linux_osdep.c
-CTRL_SRCS-$(CONFIG_FreeBSD) += xc_freebsd.c xc_freebsd_osdep.c
+CTRL_SRCS-$(CONFIG_Linux) += xc_linux.c
+CTRL_SRCS-$(CONFIG_FreeBSD) += xc_freebsd.c
 CTRL_SRCS-$(CONFIG_SunOS) += xc_solaris.c
 CTRL_SRCS-$(CONFIG_NetBSD) += xc_netbsd.c
 CTRL_SRCS-$(CONFIG_NetBSDRump) += xc_netbsd.c
@@ -125,7 +125,7 @@ $(CTRL_PIC_OBJS) $(GUEST_PIC_OBJS): CFLAGS += -include $(XEN_ROOT)/tools/config.
 # libxenguest includes xc_private.h, so needs this despite not using
 # this functionality directly.
 $(CTRL_LIB_OBJS) $(GUEST_LIB_OBJS) \
-$(CTRL_PIC_OBJS) $(GUEST_PIC_OBJS): CFLAGS += $(CFLAGS_libxencall)
+$(CTRL_PIC_OBJS) $(GUEST_PIC_OBJS): CFLAGS += $(CFLAGS_libxencall) $(CFLAGS_libxenforeignmemory)
 
 $(CTRL_LIB_OBJS) $(CTRL_PIC_OBJS): CFLAGS += $(CFLAGS_libxengnttab) $(CFLAGS_libxengntshr)
 
@@ -207,7 +207,7 @@ libxenctrl.so.$(MAJOR): libxenctrl.so.$(MAJOR).$(MINOR)
 	$(SYMLINK_SHLIB) $< $@
 
 libxenctrl.so.$(MAJOR).$(MINOR): $(CTRL_PIC_OBJS)
-	$(CC) $(LDFLAGS) $(PTHREAD_LDFLAGS) -Wl,$(SONAME_LDFLAG) -Wl,libxenctrl.so.$(MAJOR) $(SHLIB_LDFLAGS) -o $@ $^ $(LDLIBS_libxentoollog) $(LDLIBS_libxenevtchn) $(LDLIBS_libxengnttab) $(LDLIBS_libxengntshr) $(LDLIBS_libxencall) $(PTHREAD_LIBS) $(APPEND_LDFLAGS)
+	$(CC) $(LDFLAGS) $(PTHREAD_LDFLAGS) -Wl,$(SONAME_LDFLAG) -Wl,libxenctrl.so.$(MAJOR) $(SHLIB_LDFLAGS) -o $@ $^ $(LDLIBS_libxentoollog) $(LDLIBS_libxenevtchn) $(LDLIBS_libxengnttab) $(LDLIBS_libxengntshr) $(LDLIBS_libxencall) $(LDLIBS_libxenforeignmemory) $(PTHREAD_LIBS) $(APPEND_LDFLAGS)
 
 # libxenguest
 
diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index cb41c07..76beafd 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -1369,32 +1369,6 @@ int xc_lockprof_query(xc_interface *xch,
 void *xc_memalign(xc_interface *xch, size_t alignment, size_t size);
 
 /**
- * Memory maps a range within one domain to a local address range.  Mappings
- * should be unmapped with munmap and should follow the same rules as mmap
- * regarding page alignment.  Returns NULL on failure.
- *
- * @parm xch a handle on an open hypervisor interface
- * @parm dom the domain to map memory from
- * @parm size the amount of memory to map (in multiples of page size)
- * @parm prot same flag as in mmap().
- * @parm mfn the frame address to map.
- */
-void *xc_map_foreign_range(xc_interface *xch, uint32_t dom,
-                            int size, int prot,
-                            unsigned long mfn );
-
-void *xc_map_foreign_pages(xc_interface *xch, uint32_t dom, int prot,
-                           const xen_pfn_t *arr, int num );
-
-/**
- * Like xc_map_foreign_pages(), except it can succeed partially.
- * When a page cannot be mapped, its respective field in @err is
- * set to the corresponding errno value.
- */
-void *xc_map_foreign_bulk(xc_interface *xch, uint32_t dom, int prot,
-                          const xen_pfn_t *arr, int *err, unsigned int num);
-
-/**
  * Translates a virtual address in the context of a given domain and
  * vcpu returning the GFN containing the address (that is, an MFN for 
  * PV guests, a PFN for HVM guests).  Returns 0 for failure.
diff --git a/tools/libxc/include/xenctrl_compat.h b/tools/libxc/include/xenctrl_compat.h
index d99fa11..54c23a4 100644
--- a/tools/libxc/include/xenctrl_compat.h
+++ b/tools/libxc/include/xenctrl_compat.h
@@ -12,6 +12,42 @@
 #ifndef XENCTRL_COMPAT_H
 #define XENCTRL_COMPAT_H
 
+#ifdef XC_WANT_COMPAT_MAP_FOREIGN_API
+/**
+ * Memory maps a range within one domain to a local address range.  Mappings
+ * should be unmapped with munmap and should follow the same rules as mmap
+ * regarding page alignment.  Returns NULL on failure.
+ *
+ * @parm xch a handle on an open hypervisor interface
+ * @parm dom the domain to map memory from
+ * @parm size the amount of memory to map (in multiples of page size)
+ * @parm prot same flag as in mmap().
+ * @parm mfn the frame address to map.
+ */
+void *xc_map_foreign_range(xc_interface *xch, uint32_t dom,
+                            int size, int prot,
+                            unsigned long mfn );
+
+void *xc_map_foreign_pages(xc_interface *xch, uint32_t dom, int prot,
+                           const xen_pfn_t *arr, int num );
+
+/* Nothing within the library itself other than the compat wrapper
+ * itself should be using this, everything inside has access to
+ * xenforeignmemory_map().
+ */
+#if !defined(XC_INTERNAL_COMPAT_MAP_FOREIGN_API) || \
+     defined(XC_BUILDING_COMPAT_MAP_FOREIGN_API)
+/**
+ * Like xc_map_foreign_pages(), except it can succeed partially.
+ * When a page cannot be mapped, its respective field in @err is
+ * set to the corresponding errno value.
+ */
+void *xc_map_foreign_bulk(xc_interface *xch, uint32_t dom, int prot,
+                          const xen_pfn_t *arr, int *err, unsigned int num);
+#endif
+
+#endif
+
 #ifdef XC_WANT_COMPAT_EVTCHN_API
 
 typedef struct xenevtchn_handle xc_evtchn;
diff --git a/tools/libxc/xc_foreign_memory.c b/tools/libxc/xc_foreign_memory.c
index d1130e6..1737c10 100644
--- a/tools/libxc/xc_foreign_memory.c
+++ b/tools/libxc/xc_foreign_memory.c
@@ -17,6 +17,7 @@
  * License along with this library; If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define XC_BUILDING_COMPAT_MAP_FOREIGN_API
 #include "xc_private.h"
 
 void *xc_map_foreign_pages(xc_interface *xch, uint32_t dom, int prot,
@@ -34,7 +35,7 @@ void *xc_map_foreign_pages(xc_interface *xch, uint32_t dom, int prot,
     if (!err)
         return NULL;
 
-    res = xc_map_foreign_bulk(xch, dom, prot, arr, err, num);
+    res = xenforeignmemory_map(xch->fmem, dom, prot, arr, err, num);
     if (res) {
         for (i = 0; i < num; i++) {
             if (err[i]) {
@@ -100,53 +101,11 @@ void *xc_map_foreign_ranges(xc_interface *xch,
     return ret;
 }
 
-/*
- * stub for all not yet converted OSes (NetBSD and Solaris). New OSes should
- * just implement xc_map_foreign_bulk.
- */
-#if defined(__NetBSD__) || defined(__sun__)
-void *osdep_map_foreign_batch(xc_interface *xch, uint32_t dom, int prot,
-                              xen_pfn_t *arr, int num );
-void *xc_map_foreign_bulk(xc_interface *xch,
-                          uint32_t dom, int prot,
+void *xc_map_foreign_bulk(xc_interface *xch, uint32_t dom, int prot,
                           const xen_pfn_t *arr, int *err, unsigned int num)
 {
-    xen_pfn_t *pfn;
-    unsigned int i;
-    void *ret;
-
-    if ((int)num <= 0) {
-        errno = EINVAL;
-        return NULL;
-    }
-
-    pfn = malloc(num * sizeof(*pfn));
-    if (!pfn) {
-        errno = ENOMEM;
-        return NULL;
-    }
-
-    memcpy(pfn, arr, num * sizeof(*arr));
-    ret = osdep_map_foreign_batch(xch, dom, prot, pfn, num);
-
-    if (ret) {
-        for (i = 0; i < num; ++i)
-            switch (pfn[i] ^ arr[i]) {
-            case 0:
-                err[i] = 0;
-                break;
-            default:
-                err[i] = -EINVAL;
-                break;
-            }
-    } else
-        memset(err, 0, num * sizeof(*err));
-
-    free(pfn);
-
-    return ret;
+    return xenforeignmemory_map(xch->fmem, dom, prot, arr, err, num);
 }
-#endif
 
 /*
  * Local variables:
diff --git a/tools/libxc/xc_minios.c b/tools/libxc/xc_minios.c
index d8ca948..1799daa 100644
--- a/tools/libxc/xc_minios.c
+++ b/tools/libxc/xc_minios.c
@@ -39,40 +39,11 @@ void minios_interface_close_fd(int fd);
 
 extern void minios_interface_close_fd(int fd);
 
-int osdep_privcmd_open(xc_interface *xch)
-{
-    int fd = alloc_fd(FTYPE_XC);
-
-    if ( fd == -1)
-        return -1;
-
-    xch->privcmdfd = fd;
-    return 0;
-}
-
-int osdep_privcmd_close(xc_interface *xch)
-{
-    int fd = xch->privcmdfd;
-    return close(fd);
-}
-
 void minios_interface_close_fd(int fd)
 {
     files[fd].type = FTYPE_NONE;
 }
 
-void *xc_map_foreign_bulk(xc_interface *xch,
-                          uint32_t dom, int prot,
-                          const xen_pfn_t *arr, int *err, unsigned int num)
-{
-    unsigned long pt_prot = 0;
-    if (prot & PROT_READ)
-	pt_prot = L1_PROT_RO;
-    if (prot & PROT_WRITE)
-	pt_prot = L1_PROT;
-    return map_frames_ex(arr, num, 1, 0, 1, dom, err, pt_prot);
-}
-
 /* Optionally flush file to disk and discard page cache */
 void discard_file_cache(xc_interface *xch, int fd, int flush)
 {
diff --git a/tools/libxc/xc_netbsd.c b/tools/libxc/xc_netbsd.c
index 3470bc4..3197993 100644
--- a/tools/libxc/xc_netbsd.c
+++ b/tools/libxc/xc_netbsd.c
@@ -22,79 +22,6 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <malloc.h>
-#include <sys/mman.h>
-
-int osdep_privcmd_open(xc_interface *xch)
-{
-    int flags, saved_errno;
-    int fd = open("/kern/xen/privcmd", O_RDWR);
-
-    if ( fd == -1 )
-    {
-        PERROR("Could not obtain handle on privileged command interface");
-        return -1;
-    }
-
-    /* Although we return the file handle as the 'xc handle' the API
-       does not specify / guarentee that this integer is in fact
-       a file handle. Thus we must take responsiblity to ensure
-       it doesn't propagate (ie leak) outside the process */
-    if ( (flags = fcntl(fd, F_GETFD)) < 0 )
-    {
-        PERROR("Could not get file handle flags");
-        goto error;
-    }
-    flags |= FD_CLOEXEC;
-    if ( fcntl(fd, F_SETFD, flags) < 0 )
-    {
-        PERROR("Could not set file handle flags");
-        goto error;
-    }
-
-    xch->privcmdfd = fd;
-    return 0;
-
- error:
-    saved_errno = errno;
-    close(fd);
-    errno = saved_errno;
-    return -1;
-}
-
-int osdep_privcmd_close(xc_interface *xch)
-{
-    int fd = xch->privcmdfd;
-    return close(fd);
-}
-
-void *osdep_map_foreign_batch(xc_interface *xch,
-                              uint32_t dom, int prot,
-                              xen_pfn_t *arr, int num)
-{
-    int fd = xch->privcmdfd;
-    privcmd_mmapbatch_t ioctlx;
-    void *addr;
-    addr = mmap(NULL, num*XC_PAGE_SIZE, prot, MAP_ANON | MAP_SHARED, -1, 0);
-    if ( addr == MAP_FAILED ) {
-        PERROR("osdep_map_foreign_batch: mmap failed");
-        return NULL;
-    }
-
-    ioctlx.num=num;
-    ioctlx.dom=dom;
-    ioctlx.addr=(unsigned long)addr;
-    ioctlx.arr=arr;
-    if ( ioctl(fd, IOCTL_PRIVCMD_MMAPBATCH, &ioctlx) < 0 )
-    {
-        int saved_errno = errno;
-        PERROR("osdep_map_foreign_batch: ioctl failed");
-        (void)munmap(addr, num*XC_PAGE_SIZE);
-        errno = saved_errno;
-        return NULL;
-    }
-    return addr;
-
-}
 
 /* Optionally flush file to disk and discard page cache */
 void discard_file_cache(xc_interface *xch, int fd, int flush) 
diff --git a/tools/libxc/xc_private.c b/tools/libxc/xc_private.c
index ca4c17a..c41e433 100644
--- a/tools/libxc/xc_private.c
+++ b/tools/libxc/xc_private.c
@@ -58,11 +58,12 @@ struct xc_interface_core *xc_interface_open(xentoollog_logger *logger,
     if (open_flags & XC_OPENFLAG_DUMMY)
         return xch; /* We are done */
 
-    if ( osdep_privcmd_open(xch) < 0 )
-        goto err;
-
     xch->xcall = xencall_open(xch->error_handler,
         open_flags & XC_OPENFLAG_NON_REENTRANT ? XENCALL_OPENFLAG_NON_REENTRANT : 0U);
+    if ( xch->xcall == NULL )
+        goto err;
+
+    xch->fmem = xenforeignmemory_open(xch->error_handler, 0);
 
     if ( xch->xcall == NULL )
         goto err;
@@ -70,7 +71,7 @@ struct xc_interface_core *xc_interface_open(xentoollog_logger *logger,
     return xch;
 
  err:
-    osdep_privcmd_close(xch);
+    xencall_close(xch->xcall);
     xtl_logger_destroy(xch->error_handler_tofree);
     if (xch != &xch_buf) free(xch);
     return NULL;
@@ -86,8 +87,8 @@ int xc_interface_close(xc_interface *xch)
     rc = xencall_close(xch->xcall);
     if (rc) PERROR("Could not close xencall interface");
 
-    rc = osdep_privcmd_close(xch);
-    if (rc) PERROR("Could not close hypervisor interface");
+    rc = xenforeignmemory_close(xch->fmem);
+    if (rc) PERROR("Could not close foreign memory interface");
 
     xtl_logger_destroy(xch->dombuild_logger_tofree);
     xtl_logger_destroy(xch->error_handler_tofree);
diff --git a/tools/libxc/xc_private.h b/tools/libxc/xc_private.h
index ecf2451..aa8daf1 100644
--- a/tools/libxc/xc_private.h
+++ b/tools/libxc/xc_private.h
@@ -29,8 +29,13 @@
 #include <sys/ioctl.h>
 
 #include "_paths.h"
+
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
+#define XC_INTERNAL_COMPAT_MAP_FOREIGN_API
 #include "xenctrl.h"
+
 #include <xencall.h>
+#include <xenforeignmemory.h>
 
 #include <xen/sys/privcmd.h>
 
@@ -94,11 +99,11 @@ struct xc_interface_core {
     FILE *dombuild_logger_file;
     const char *currently_progress_reporting;
 
-    /* Privcmd interface */
-    int privcmdfd;
-
     /* Hypercall interface */
     xencall_handle *xcall;
+
+    /* Foreign mappings */
+    xenforeignmemory_handle *fmem;
 };
 
 int osdep_privcmd_open(xc_interface *xch);
diff --git a/tools/libxc/xc_solaris.c b/tools/libxc/xc_solaris.c
index d686867..5128f3f 100644
--- a/tools/libxc/xc_solaris.c
+++ b/tools/libxc/xc_solaris.c
@@ -19,81 +19,8 @@
 
 #include "xc_private.h"
 
-#include <xen/memory.h>
-#include <unistd.h>
-#include <fcntl.h>
 #include <malloc.h>
 
-int osdep_privcmd_open(xc_interface *xch)
-{
-    int flags, saved_errno;
-    int fd = open("/dev/xen/privcmd", O_RDWR);
-
-    if ( fd == -1 )
-    {
-        PERROR("Could not obtain handle on privileged command interface");
-        return -1;
-    }
-
-    /* Although we return the file handle as the 'xc handle' the API
-       does not specify / guarentee that this integer is in fact
-       a file handle. Thus we must take responsiblity to ensure
-       it doesn't propagate (ie leak) outside the process */
-    if ( (flags = fcntl(fd, F_GETFD)) < 0 )
-    {
-        PERROR("Could not get file handle flags");
-        goto error;
-    }
-    flags |= FD_CLOEXEC;
-    if ( fcntl(fd, F_SETFD, flags) < 0 )
-    {
-        PERROR("Could not set file handle flags");
-        goto error;
-    }
-
-    xch->privcmdfd = fd;
-    return 0;
-
- error:
-    saved_errno = errno;
-    close(fd);
-    errno = saved_errno;
-    return -1;
-}
-
-int osdep_privcmd_close(xc_interface *xch)
-{
-    int fd = xch->privcmdfd;
-    return close(fd);
-}
-
-void *osdep_map_foreign_batch(xc_interface *xch,
-                              uint32_t dom, int prot,
-                              xen_pfn_t *arr, int num)
-{
-    int fd = xch->privcmdfd;
-    privcmd_mmapbatch_t ioctlx;
-    void *addr;
-    addr = mmap(NULL, num*XC_PAGE_SIZE, prot, MAP_SHARED, fd, 0);
-    if ( addr == MAP_FAILED )
-        return NULL;
-
-    ioctlx.num=num;
-    ioctlx.dom=dom;
-    ioctlx.addr=(unsigned long)addr;
-    ioctlx.arr=arr;
-    if ( ioctl(fd, IOCTL_PRIVCMD_MMAPBATCH, &ioctlx) < 0 )
-    {
-        int saved_errno = errno;
-        PERROR("XXXXXXXX");
-        (void)munmap(addr, num*XC_PAGE_SIZE);
-        errno = saved_errno;
-        return NULL;
-    }
-    return addr;
-
-}
-
 /* Optionally flush file to disk and discard page cache */
 void discard_file_cache(xc_interface *xch, int fd, int flush) 
 {
diff --git a/tools/libxc/xc_sr_restore.c b/tools/libxc/xc_sr_restore.c
index 05159bb..8e9e9c7 100644
--- a/tools/libxc/xc_sr_restore.c
+++ b/tools/libxc/xc_sr_restore.c
@@ -316,8 +316,8 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned count,
     if ( nr_pages == 0 )
         goto done;
 
-    mapping = guest_page = xc_map_foreign_bulk(
-        xch, ctx->domid, PROT_READ | PROT_WRITE,
+    mapping = guest_page = xenforeignmemory_map(xch->fmem,
+        ctx->domid, PROT_READ | PROT_WRITE,
         mfns, map_errs, nr_pages);
     if ( !mapping )
     {
diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
index 88d85ef..1ff9635 100644
--- a/tools/libxc/xc_sr_save.c
+++ b/tools/libxc/xc_sr_save.c
@@ -153,8 +153,8 @@ static int write_batch(struct xc_sr_context *ctx)
 
     if ( nr_pages > 0 )
     {
-        guest_mapping = xc_map_foreign_bulk(
-            xch, ctx->domid, PROT_READ, mfns, errors, nr_pages);
+        guest_mapping = xenforeignmemory_map(xch->fmem,
+            ctx->domid, PROT_READ, mfns, errors, nr_pages);
         if ( !guest_mapping )
         {
             PERROR("Failed to map guest pages");
diff --git a/tools/libxc/xg_private.h b/tools/libxc/xg_private.h
index a76d6b3..175c1e5 100644
--- a/tools/libxc/xg_private.h
+++ b/tools/libxc/xg_private.h
@@ -26,9 +26,8 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
-#include "xenctrl.h"
-#include "xenguest.h"
 #include "xc_private.h"
+#include "xenguest.h"
 
 #include <xen/memory.h>
 #include <xen/elfnote.h>
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 77bf251..cd0f4cc 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -50,6 +50,7 @@
 
 #include <xenevtchn.h>
 #include <xenstore.h>
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 #include <xenguest.h>
 #include <xc_dom.h>
diff --git a/tools/misc/xen-mfndump.c b/tools/misc/xen-mfndump.c
index ceeeaa9..1b22ad7 100644
--- a/tools/misc/xen-mfndump.c
+++ b/tools/misc/xen-mfndump.c
@@ -1,3 +1,4 @@
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 #include <xc_private.h>
 #include <xc_core.h>
diff --git a/tools/ocaml/libs/xc/xenctrl_stubs.c b/tools/ocaml/libs/xc/xenctrl_stubs.c
index e2fa7e4..74928e9 100644
--- a/tools/ocaml/libs/xc/xenctrl_stubs.c
+++ b/tools/ocaml/libs/xc/xenctrl_stubs.c
@@ -29,6 +29,7 @@
 #include <stdint.h>
 #include <string.h>
 
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 
 #include "mmap_stubs.h"
diff --git a/tools/python/xen/lowlevel/xc/xc.c b/tools/python/xen/lowlevel/xc/xc.c
index 6dd79e0..c40a4e9 100644
--- a/tools/python/xen/lowlevel/xc/xc.c
+++ b/tools/python/xen/lowlevel/xc/xc.c
@@ -5,6 +5,7 @@
  */
 
 #include <Python.h>
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 #include <fcntl.h>
 #include <netinet/in.h>
@@ -15,6 +16,7 @@
 #include <netdb.h>
 #include <arpa/inet.h>
 
+#include <xen/elfnote.h>
 #include <xen/tmem.h>
 #include "xc_dom.h"
 #include <xen/hvm/hvm_info_table.h>
diff --git a/tools/tests/mce-test/tools/xen-mceinj.c b/tools/tests/mce-test/tools/xen-mceinj.c
index d22f0ad..061ec7c 100644
--- a/tools/tests/mce-test/tools/xen-mceinj.c
+++ b/tools/tests/mce-test/tools/xen-mceinj.c
@@ -30,6 +30,7 @@
 #include <signal.h>
 #include <stdarg.h>
 
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 #include <xenguest.h>
 #include <inttypes.h>
diff --git a/tools/tests/mem-sharing/memshrtool.c b/tools/tests/mem-sharing/memshrtool.c
index 6454bc3..437c7c9 100644
--- a/tools/tests/mem-sharing/memshrtool.c
+++ b/tools/tests/mem-sharing/memshrtool.c
@@ -11,6 +11,7 @@
 #include <string.h>
 #include <sys/mman.h>
 
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include "xenctrl.h"
 
 static int usage(const char* prog)
diff --git a/tools/xenmon/xenbaked.c b/tools/xenmon/xenbaked.c
index da46126..eacacb0 100644
--- a/tools/xenmon/xenbaked.c
+++ b/tools/xenmon/xenbaked.c
@@ -38,6 +38,7 @@
 #include <errno.h>
 #include <signal.h>
 #include <xenevtchn.h>
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 #include <xen/xen.h>
 #include <string.h>
diff --git a/tools/xenpaging/pagein.c b/tools/xenpaging/pagein.c
index 7cb0f33..2a298b4 100644
--- a/tools/xenpaging/pagein.c
+++ b/tools/xenpaging/pagein.c
@@ -1,6 +1,5 @@
 /* Trigger a page-in in a separate thread-of-execution to avoid deadlock */
 #include <pthread.h>
-#include <xc_private.h>
 #include "xenpaging.h"
 
 struct page_in_args {
diff --git a/tools/xenpaging/xenpaging.c b/tools/xenpaging/xenpaging.c
index c4bc713..0377507 100644
--- a/tools/xenpaging/xenpaging.c
+++ b/tools/xenpaging/xenpaging.c
@@ -27,7 +27,6 @@
 #include <signal.h>
 #include <unistd.h>
 #include <poll.h>
-#include <xc_private.h>
 #include <xenstore.h>
 #include <getopt.h>
 
diff --git a/tools/xenpaging/xenpaging.h b/tools/xenpaging/xenpaging.h
index d0f8d20..d6c8ee5 100644
--- a/tools/xenpaging/xenpaging.h
+++ b/tools/xenpaging/xenpaging.h
@@ -25,6 +25,8 @@
 
 
 #include <xenevtchn.h>
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
+#include <xenctrl.h>
 #include <xc_private.h>
 #include <xen/event_channel.h>
 #include <xen/vm_event.h>
diff --git a/tools/xenstore/xenstored_core.c b/tools/xenstore/xenstored_core.c
index f62c192..51fb0b3 100644
--- a/tools/xenstore/xenstored_core.c
+++ b/tools/xenstore/xenstored_core.c
@@ -50,7 +50,6 @@
 #include "xenstored_watch.h"
 #include "xenstored_transaction.h"
 #include "xenstored_domain.h"
-#include "xenctrl.h"
 #include "tdb.h"
 
 #include "hashtable.h"
diff --git a/tools/xenstore/xenstored_core.h b/tools/xenstore/xenstored_core.h
index 624737d..3a497f7 100644
--- a/tools/xenstore/xenstored_core.h
+++ b/tools/xenstore/xenstored_core.h
@@ -19,6 +19,7 @@
 #ifndef _XENSTORED_CORE_H
 #define _XENSTORED_CORE_H
 
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 #include <xengnttab.h>
 
diff --git a/tools/xentrace/xenctx.c b/tools/xentrace/xenctx.c
index 4ed120a..e647179 100644
--- a/tools/xentrace/xenctx.c
+++ b/tools/xentrace/xenctx.c
@@ -25,7 +25,8 @@
 #include <getopt.h>
 #include <limits.h>
 
-#include "xenctrl.h"
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
+#include <xenctrl.h>
 #include <xen/foreign/x86_32.h>
 #include <xen/foreign/x86_64.h>
 #include <xen/hvm/save.h>
diff --git a/tools/xentrace/xentrace.c b/tools/xentrace/xentrace.c
index 6cbe0ac..f09fe6c 100644
--- a/tools/xentrace/xentrace.c
+++ b/tools/xentrace/xentrace.c
@@ -30,6 +30,7 @@
 #include <xen/xen.h>
 #include <xen/trace.h>
 
+#define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenevtchn.h>
 #include <xenctrl.h>
 
-- 
2.1.4