[PATCH] linux-user: add option to chroot before emulation

[PATCH] linux-user: add option to chroot before emulation

[Matteo Croce]

From: Matteo Croce <mcroce@microsoft.com>

Add a '-c' option which does a chroot() just before starting the
emulation. This is useful when the static QEMU user binary can't
be copied into the target root filesystem, e.g. if it's readonly.

Move some code which accesses /proc/sys/vm/mmap_min_addr before
the chroot, otherwise it would fail.

Signed-off-by: Matteo Croce <mcroce@microsoft.com>
---
 linux-user/main.c | 128 +++++++++++++++++++++++++++-------------------
 1 file changed, 75 insertions(+), 53 deletions(-)

diff --git a/linux-user/main.c b/linux-user/main.c
index 24d1eb73ad..4788e4b5bc 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -60,6 +60,7 @@ static const char *seed_optarg;
 unsigned long mmap_min_addr;
 unsigned long guest_base;
 bool have_guest_base;
+static const char *qemu_chroot;
 
 /*
  * Used to implement backwards-compatibility for the `-strace`, and
@@ -304,6 +305,11 @@ static void handle_arg_pagesize(const char *arg)
     }
 }
 
+static void handle_arg_chroot(const char *arg)
+{
+    qemu_chroot = arg;
+}
+
 static void handle_arg_seed(const char *arg)
 {
     seed_optarg = arg;
@@ -450,6 +456,8 @@ static const struct qemu_argument arg_table[] = {
      "logfile",     "write logs to 'logfile' (default stderr)"},
     {"p",          "QEMU_PAGESIZE",    true,  handle_arg_pagesize,
      "pagesize",   "set the host page size to 'pagesize'"},
+    {"c",          "QEMU_CHROOT",      true,  handle_arg_chroot,
+     "chroot",     "chroot to 'chroot' before starting emulation"},
     {"singlestep", "QEMU_SINGLESTEP",  false, handle_arg_singlestep,
      "",           "run in singlestep mode"},
     {"strace",     "QEMU_STRACE",      false, handle_arg_strace,
@@ -688,6 +696,73 @@ int main(int argc, char **argv, char **envp)
 
     init_qemu_uname_release();
 
+    /*
+     * Read in mmap_min_addr kernel parameter.  This value is used
+     * When loading the ELF image to determine whether guest_base
+     * is needed.  It is also used in mmap_find_vma.
+     */
+    {
+        FILE *fp;
+
+        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {
+            unsigned long tmp;
+            if (fscanf(fp, "%lu", &tmp) == 1 && tmp != 0) {
+                mmap_min_addr = tmp;
+                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n",
+                              mmap_min_addr);
+            }
+            fclose(fp);
+        }
+    }
+
+    /*
+     * We prefer to not make NULL pointers accessible to QEMU.
+     * If we're in a chroot with no /proc, fall back to 1 page.
+     */
+    if (mmap_min_addr == 0) {
+        mmap_min_addr = qemu_host_page_size;
+        qemu_log_mask(CPU_LOG_PAGE,
+                      "host mmap_min_addr=0x%lx (fallback)\n",
+                      mmap_min_addr);
+    }
+
+    /*
+     * Prepare copy of argv vector for target.
+     */
+    target_argc = argc - optind;
+    target_argv = calloc(target_argc + 1, sizeof (char *));
+    if (target_argv == NULL) {
+        (void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
+        exit(EXIT_FAILURE);
+    }
+
+    /*
+     * If argv0 is specified (using '-0' switch) we replace
+     * argv[0] pointer with the given one.
+     */
+    i = 0;
+    if (argv0 != NULL) {
+        target_argv[i++] = strdup(argv0);
+    }
+    for (; i < target_argc; i++) {
+        target_argv[i] = strdup(argv[optind + i]);
+    }
+    target_argv[target_argc] = NULL;
+
+    /*
+     * Change root if requested wuth '-c'
+     */
+    if (qemu_chroot) {
+        if (chroot(qemu_chroot) < 0) {
+            error_report("chroot failed");
+            exit(1);
+        }
+        if (chdir("/")) {
+            error_report("not able to chdir to /: %s", strerror(errno));
+            exit(1);
+        }
+    }
+
     execfd = qemu_getauxval(AT_EXECFD);
     if (execfd == 0) {
         execfd = open(exec_path, O_RDONLY);
@@ -746,59 +821,6 @@ int main(int argc, char **argv, char **envp)
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-    /*
-     * Read in mmap_min_addr kernel parameter.  This value is used
-     * When loading the ELF image to determine whether guest_base
-     * is needed.  It is also used in mmap_find_vma.
-     */
-    {
-        FILE *fp;
-
-        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {
-            unsigned long tmp;
-            if (fscanf(fp, "%lu", &tmp) == 1 && tmp != 0) {
-                mmap_min_addr = tmp;
-                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n",
-                              mmap_min_addr);
-            }
-            fclose(fp);
-        }
-    }
-
-    /*
-     * We prefer to not make NULL pointers accessible to QEMU.
-     * If we're in a chroot with no /proc, fall back to 1 page.
-     */
-    if (mmap_min_addr == 0) {
-        mmap_min_addr = qemu_host_page_size;
-        qemu_log_mask(CPU_LOG_PAGE,
-                      "host mmap_min_addr=0x%lx (fallback)\n",
-                      mmap_min_addr);
-    }
-
-    /*
-     * Prepare copy of argv vector for target.
-     */
-    target_argc = argc - optind;
-    target_argv = calloc(target_argc + 1, sizeof (char *));
-    if (target_argv == NULL) {
-        (void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
-        exit(EXIT_FAILURE);
-    }
-
-    /*
-     * If argv0 is specified (using '-0' switch) we replace
-     * argv[0] pointer with the given one.
-     */
-    i = 0;
-    if (argv0 != NULL) {
-        target_argv[i++] = strdup(argv0);
-    }
-    for (; i < target_argc; i++) {
-        target_argv[i] = strdup(argv[optind + i]);
-    }
-    target_argv[target_argc] = NULL;
-
     ts = g_new0(TaskState, 1);
     init_task_state(ts);
     /* build Task State */
-- 
2.28.0

Re: [PATCH] linux-user: add option to chroot before emulation

[no-reply]

Patchew URL: https://patchew.org/QEMU/20201208001727.17433-1-mcroce@linux.microsoft.com/



Hi,

This series seems to have some coding style problems. See output below for
more information:

Type: series
Message-id: 20201208001727.17433-1-mcroce@linux.microsoft.com
Subject: [PATCH] linux-user: add option to chroot before emulation

=== TEST SCRIPT BEGIN ===
#!/bin/bash
git rev-parse base > /dev/null || exit 0
git config --local diff.renamelimit 0
git config --local diff.renames True
git config --local diff.algorithm histogram
./scripts/checkpatch.pl --mailback base..
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
From https://github.com/patchew-project/qemu
 * [new tag]         patchew/20201208001727.17433-1-mcroce@linux.microsoft.com -> patchew/20201208001727.17433-1-mcroce@linux.microsoft.com
Switched to a new branch 'test'
939b01d linux-user: add option to chroot before emulation

=== OUTPUT BEGIN ===
ERROR: do not use assignment in if condition
#62: FILE: linux-user/main.c:707:
+        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {

ERROR: space prohibited between function name and open parenthesis '('
#88: FILE: linux-user/main.c:733:
+    target_argv = calloc(target_argc + 1, sizeof (char *));

total: 2 errors, 0 warnings, 158 lines checked

Commit 939b01d16330 (linux-user: add option to chroot before emulation) has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.
=== OUTPUT END ===

Test command exited with code: 1


The full log is available at
http://patchew.org/logs/20201208001727.17433-1-mcroce@linux.microsoft.com/testing.checkpatch/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-devel@redhat.com

Re: [PATCH] linux-user: add option to chroot before emulation

[Laurent Vivier]

Le 08/12/2020 à 01:17, Matteo Croce a écrit :
> From: Matteo Croce <mcroce@microsoft.com>
> 
> Add a '-c' option which does a chroot() just before starting the
> emulation. This is useful when the static QEMU user binary can't
> be copied into the target root filesystem, e.g. if it's readonly.

Did you try to use the binfmt_misc 'F' flag (fix binary)?

https://www.kernel.org/doc/Documentation/admin-guide/binfmt-misc.rst

``F`` - fix binary

The usual behaviour of binfmt_misc is to spawn the
binary lazily when the misc format file is invoked.  However,
this doesn``t work very well in the face of mount namespaces and
changeroots, so the ``F`` mode opens the binary as soon as the
emulation is installed and uses the opened image to spawn the
emulator, meaning it is always available once installed,
regardless of how the environment changes.

This can be configured with scripts/qemu-binfmt-conf.sh and
"--persistent yes"" option

Thanks,
Laurent
> 
> Move some code which accesses /proc/sys/vm/mmap_min_addr before
> the chroot, otherwise it would fail.
> 
> Signed-off-by: Matteo Croce <mcroce@microsoft.com>
> ---
>  linux-user/main.c | 128 +++++++++++++++++++++++++++-------------------
>  1 file changed, 75 insertions(+), 53 deletions(-)
> 
> diff --git a/linux-user/main.c b/linux-user/main.c
> index 24d1eb73ad..4788e4b5bc 100644
> --- a/linux-user/main.c
> +++ b/linux-user/main.c
> @@ -60,6 +60,7 @@ static const char *seed_optarg;
>  unsigned long mmap_min_addr;
>  unsigned long guest_base;
>  bool have_guest_base;
> +static const char *qemu_chroot;
>  
>  /*
>   * Used to implement backwards-compatibility for the `-strace`, and
> @@ -304,6 +305,11 @@ static void handle_arg_pagesize(const char *arg)
>      }
>  }
>  
> +static void handle_arg_chroot(const char *arg)
> +{
> +    qemu_chroot = arg;
> +}
> +
>  static void handle_arg_seed(const char *arg)
>  {
>      seed_optarg = arg;
> @@ -450,6 +456,8 @@ static const struct qemu_argument arg_table[] = {
>       "logfile",     "write logs to 'logfile' (default stderr)"},
>      {"p",          "QEMU_PAGESIZE",    true,  handle_arg_pagesize,
>       "pagesize",   "set the host page size to 'pagesize'"},
> +    {"c",          "QEMU_CHROOT",      true,  handle_arg_chroot,
> +     "chroot",     "chroot to 'chroot' before starting emulation"},
>      {"singlestep", "QEMU_SINGLESTEP",  false, handle_arg_singlestep,
>       "",           "run in singlestep mode"},
>      {"strace",     "QEMU_STRACE",      false, handle_arg_strace,
> @@ -688,6 +696,73 @@ int main(int argc, char **argv, char **envp)
>  
>      init_qemu_uname_release();
>  
> +    /*
> +     * Read in mmap_min_addr kernel parameter.  This value is used
> +     * When loading the ELF image to determine whether guest_base
> +     * is needed.  It is also used in mmap_find_vma.
> +     */
> +    {
> +        FILE *fp;
> +
> +        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {
> +            unsigned long tmp;
> +            if (fscanf(fp, "%lu", &tmp) == 1 && tmp != 0) {
> +                mmap_min_addr = tmp;
> +                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n",
> +                              mmap_min_addr);
> +            }
> +            fclose(fp);
> +        }
> +    }
> +
> +    /*
> +     * We prefer to not make NULL pointers accessible to QEMU.
> +     * If we're in a chroot with no /proc, fall back to 1 page.
> +     */
> +    if (mmap_min_addr == 0) {
> +        mmap_min_addr = qemu_host_page_size;
> +        qemu_log_mask(CPU_LOG_PAGE,
> +                      "host mmap_min_addr=0x%lx (fallback)\n",
> +                      mmap_min_addr);
> +    }
> +
> +    /*
> +     * Prepare copy of argv vector for target.
> +     */
> +    target_argc = argc - optind;
> +    target_argv = calloc(target_argc + 1, sizeof (char *));
> +    if (target_argv == NULL) {
> +        (void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
> +        exit(EXIT_FAILURE);
> +    }
> +
> +    /*
> +     * If argv0 is specified (using '-0' switch) we replace
> +     * argv[0] pointer with the given one.
> +     */
> +    i = 0;
> +    if (argv0 != NULL) {
> +        target_argv[i++] = strdup(argv0);
> +    }
> +    for (; i < target_argc; i++) {
> +        target_argv[i] = strdup(argv[optind + i]);
> +    }
> +    target_argv[target_argc] = NULL;
> +
> +    /*
> +     * Change root if requested wuth '-c'
> +     */
> +    if (qemu_chroot) {
> +        if (chroot(qemu_chroot) < 0) {
> +            error_report("chroot failed");
> +            exit(1);
> +        }
> +        if (chdir("/")) {
> +            error_report("not able to chdir to /: %s", strerror(errno));
> +            exit(1);
> +        }
> +    }
> +
>      execfd = qemu_getauxval(AT_EXECFD);
>      if (execfd == 0) {
>          execfd = open(exec_path, O_RDONLY);
> @@ -746,59 +821,6 @@ int main(int argc, char **argv, char **envp)
>      target_environ = envlist_to_environ(envlist, NULL);
>      envlist_free(envlist);
>  
> -    /*
> -     * Read in mmap_min_addr kernel parameter.  This value is used
> -     * When loading the ELF image to determine whether guest_base
> -     * is needed.  It is also used in mmap_find_vma.
> -     */
> -    {
> -        FILE *fp;
> -
> -        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {
> -            unsigned long tmp;
> -            if (fscanf(fp, "%lu", &tmp) == 1 && tmp != 0) {
> -                mmap_min_addr = tmp;
> -                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n",
> -                              mmap_min_addr);
> -            }
> -            fclose(fp);
> -        }
> -    }
> -
> -    /*
> -     * We prefer to not make NULL pointers accessible to QEMU.
> -     * If we're in a chroot with no /proc, fall back to 1 page.
> -     */
> -    if (mmap_min_addr == 0) {
> -        mmap_min_addr = qemu_host_page_size;
> -        qemu_log_mask(CPU_LOG_PAGE,
> -                      "host mmap_min_addr=0x%lx (fallback)\n",
> -                      mmap_min_addr);
> -    }
> -
> -    /*
> -     * Prepare copy of argv vector for target.
> -     */
> -    target_argc = argc - optind;
> -    target_argv = calloc(target_argc + 1, sizeof (char *));
> -    if (target_argv == NULL) {
> -        (void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
> -        exit(EXIT_FAILURE);
> -    }
> -
> -    /*
> -     * If argv0 is specified (using '-0' switch) we replace
> -     * argv[0] pointer with the given one.
> -     */
> -    i = 0;
> -    if (argv0 != NULL) {
> -        target_argv[i++] = strdup(argv0);
> -    }
> -    for (; i < target_argc; i++) {
> -        target_argv[i] = strdup(argv[optind + i]);
> -    }
> -    target_argv[target_argc] = NULL;
> -
>      ts = g_new0(TaskState, 1);
>      init_task_state(ts);
>      /* build Task State */
> 

Re: [PATCH] linux-user: add option to chroot before emulation

[Matteo Croce]

On Tue, Dec 8, 2020 at 9:21 AM Laurent Vivier <laurent@vivier.eu> wrote:
>
> Le 08/12/2020 à 01:17, Matteo Croce a écrit :
> > From: Matteo Croce <mcroce@microsoft.com>
> >
> > Add a '-c' option which does a chroot() just before starting the
> > emulation. This is useful when the static QEMU user binary can't
> > be copied into the target root filesystem, e.g. if it's readonly.
>
> Did you try to use the binfmt_misc 'F' flag (fix binary)?
>
> https://www.kernel.org/doc/Documentation/admin-guide/binfmt-misc.rst
>
> ``F`` - fix binary
>
> The usual behaviour of binfmt_misc is to spawn the
> binary lazily when the misc format file is invoked.  However,
> this doesn``t work very well in the face of mount namespaces and
> changeroots, so the ``F`` mode opens the binary as soon as the
> emulation is installed and uses the opened image to spawn the
> emulator, meaning it is always available once installed,
> regardless of how the environment changes.
>
> This can be configured with scripts/qemu-binfmt-conf.sh and
> "--persistent yes"" option
>

Yes, this works too.
Basically it's the same trick, open the binary early and then emulate.
The only difference is that with binfmt emulation the procfs open
still fails:

# strace -feopenat chroot debian /bin/true 2>&1 |grep /proc/sys
[pid  9359] openat(AT_FDCWD, "/proc/sys/vm/mmap_min_addr", O_RDONLY) =
-1 ENOENT (No such file or directory)

vs

# strace -feopenat qemu-aarch64 -c debian /bin/true 2>&1 |grep /proc/sys
[pid  9348] openat(AT_FDCWD, "/proc/sys/vm/mmap_min_addr", O_RDONLY) = 3

-- 
per aspera ad upstream

Re: [PATCH] linux-user: add option to chroot before emulation

[Laurent Vivier]

Le 08/12/2020 à 17:04, Matteo Croce a écrit :
> On Tue, Dec 8, 2020 at 9:21 AM Laurent Vivier <laurent@vivier.eu> wrote:
>>
>> Le 08/12/2020 à 01:17, Matteo Croce a écrit :
>>> From: Matteo Croce <mcroce@microsoft.com>
>>>
>>> Add a '-c' option which does a chroot() just before starting the
>>> emulation. This is useful when the static QEMU user binary can't
>>> be copied into the target root filesystem, e.g. if it's readonly.
>>
>> Did you try to use the binfmt_misc 'F' flag (fix binary)?
>>
>> https://www.kernel.org/doc/Documentation/admin-guide/binfmt-misc.rst
>>
>> ``F`` - fix binary
>>
>> The usual behaviour of binfmt_misc is to spawn the
>> binary lazily when the misc format file is invoked.  However,
>> this doesn``t work very well in the face of mount namespaces and
>> changeroots, so the ``F`` mode opens the binary as soon as the
>> emulation is installed and uses the opened image to spawn the
>> emulator, meaning it is always available once installed,
>> regardless of how the environment changes.
>>
>> This can be configured with scripts/qemu-binfmt-conf.sh and
>> "--persistent yes"" option
>>
> 
> Yes, this works too.
> Basically it's the same trick, open the binary early and then emulate.
> The only difference is that with binfmt emulation the procfs open
> still fails:
> 
> # strace -feopenat chroot debian /bin/true 2>&1 |grep /proc/sys
> [pid  9359] openat(AT_FDCWD, "/proc/sys/vm/mmap_min_addr", O_RDONLY) =
> -1 ENOENT (No such file or directory)
> 
> vs
> 
> # strace -feopenat qemu-aarch64 -c debian /bin/true 2>&1 |grep /proc/sys
> [pid  9348] openat(AT_FDCWD, "/proc/sys/vm/mmap_min_addr", O_RDONLY) = 3
> 

If you want to have the /proc mounted earlier you can use "unshare".

something like:

unshare --mount-proc -R debian /bin/true

There is also the "-L" option, something like:

qemu-aarch64 -L debian debian/bin/true

Thanks,
Laurent