[PATCH V2 0/8] runqemu fixes

[PATCH V2 0/8] runqemu fixes

[Robert Yang]

The following changes since commit d77fe838ab7631a19e90ff4226f0712e54aa4e22:

  scripts: add tool to scan for bashisms recipe shell scripts (2016-09-16 15:19:31 +0100)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib rbt/rq
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=rbt/rq

Joshua Lock (6):
  runqemu: add guidance to resolve issues with missing files
  qemuboot: write the full kernel filename, not the link name
  runqemu: clarify an INFO message
  qemuboot: also write the kernel link name to the conf file
  runqemu: try symlinks when kernel or rootfs can't be found
  runqemu: work even if a *.qemuboot.conf isn't found

Robert Yang (2):
  runqemu: use OECORE_NATIVE_SYSROOT from sdk
  runqemu: improve finding of rootfs, kernel and dtb

 meta/classes/qemuboot.bbclass |  13 ++++-
 scripts/runqemu               | 108 ++++++++++++++++++++++++++++++------------
 2 files changed, 90 insertions(+), 31 deletions(-)

-- 
2.9.0

[PATCH V2 1/8] runqemu: add guidance to resolve issues with missing files

[Robert Yang]

From: Joshua Lock <joshua.g.lock@intel.com>

When a required binary cannot be found print some guidance pointing
to using a sourced OE build environment or a qemuboot.conf file,
based on a similar message from the previous shell-based runqemu.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 scripts/runqemu | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/scripts/runqemu b/scripts/runqemu
index b6bc0ba..c71a47c 100755
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -28,6 +28,16 @@ import shutil
 import glob
 import configparser
 
+class OEPathError(Exception):
+    """Custom Exception to give better guidance on missing binaries"""
+    def __init__(self, message):
+        self.message = "In order for this script to dynamically infer paths\n \
+kernels or filesystem images, you either need bitbake in your PATH\n \
+or to source oe-init-build-env before running this script.\n\n \
+Dynamic path inference can be avoided by passing a *.qemuboot.conf to\n \
+runqemu, i.e. `runqemu /path/to/my-image-name.qemuboot.conf`\n\n %s" % message
+
+
 def create_logger():
     logger = logging.getLogger('runqemu')
     logger.setLevel(logging.INFO)
@@ -537,7 +547,7 @@ class BaseConfig(object):
             elif os.getenv('DEPLOY_DIR_IMAGE'):
                 deploy_dir_image = os.getenv('DEPLOY_DIR_IMAGE')
             else:
-                raise Exception("DEPLOY_DIR_IMAGE is NULL!")
+                raise OEPathError("DEPLOY_DIR_IMAGE is NULL!")
 
             if self.rootfs and not os.path.exists(self.rootfs):
                 # Lazy rootfs
@@ -691,7 +701,7 @@ class BaseConfig(object):
         lockdir = "/tmp/qemu-tap-locks"
 
         if not (self.qemuifup and self.qemuifdown and ip):
-            raise Exception("runqemu-ifup, runqemu-ifdown or ip not found")
+            raise OEPathError("runqemu-ifup, runqemu-ifdown or ip not found")
 
         if not os.path.exists(lockdir):
             # There might be a race issue when multi runqemu processess are
@@ -808,7 +818,7 @@ class BaseConfig(object):
 
         qemu_bin = '%s/%s' % (self.get('STAGING_BINDIR_NATIVE'), qemu_system)
         if not os.access(qemu_bin, os.X_OK):
-            raise Exception("No QEMU binary '%s' could be found" % qemu_bin)
+            raise OEPathError("No QEMU binary '%s' could be found" % qemu_bin)
 
         check_libgl(qemu_bin)
 
@@ -923,6 +933,9 @@ def main():
 if __name__ == "__main__":
     try:
         ret = main()
+    except OEPathError as err:
+        ret = 1
+        logger.error(err.message)
     except Exception as esc:
         ret = 1
         import traceback
-- 
2.9.0

[PATCH V2 2/8] qemuboot: write the full kernel filename, not the link name

[Robert Yang]

From: Joshua Lock <joshua.g.lock@intel.com>

KERNEL_IMAGETYPE gives the filename of a symlink to the kernel,
which may not be available i.e. if the user downloads some build
artefacts to run on a local machine. It's also possible that the
link will point to a newer kernel than was intended for use with
the rootfs in the qemuboot.conf.

It's much more reliable to read the name of the file
KERNEL_IMAGETYPE is linking to and assign the full filename to
QB_DEFAULT_KERNEL.

[YOCTO #10285]

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 meta/classes/qemuboot.bbclass | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/meta/classes/qemuboot.bbclass b/meta/classes/qemuboot.bbclass
index 8500c73..0892db3 100644
--- a/meta/classes/qemuboot.bbclass
+++ b/meta/classes/qemuboot.bbclass
@@ -63,6 +63,14 @@ python write_qemuboot_conf() {
     cf.add_section('config_bsp')
     for k in build_vars + qb_vars:
         cf.set('config_bsp', k, '%s' % d.getVar(k, True))
+
+    # QB_DEFAULT_KERNEL's value of KERNEL_IMAGETYPE is the name of a symlink
+    # to the kernel file, which hinders relocatability of the qb conf.
+    # Read the link and replace it with the full filename of the target.
+    kernel_link = os.path.join(d.getVar('DEPLOY_DIR_IMAGE', True), d.getVar('QB_DEFAULT_KERNEL', True))
+    kernel = os.readlink(kernel_link)
+    cf.set('config_bsp', 'QB_DEFAULT_KERNEL', kernel)
+
     with open(qemuboot, 'w') as f:
         cf.write(f)
 
-- 
2.9.0

[PATCH V2 3/8] runqemu: clarify an INFO message

[Robert Yang]

From: Joshua Lock <joshua.g.lock@intel.com>

Make it clearer that we are looking for a file which ends with
qemuboot.conf

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 scripts/runqemu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/runqemu b/scripts/runqemu
index c71a47c..6aaae44 100755
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -205,7 +205,7 @@ class BaseConfig(object):
     def is_deploy_dir_image(self, p):
         if os.path.isdir(p):
             if not re.search('.qemuboot.conf$', '\n'.join(os.listdir(p)), re.M):
-                logger.info("Can't find required qemuboot.conf in %s" % p)
+                logger.info("Can't find required *.qemuboot.conf in %s" % p)
                 return False
             if not re.search('-image-', '\n'.join(os.listdir(p))):
                 logger.info("Can't find *-image-* in %s" % p)
-- 
2.9.0

[PATCH V2 4/8] qemuboot: also write the kernel link name to the conf file

[Robert Yang]

From: Joshua Lock <joshua.g.lock@intel.com>

This will allow runqemu to fall back to trying the link name when
a file matching the full name can't be found.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 meta/classes/qemuboot.bbclass | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/meta/classes/qemuboot.bbclass b/meta/classes/qemuboot.bbclass
index 0892db3..802eb59 100644
--- a/meta/classes/qemuboot.bbclass
+++ b/meta/classes/qemuboot.bbclass
@@ -48,8 +48,9 @@ python write_qemuboot_conf() {
     import configparser
 
     build_vars = ['MACHINE', 'TUNE_ARCH', 'DEPLOY_DIR_IMAGE', \
-                'IMAGE_NAME', 'IMAGE_LINK_NAME', 'STAGING_DIR_NATIVE', \
-                'STAGING_BINDIR_NATIVE', 'STAGING_DIR_HOST']
+                'KERNEL_IMAGETYPE', 'IMAGE_NAME', 'IMAGE_LINK_NAME', \
+                'STAGING_DIR_NATIVE', 'STAGING_BINDIR_NATIVE', \
+                'STAGING_DIR_HOST']
 
     # Vars from bsp
     qb_vars = []
-- 
2.9.0

[PATCH V2 5/8] runqemu: try symlinks when kernel or rootfs can't be found

[Robert Yang]

From: Joshua Lock <joshua.g.lock@intel.com>

If the kernel or rootfs names written to the qemuboot.conf can't
be found, try and find the symlinked variant of the filename.

This will help usability of runqemu, for example where a user
downloads an image and associated files as the symlinked names
yet the qemuboot.conf variables point to the full, non-linked,
file names.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 scripts/runqemu | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/scripts/runqemu b/scripts/runqemu
index 6aaae44..38f9b30 100755
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -451,7 +451,12 @@ class BaseConfig(object):
             if all_files:
                 self.rootfs = all_files[0]
             else:
-                raise Exception("Failed to find rootfs: %s" % cmd)
+                cmd = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'), self.get('IMAGE_LINK_NAME'), self.fstype)
+                all_files = glob.glob(cmd)
+                if all_files:
+                    self.rootfs = all_files[0]
+                else:
+                    raise Exception("Failed to find rootfs: %s" % cmd)
 
         if not os.path.exists(self.rootfs):
             raise Exception("Can't find rootfs: %s" % self.rootfs)
@@ -462,13 +467,18 @@ class BaseConfig(object):
         if self.fstype in self.vmtypes:
             return
         kernel = self.kernel
+        deploy_dir_image = self.get('DEPLOY_DIR_IMAGE')
         if not kernel:
-            kernel = "%s/%s" % (self.get('DEPLOY_DIR_IMAGE'), self.get('QB_DEFAULT_KERNEL'))
+            kernel = "%s/%s" % (deploy_dir_image, self.get('QB_DEFAULT_KERNEL'))
 
         if os.path.exists(kernel):
             self.kernel = kernel
         else:
-            raise Exception("KERNEL %s not found" % kernel)
+            kernel = "%s/%s" % (deploy_dir_image, self.get('KERNEL_IMAGETYPE'))
+            if kernel != deploy_dir_image and os.path.exists(kernel):
+                self.kernel = kernel
+            else:
+                raise Exception("KERNEL %s not found" % kernel)
 
         dtb = self.get('QB_DTB')
         if dtb:
-- 
2.9.0

[PATCH V2 6/8] runqemu: work even if a *.qemuboot.conf isn't found

[Robert Yang]

From: Joshua Lock <joshua.g.lock@intel.com>

A qemuboot conf file is a convenience but it should still be
possible to invoke runqemu without them, especially for examples
such as using the SDK with an extracted rootfs via NFS.

As read_qemuboot() is always called we need to be sure that function
can return cleanly, without throwing Exceptions, even if a qemuboot
conf file isn't found.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 scripts/runqemu | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scripts/runqemu b/scripts/runqemu
index 38f9b30..10122af 100755
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -557,7 +557,8 @@ class BaseConfig(object):
             elif os.getenv('DEPLOY_DIR_IMAGE'):
                 deploy_dir_image = os.getenv('DEPLOY_DIR_IMAGE')
             else:
-                raise OEPathError("DEPLOY_DIR_IMAGE is NULL!")
+                logger.info("Can't find qemuboot conf file, DEPLOY_DIR_IMAGE is NULL!")
+                return
 
             if self.rootfs and not os.path.exists(self.rootfs):
                 # Lazy rootfs
@@ -574,6 +575,11 @@ class BaseConfig(object):
                     self.qemuboot = qbs.split()[0]
                     self.qbconfload = True
 
+        if not self.qemuboot:
+            # If we haven't found a .qemuboot.conf at this point it probably
+            # doesn't exist, continue without
+            return
+
         if not os.path.exists(self.qemuboot):
             raise Exception("Failed to find <image>.qemuboot.conf!")
 
-- 
2.9.0

[PATCH V2 7/8] runqemu: use OECORE_NATIVE_SYSROOT from sdk

[Robert Yang]

There is no STAGING_DIR_NATIVE or bitbake in a extracted sdk,
so check OECORE_NATIVE_SYSROOT and use it.

Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
 scripts/runqemu | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/scripts/runqemu b/scripts/runqemu
index 10122af..60e2093 100755
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -619,18 +619,25 @@ class BaseConfig(object):
                         self.set(nv, s.group(1))
             else:
                 # when we're invoked from a running bitbake instance we won't
-                # be able to call `bitbake -e` but should have OE_TMPDIR set in
-                # the environment and can guess paths based on it
+                # be able to call `bitbake -e`, then try:
+                # - get OE_TMPDIR from environment and guess paths based on it
+                # - get OECORE_NATIVE_SYSROOT from environment (for sdk)
                 tmpdir = os.environ.get('OE_TMPDIR', None)
+                oecore_native_sysroot = os.environ.get('OECORE_NATIVE_SYSROOT', None)
                 if tmpdir:
                     logger.info('Setting STAGING_DIR_NATIVE and STAGING_BINDIR_NATIVE relative to OE_TMPDIR (%s)' % tmpdir)
                     hostos, _, _, _, machine = os.uname()
                     buildsys = '%s-%s' % (machine, hostos.lower())
                     staging_dir_native = '%s/sysroots/%s' % (tmpdir, buildsys)
                     self.set('STAGING_DIR_NATIVE', staging_dir_native)
+                elif oecore_native_sysroot:
+                    logger.info('Setting STAGING_DIR_NATIVE to OECORE_NATIVE_SYSROOT (%s)' % oecore_native_sysroot)
+                    self.set('STAGING_DIR_NATIVE', oecore_native_sysroot)
+                if self.get('STAGING_DIR_NATIVE'):
                     # we have to assume that STAGING_BINDIR_NATIVE is at usr/bin
-                    staging_bindir_native = '%s/usr/bin' % staging_dir_native
-                    self.set('STAGING_BINDIR_NATIVE', staging_bindir_native)
+                    staging_bindir_native = '%s/usr/bin' % self.get('STAGING_DIR_NATIVE')
+                    logger.info('Setting STAGING_BINDIR_NATIVE to %s' % staging_bindir_native)
+                    self.set('STAGING_BINDIR_NATIVE', '%s/usr/bin' % self.get('STAGING_DIR_NATIVE'))
 
     def print_config(self):
         logger.info('Continuing with the following parameters:\n')
-- 
2.9.0

[PATCH V2 8/8] runqemu: improve finding of rootfs, kernel and dtb

[Robert Yang]

* Search rootfs in the following order:
  - IMAGE_NAME*.FSTYPE
  - IMAGE_LINK_NAME*.FSTYPE

* Search kernel in the following order:
  - QB_DEFAULT_KERNEL
  - KERNEL_IMAGETYPE
  - KERNEL_IMAGETYPE*

* Search dtb in the following order:
   - QB_DTB
   - QB_DTB*
   - *.dtb

* Fix DTB, it should only work with "-kernel" option.

[YOCTO #10265]

Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
 scripts/runqemu | 68 ++++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 41 insertions(+), 27 deletions(-)

diff --git a/scripts/runqemu b/scripts/runqemu
index 60e2093..1c4e69b 100755
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -157,6 +157,7 @@ class BaseConfig(object):
         self.kernel = ''
         self.kernel_cmdline = ''
         self.kernel_cmdline_script = ''
+        self.dtb = ''
         self.fstype = ''
         self.kvm_enabled = False
         self.vhost_enabled = False
@@ -440,23 +441,23 @@ class BaseConfig(object):
         if self.fstype == 'nfs':
             return
 
+        cmd_name = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'), self.get('IMAGE_NAME'), self.fstype)
+        cmd_link = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'), self.get('IMAGE_LINK_NAME'), self.fstype)
+        cmds = (cmd_name, cmd_link)
         if self.rootfs and not os.path.exists(self.rootfs):
             # Lazy rootfs
             self.rootfs = "%s/%s-%s.%s" % (self.get('DEPLOY_DIR_IMAGE'),
                     self.rootfs, self.get('MACHINE'),
                     self.fstype)
         elif not self.rootfs:
-            cmd = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'), self.get('IMAGE_NAME'), self.fstype)
-            all_files = glob.glob(cmd)
-            if all_files:
-                self.rootfs = all_files[0]
-            else:
-                cmd = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'), self.get('IMAGE_LINK_NAME'), self.fstype)
+            for cmd in cmds:
                 all_files = glob.glob(cmd)
                 if all_files:
                     self.rootfs = all_files[0]
-                else:
-                    raise Exception("Failed to find rootfs: %s" % cmd)
+                    break
+
+        if not self.rootfs:
+            raise Exception("Failed to find rootfs: %s or %s" % cmds)
 
         if not os.path.exists(self.rootfs):
             raise Exception("Can't find rootfs: %s" % self.rootfs)
@@ -466,28 +467,37 @@ class BaseConfig(object):
         # The vm image doesn't need a kernel
         if self.fstype in self.vmtypes:
             return
-        kernel = self.kernel
+
         deploy_dir_image = self.get('DEPLOY_DIR_IMAGE')
-        if not kernel:
-            kernel = "%s/%s" % (deploy_dir_image, self.get('QB_DEFAULT_KERNEL'))
+        if not self.kernel:
+            kernel_match_name = "%s/%s" % (deploy_dir_image, self.get('QB_DEFAULT_KERNEL'))
+            kernel_match_link = "%s/%s" % (deploy_dir_image, self.get('KERNEL_IMAGETYPE'))
+            kernel_startswith = "%s/%s*" % (deploy_dir_image, self.get('KERNEL_IMAGETYPE'))
+            cmds = (kernel_match_name, kernel_match_link, kernel_startswith)
+            for cmd in cmds:
+                all_files = glob.glob(cmd)
+                if all_files:
+                    self.kernel = all_files[0]
+                    break
+            if not self.kernel:
+                raise Exception('KERNEL not found: %s, %s or %s' % cmds)
 
-        if os.path.exists(kernel):
-            self.kernel = kernel
-        else:
-            kernel = "%s/%s" % (deploy_dir_image, self.get('KERNEL_IMAGETYPE'))
-            if kernel != deploy_dir_image and os.path.exists(kernel):
-                self.kernel = kernel
-            else:
-                raise Exception("KERNEL %s not found" % kernel)
+        if not os.path.exists(self.kernel):
+            raise Exception("KERNEL %s not found" % self.kernel)
 
         dtb = self.get('QB_DTB')
         if dtb:
-            dtb = "%s/%s" % (self.get('DEPLOY_DIR_IMAGE'), dtb)
-            if os.path.exists(dtb):
-                self.set('QB_DTB', '-dtb %s' % dtb)
-            else:
-                raise Exception("DTB %s not found" % dtb)
-
+            cmd_match = "%s/%s" % (deploy_dir_image, dtb)
+            cmd_startswith = "%s/%s*" % (deploy_dir_image, dtb)
+            cmd_wild = "%s/*.dtb" % deploy_dir_image
+            cmds = (cmd_match, cmd_startswith, cmd_wild)
+            for cmd in cmds:
+                all_files = glob.glob(cmd)
+                if all_files:
+                    self.dtb = all_files[0]
+                    break
+            if not os.path.exists(self.dtb):
+                raise Exception('DTB not found: %s, %s or %s' % cmds)
 
     def check_biosdir(self):
         """Check custombiosdir"""
@@ -643,6 +653,8 @@ class BaseConfig(object):
         logger.info('Continuing with the following parameters:\n')
         if not self.fstype in self.vmtypes:
             print('KERNEL: [%s]' % self.kernel)
+            if self.dtb:
+                print('DTB: [%s]' % self.dtb)
         print('MACHINE: [%s]' % self.get('MACHINE'))
         print('FSTYPE: [%s]' % self.fstype)
         if self.fstype  == 'nfs':
@@ -687,7 +699,7 @@ class BaseConfig(object):
                 elif os.path.exists(src2):
                     src = src2
                 if not src:
-                    raise Exception("No NFS_DIR is set but can't find %s or %s to extract" % (src1, src2))
+                    raise Exception("No NFS_DIR is set, and can't find %s or %s to extract" % (src1, src2))
                 logger.info('NFS_DIR not found, extracting %s to %s' % (src, dest))
                 cmd = 'runqemu-extract-sdk %s %s' % (src, dest)
                 logger.info('Running %s...' % cmd)
@@ -845,7 +857,7 @@ class BaseConfig(object):
 
         check_libgl(qemu_bin)
 
-        self.qemu_opt = "%s %s %s %s %s %s" % (qemu_bin, self.get('NETWORK_CMD'), self.qemu_opt_script, self.get('ROOTFS_OPTIONS'), self.get('QB_DTB'), self.get('QB_OPT_APPEND'))
+        self.qemu_opt = "%s %s %s %s %s" % (qemu_bin, self.get('NETWORK_CMD'), self.qemu_opt_script, self.get('ROOTFS_OPTIONS'), self.get('QB_OPT_APPEND'))
 
         # Enable virtio RNG else we can run out of entropy in guests
         self.qemu_opt += " -device virtio-rng-pci"
@@ -877,6 +889,8 @@ class BaseConfig(object):
     def start_qemu(self):
         if self.kernel:
             kernel_opts = "-kernel %s -append '%s %s %s'" % (self.kernel, self.kernel_cmdline, self.kernel_cmdline_script, self.get('QB_KERNEL_CMDLINE_APPEND'))
+            if self.dtb:
+                kernel_opts += " -dtb %s" % self.dtb
         else:
             kernel_opts = ""
         cmd = "%s %s" % (self.qemu_opt, kernel_opts)
-- 
2.9.0

Re: [PATCH V2 8/8] runqemu: improve finding of rootfs, kernel and dtb

[Joshua Lock]

On Sun, 2016-09-18 at 00:39 -0700, Robert Yang wrote:
> * Search rootfs in the following order:
>   - IMAGE_NAME*.FSTYPE
>   - IMAGE_LINK_NAME*.FSTYPE
> 
> * Search kernel in the following order:
>   - QB_DEFAULT_KERNEL
>   - KERNEL_IMAGETYPE
>   - KERNEL_IMAGETYPE*
> 
> * Search dtb in the following order:
>    - QB_DTB
>    - QB_DTB*
>    - *.dtb
> 
> * Fix DTB, it should only work with "-kernel" option.
> 
> [YOCTO #10265]
> 
> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> ---
>  scripts/runqemu | 68 ++++++++++++++++++++++++++++++++++-------------
> ----------
>  1 file changed, 41 insertions(+), 27 deletions(-)
> 
> diff --git a/scripts/runqemu b/scripts/runqemu
> index 60e2093..1c4e69b 100755
> --- a/scripts/runqemu
> +++ b/scripts/runqemu
> @@ -157,6 +157,7 @@ class BaseConfig(object):
>          self.kernel = ''
>          self.kernel_cmdline = ''
>          self.kernel_cmdline_script = ''
> +        self.dtb = ''
>          self.fstype = ''
>          self.kvm_enabled = False
>          self.vhost_enabled = False
> @@ -440,23 +441,23 @@ class BaseConfig(object):
>          if self.fstype == 'nfs':
>              return
>  
> +        cmd_name = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'),
> self.get('IMAGE_NAME'), self.fstype)
> +        cmd_link = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'),
> self.get('IMAGE_LINK_NAME'), self.fstype)
> +        cmds = (cmd_name, cmd_link)
>          if self.rootfs and not os.path.exists(self.rootfs):
>              # Lazy rootfs
>              self.rootfs = "%s/%s-%s.%s" %
> (self.get('DEPLOY_DIR_IMAGE'),
>                      self.rootfs, self.get('MACHINE'),
>                      self.fstype)
>          elif not self.rootfs:
> -            cmd = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'),
> self.get('IMAGE_NAME'), self.fstype)
> -            all_files = glob.glob(cmd)
> -            if all_files:
> -                self.rootfs = all_files[0]
> -            else:
> -                cmd = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'),
> self.get('IMAGE_LINK_NAME'), self.fstype)
> +            for cmd in cmds:
>                  all_files = glob.glob(cmd)
>                  if all_files:
>                      self.rootfs = all_files[0]
> -                else:
> -                    raise Exception("Failed to find rootfs: %s" %
> cmd)
> +                    break
> +
> +        if not self.rootfs:
> +            raise Exception("Failed to find rootfs: %s or %s" %
> cmds)
>  
>          if not os.path.exists(self.rootfs):
>              raise Exception("Can't find rootfs: %s" % self.rootfs)
> @@ -466,28 +467,37 @@ class BaseConfig(object):
>          # The vm image doesn't need a kernel
>          if self.fstype in self.vmtypes:
>              return
> -        kernel = self.kernel
> +
>          deploy_dir_image = self.get('DEPLOY_DIR_IMAGE')
> -        if not kernel:
> -            kernel = "%s/%s" % (deploy_dir_image,
> self.get('QB_DEFAULT_KERNEL'))
> +        if not self.kernel:
> +            kernel_match_name = "%s/%s" % (deploy_dir_image,
> self.get('QB_DEFAULT_KERNEL'))
> +            kernel_match_link = "%s/%s" % (deploy_dir_image,
> self.get('KERNEL_IMAGETYPE'))
> +            kernel_startswith = "%s/%s*" % (deploy_dir_image,
> self.get('KERNEL_IMAGETYPE'))

There are qemuboot.conf files in the wild which won't contain
KERNEL_IMAGETYPE, at which point we're just looking for matches to
DEPLOY_DIR_IMAGE or DEPLOY_DIR_IMAGE/*

I think we need to add some extra handling so that we don't end up
setting kernel to either DEPLOY_DIR_IMAGE or its first child?

Joshua

> +            cmds = (kernel_match_name, kernel_match_link,
> kernel_startswith)
> +            for cmd in cmds:
> +                all_files = glob.glob(cmd)
> +                if all_files:
> +                    self.kernel = all_files[0]
> +                    break
> +            if not self.kernel:
> +                raise Exception('KERNEL not found: %s, %s or %s' %
> cmds)
>  
> -        if os.path.exists(kernel):
> -            self.kernel = kernel
> -        else:
> -            kernel = "%s/%s" % (deploy_dir_image,
> self.get('KERNEL_IMAGETYPE'))
> -            if kernel != deploy_dir_image and
> os.path.exists(kernel):
> -                self.kernel = kernel
> -            else:
> -                raise Exception("KERNEL %s not found" % kernel)
> +        if not os.path.exists(self.kernel):
> +            raise Exception("KERNEL %s not found" % self.kernel)
>  
>          dtb = self.get('QB_DTB')
>          if dtb:
> -            dtb = "%s/%s" % (self.get('DEPLOY_DIR_IMAGE'), dtb)
> -            if os.path.exists(dtb):
> -                self.set('QB_DTB', '-dtb %s' % dtb)
> -            else:
> -                raise Exception("DTB %s not found" % dtb)
> -
> +            cmd_match = "%s/%s" % (deploy_dir_image, dtb)
> +            cmd_startswith = "%s/%s*" % (deploy_dir_image, dtb)
> +            cmd_wild = "%s/*.dtb" % deploy_dir_image
> +            cmds = (cmd_match, cmd_startswith, cmd_wild)
> +            for cmd in cmds:
> +                all_files = glob.glob(cmd)
> +                if all_files:
> +                    self.dtb = all_files[0]
> +                    break
> +            if not os.path.exists(self.dtb):
> +                raise Exception('DTB not found: %s, %s or %s' %
> cmds)
>  
>      def check_biosdir(self):
>          """Check custombiosdir"""
> @@ -643,6 +653,8 @@ class BaseConfig(object):
>          logger.info('Continuing with the following parameters:\n')
>          if not self.fstype in self.vmtypes:
>              print('KERNEL: [%s]' % self.kernel)
> +            if self.dtb:
> +                print('DTB: [%s]' % self.dtb)
>          print('MACHINE: [%s]' % self.get('MACHINE'))
>          print('FSTYPE: [%s]' % self.fstype)
>          if self.fstype  == 'nfs':
> @@ -687,7 +699,7 @@ class BaseConfig(object):
>                  elif os.path.exists(src2):
>                      src = src2
>                  if not src:
> -                    raise Exception("No NFS_DIR is set but can't
> find %s or %s to extract" % (src1, src2))
> +                    raise Exception("No NFS_DIR is set, and can't
> find %s or %s to extract" % (src1, src2))
>                  logger.info('NFS_DIR not found, extracting %s to %s'
> % (src, dest))
>                  cmd = 'runqemu-extract-sdk %s %s' % (src, dest)
>                  logger.info('Running %s...' % cmd)
> @@ -845,7 +857,7 @@ class BaseConfig(object):
>  
>          check_libgl(qemu_bin)
>  
> -        self.qemu_opt = "%s %s %s %s %s %s" % (qemu_bin,
> self.get('NETWORK_CMD'), self.qemu_opt_script,
> self.get('ROOTFS_OPTIONS'), self.get('QB_DTB'),
> self.get('QB_OPT_APPEND'))
> +        self.qemu_opt = "%s %s %s %s %s" % (qemu_bin,
> self.get('NETWORK_CMD'), self.qemu_opt_script,
> self.get('ROOTFS_OPTIONS'), self.get('QB_OPT_APPEND'))
>  
>          # Enable virtio RNG else we can run out of entropy in guests
>          self.qemu_opt += " -device virtio-rng-pci"
> @@ -877,6 +889,8 @@ class BaseConfig(object):
>      def start_qemu(self):
>          if self.kernel:
>              kernel_opts = "-kernel %s -append '%s %s %s'" %
> (self.kernel, self.kernel_cmdline, self.kernel_cmdline_script,
> self.get('QB_KERNEL_CMDLINE_APPEND'))
> +            if self.dtb:
> +                kernel_opts += " -dtb %s" % self.dtb
>          else:
>              kernel_opts = ""
>          cmd = "%s %s" % (self.qemu_opt, kernel_opts)
> -- 
> 2.9.0
> 

Re: [PATCH V2 8/8] runqemu: improve finding of rootfs, kernel and dtb

[Robert Yang]

On 09/19/2016 05:12 PM, Joshua Lock wrote:
> On Sun, 2016-09-18 at 00:39 -0700, Robert Yang wrote:
>> * Search rootfs in the following order:
>>   - IMAGE_NAME*.FSTYPE
>>   - IMAGE_LINK_NAME*.FSTYPE
>>
>> * Search kernel in the following order:
>>   - QB_DEFAULT_KERNEL
>>   - KERNEL_IMAGETYPE
>>   - KERNEL_IMAGETYPE*
>>
>> * Search dtb in the following order:
>>    - QB_DTB
>>    - QB_DTB*
>>    - *.dtb
>>
>> * Fix DTB, it should only work with "-kernel" option.
>>
>> [YOCTO #10265]
>>
>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>> ---
>>  scripts/runqemu | 68 ++++++++++++++++++++++++++++++++++-------------
>> ----------
>>  1 file changed, 41 insertions(+), 27 deletions(-)
>>
>> diff --git a/scripts/runqemu b/scripts/runqemu
>> index 60e2093..1c4e69b 100755
>> --- a/scripts/runqemu
>> +++ b/scripts/runqemu
>> @@ -157,6 +157,7 @@ class BaseConfig(object):
>>          self.kernel = ''
>>          self.kernel_cmdline = ''
>>          self.kernel_cmdline_script = ''
>> +        self.dtb = ''
>>          self.fstype = ''
>>          self.kvm_enabled = False
>>          self.vhost_enabled = False
>> @@ -440,23 +441,23 @@ class BaseConfig(object):
>>          if self.fstype == 'nfs':
>>              return
>>
>> +        cmd_name = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'),
>> self.get('IMAGE_NAME'), self.fstype)
>> +        cmd_link = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'),
>> self.get('IMAGE_LINK_NAME'), self.fstype)
>> +        cmds = (cmd_name, cmd_link)
>>          if self.rootfs and not os.path.exists(self.rootfs):
>>              # Lazy rootfs
>>              self.rootfs = "%s/%s-%s.%s" %
>> (self.get('DEPLOY_DIR_IMAGE'),
>>                      self.rootfs, self.get('MACHINE'),
>>                      self.fstype)
>>          elif not self.rootfs:
>> -            cmd = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'),
>> self.get('IMAGE_NAME'), self.fstype)
>> -            all_files = glob.glob(cmd)
>> -            if all_files:
>> -                self.rootfs = all_files[0]
>> -            else:
>> -                cmd = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'),
>> self.get('IMAGE_LINK_NAME'), self.fstype)
>> +            for cmd in cmds:
>>                  all_files = glob.glob(cmd)
>>                  if all_files:
>>                      self.rootfs = all_files[0]
>> -                else:
>> -                    raise Exception("Failed to find rootfs: %s" %
>> cmd)
>> +                    break
>> +
>> +        if not self.rootfs:
>> +            raise Exception("Failed to find rootfs: %s or %s" %
>> cmds)
>>
>>          if not os.path.exists(self.rootfs):
>>              raise Exception("Can't find rootfs: %s" % self.rootfs)
>> @@ -466,28 +467,37 @@ class BaseConfig(object):
>>          # The vm image doesn't need a kernel
>>          if self.fstype in self.vmtypes:
>>              return
>> -        kernel = self.kernel
>> +
>>          deploy_dir_image = self.get('DEPLOY_DIR_IMAGE')
>> -        if not kernel:
>> -            kernel = "%s/%s" % (deploy_dir_image,
>> self.get('QB_DEFAULT_KERNEL'))
>> +        if not self.kernel:
>> +            kernel_match_name = "%s/%s" % (deploy_dir_image,
>> self.get('QB_DEFAULT_KERNEL'))
>> +            kernel_match_link = "%s/%s" % (deploy_dir_image,
>> self.get('KERNEL_IMAGETYPE'))
>> +            kernel_startswith = "%s/%s*" % (deploy_dir_image,
>> self.get('KERNEL_IMAGETYPE'))
>
> There are qemuboot.conf files in the wild which won't contain
> KERNEL_IMAGETYPE, at which point we're just looking for matches to
> DEPLOY_DIR_IMAGE or DEPLOY_DIR_IMAGE/*
>
> I think we need to add some extra handling so that we don't end up
> setting kernel to either DEPLOY_DIR_IMAGE or its first child?

Thanks, updated in the repo:

   git://git.openembedded.org/openembedded-core-contrib rbt/rq
   http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=rbt/rq

Joshua Lock (6):
   runqemu: add guidance to resolve issues with missing files
   qemuboot: write the full kernel filename, not the link name
   runqemu: clarify an INFO message
   qemuboot: also write the kernel link name to the conf file
   runqemu: try symlinks when kernel or rootfs can't be found
   runqemu: work even if a *.qemuboot.conf isn't found

Robert Yang (2):
   runqemu: use OECORE_NATIVE_SYSROOT from sdk
   runqemu: improve finding of rootfs, kernel and dtb

// Robert

>
> Joshua
>
>> +            cmds = (kernel_match_name, kernel_match_link,
>> kernel_startswith)
>> +            for cmd in cmds:
>> +                all_files = glob.glob(cmd)
>> +                if all_files:
>> +                    self.kernel = all_files[0]
>> +                    break
>> +            if not self.kernel:
>> +                raise Exception('KERNEL not found: %s, %s or %s' %
>> cmds)
>>
>> -        if os.path.exists(kernel):
>> -            self.kernel = kernel
>> -        else:
>> -            kernel = "%s/%s" % (deploy_dir_image,
>> self.get('KERNEL_IMAGETYPE'))
>> -            if kernel != deploy_dir_image and
>> os.path.exists(kernel):
>> -                self.kernel = kernel
>> -            else:
>> -                raise Exception("KERNEL %s not found" % kernel)
>> +        if not os.path.exists(self.kernel):
>> +            raise Exception("KERNEL %s not found" % self.kernel)
>>
>>          dtb = self.get('QB_DTB')
>>          if dtb:
>> -            dtb = "%s/%s" % (self.get('DEPLOY_DIR_IMAGE'), dtb)
>> -            if os.path.exists(dtb):
>> -                self.set('QB_DTB', '-dtb %s' % dtb)
>> -            else:
>> -                raise Exception("DTB %s not found" % dtb)
>> -
>> +            cmd_match = "%s/%s" % (deploy_dir_image, dtb)
>> +            cmd_startswith = "%s/%s*" % (deploy_dir_image, dtb)
>> +            cmd_wild = "%s/*.dtb" % deploy_dir_image
>> +            cmds = (cmd_match, cmd_startswith, cmd_wild)
>> +            for cmd in cmds:
>> +                all_files = glob.glob(cmd)
>> +                if all_files:
>> +                    self.dtb = all_files[0]
>> +                    break
>> +            if not os.path.exists(self.dtb):
>> +                raise Exception('DTB not found: %s, %s or %s' %
>> cmds)
>>
>>      def check_biosdir(self):
>>          """Check custombiosdir"""
>> @@ -643,6 +653,8 @@ class BaseConfig(object):
>>          logger.info('Continuing with the following parameters:\n')
>>          if not self.fstype in self.vmtypes:
>>              print('KERNEL: [%s]' % self.kernel)
>> +            if self.dtb:
>> +                print('DTB: [%s]' % self.dtb)
>>          print('MACHINE: [%s]' % self.get('MACHINE'))
>>          print('FSTYPE: [%s]' % self.fstype)
>>          if self.fstype  == 'nfs':
>> @@ -687,7 +699,7 @@ class BaseConfig(object):
>>                  elif os.path.exists(src2):
>>                      src = src2
>>                  if not src:
>> -                    raise Exception("No NFS_DIR is set but can't
>> find %s or %s to extract" % (src1, src2))
>> +                    raise Exception("No NFS_DIR is set, and can't
>> find %s or %s to extract" % (src1, src2))
>>                  logger.info('NFS_DIR not found, extracting %s to %s'
>> % (src, dest))
>>                  cmd = 'runqemu-extract-sdk %s %s' % (src, dest)
>>                  logger.info('Running %s...' % cmd)
>> @@ -845,7 +857,7 @@ class BaseConfig(object):
>>
>>          check_libgl(qemu_bin)
>>
>> -        self.qemu_opt = "%s %s %s %s %s %s" % (qemu_bin,
>> self.get('NETWORK_CMD'), self.qemu_opt_script,
>> self.get('ROOTFS_OPTIONS'), self.get('QB_DTB'),
>> self.get('QB_OPT_APPEND'))
>> +        self.qemu_opt = "%s %s %s %s %s" % (qemu_bin,
>> self.get('NETWORK_CMD'), self.qemu_opt_script,
>> self.get('ROOTFS_OPTIONS'), self.get('QB_OPT_APPEND'))
>>
>>          # Enable virtio RNG else we can run out of entropy in guests
>>          self.qemu_opt += " -device virtio-rng-pci"
>> @@ -877,6 +889,8 @@ class BaseConfig(object):
>>      def start_qemu(self):
>>          if self.kernel:
>>              kernel_opts = "-kernel %s -append '%s %s %s'" %
>> (self.kernel, self.kernel_cmdline, self.kernel_cmdline_script,
>> self.get('QB_KERNEL_CMDLINE_APPEND'))
>> +            if self.dtb:
>> +                kernel_opts += " -dtb %s" % self.dtb
>>          else:
>>              kernel_opts = ""
>>          cmd = "%s %s" % (self.qemu_opt, kernel_opts)
>> --
>> 2.9.0
>>
>