[PATCH v2 0/8] recipetool create improvements

[PATCH v2 0/8] recipetool create improvements

[Paul Eggleton]

Changes since v1:
 * Set PKGV and SUMMARY per package for split npm packages so they get
   the values appropriate to the module they contain. PKGV is
   particularly important so that the image manifests have the
   appropriate version for each module.
 * Create shrinkwrap/lockdown files next to npm recipes and add
   references to them within the recipe
 * Don't fail if license value in an npm module's package.json isn't a
   simple string (more complicated license definitions are allowed
   according to the npm documentation).
 

The following changes since commit 0d02159c8d66bb136f7da2c10fda7d1a57f40cec:

  image.bbclass: fix incomplete .rootfs customization (2016-03-07 22:01:16 +0000)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/recipetool3
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/recipetool3

Paul Eggleton (8):
  recipetool: create: be more tolerant of spacing in configure.ac
  recipetool: create: improve mapping for autotools program macros
  recipetool: create: match *LICENSE* as a license file
  recipetool: create: add license file crunching
  recipetool: create: split npm module dependencies into packages
  recipetool: create: check if npm available if npm:// URL specified
  recipetool: create: support creation of additional files by plugins
  recipetool: create: shrinkwrap and lockdown npm modules

 meta/classes/npm.bbclass                  |  20 ++++
 meta/lib/oe/package.py                    |  32 +++++++
 scripts/lib/devtool/standard.py           |   6 +-
 scripts/lib/recipetool/create.py          | 146 +++++++++++++++++++++++++++++-
 scripts/lib/recipetool/create_buildsys.py |  54 ++++++++---
 scripts/lib/recipetool/create_npm.py      | 114 ++++++++++++++++++++++-
 6 files changed, 349 insertions(+), 23 deletions(-)

-- 
2.5.0

[PATCH v2 1/8] recipetool: create: be more tolerant of spacing in configure.ac

[Paul Eggleton]

Allow for whitespace in appropriate places, and ensure we match all
whitespace chars not just the space character.

(This fixes extracting dependencies from tmux's configure.ac, for
example.)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create_buildsys.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 0228269..909743b 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -413,15 +413,15 @@ class AutotoolsRecipeHandler(RecipeHandler):
         progclassmap = {'gconftool-2': 'gconf',
                 'pkg-config': 'pkgconfig'}
 
-        pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9_]*\]?, *\[?([^,\]]*)\]?[),].*')
-        pkgce_re = re.compile('PKG_CHECK_EXISTS\(\[?([^,\]]*)\]?[),].*')
-        lib_re = re.compile('AC_CHECK_LIB\(\[?([^,\]]*)\]?,.*')
-        libx_re = re.compile('AX_CHECK_LIBRARY\(\[?[^,\]]*\]?, *\[?([^,\]]*)\]?, *\[?([a-zA-Z0-9-]*)\]?,.*')
-        progs_re = re.compile('_PROGS?\(\[?[a-zA-Z0-9_]*\]?, \[?([^,\]]*)\]?[),].*')
+        pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+        pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
+        lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
+        libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
+        progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
         dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?')
-        ac_init_re = re.compile('AC_INIT\(([^,]+), *([^,]+)[,)].*')
-        am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*')
-        define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)')
+        ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
+        am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
+        define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
 
         defines = {}
         def subst_defines(value):
-- 
2.5.0

[PATCH v2 2/8] recipetool: create: improve mapping for autotools program macros

[Paul Eggleton]

Make the following improvements to mapping items specified in
AC_CHECK_PROG, AC_PATH_PROG and AX_WITH_PROG to recipes/classes:

* Produce a map of native recipe -> binary for all binaries currently in
  STAGING_BINDIR_NATIVE and use this when mapping items
* Add some more entries to the class map
* Ignore autotools binaries since they are covered by the inherit of
  autotools
* Ignore coreutils-native since that would almost always be a bogus
  dependency

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create.py          | 22 +++++++++++++++++--
 scripts/lib/recipetool/create_buildsys.py | 36 +++++++++++++++++++++++++------
 2 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 7560cdf..a77c191 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -1,6 +1,6 @@
 # Recipe creation tool - create command plugin
 #
-# Copyright (C) 2014-2015 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -44,6 +44,7 @@ class RecipeHandler(object):
     recipelibmap = {}
     recipeheadermap = {}
     recipecmakefilemap = {}
+    recipebinmap = {}
 
     @staticmethod
     def load_libmap(d):
@@ -122,6 +123,23 @@ class RecipeHandler(object):
                         RecipeHandler.recipecmakefilemap[fn] = pn
 
     @staticmethod
+    def load_binmap(d):
+        '''Build up native binary->recipe mapping'''
+        if RecipeHandler.recipebinmap:
+            return
+        sstate_manifests = d.getVar('SSTATE_MANIFESTS', True)
+        staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE', True)
+        build_arch = d.getVar('BUILD_ARCH', True)
+        fileprefix = 'manifest-%s-' % build_arch
+        for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)):
+            with open(fn, 'r') as f:
+                pn = os.path.basename(fn).rsplit('.', 1)[0][len(fileprefix):]
+                for line in f:
+                    if line.startswith(staging_bindir_native):
+                        prog = os.path.basename(line.rstrip())
+                        RecipeHandler.recipebinmap[prog] = pn
+
+    @staticmethod
     def checkfiles(path, speclist, recursive=False):
         results = []
         if recursive:
@@ -143,7 +161,7 @@ class RecipeHandler(object):
             RecipeHandler.load_libmap(d)
 
         ignorelibs = ['socket']
-        ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native']
+        ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native', 'coreutils-native']
 
         unmappedpc = []
         pcdeps = list(set(pcdeps))
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 909743b..f84ec3d 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -1,6 +1,6 @@
 # Recipe creation tool - create command build system handlers
 #
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -404,14 +404,34 @@ class AutotoolsRecipeHandler(RecipeHandler):
         values = {}
         inherits = []
 
-        # FIXME this mapping is very thin
+        # Hardcoded map, we also use a dynamic one based on what's in the sysroot
         progmap = {'flex': 'flex-native',
                 'bison': 'bison-native',
                 'm4': 'm4-native',
                 'tar': 'tar-native',
-                'ar': 'binutils-native'}
+                'ar': 'binutils-native',
+                'ranlib': 'binutils-native',
+                'ld': 'binutils-native',
+                'strip': 'binutils-native',
+                'libtool': '',
+                'autoconf': '',
+                'autoheader': '',
+                'automake': '',
+                'uname': '',
+                'rm': '',
+                'cp': '',
+                'mv': '',
+                'find': '',
+                'awk': '',
+                'sed': '',
+                }
         progclassmap = {'gconftool-2': 'gconf',
-                'pkg-config': 'pkgconfig'}
+                'pkg-config': 'pkgconfig',
+                'python': 'pythonnative',
+                'python3': 'python3native',
+                'perl': 'perlnative',
+                'makeinfo': 'texinfo',
+                }
 
         pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
         pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
@@ -462,6 +482,8 @@ class AutotoolsRecipeHandler(RecipeHandler):
         deps = []
         unmapped = []
 
+        RecipeHandler.load_binmap(tinfoil.config_data)
+
         def process_macro(keyword, value):
             for handler in handlers:
                 if handler.process_macro(srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values):
@@ -498,10 +520,12 @@ class AutotoolsRecipeHandler(RecipeHandler):
                         if progclass:
                             inherits.append(progclass)
                         else:
-                            progdep = progmap.get(prog, None)
+                            progdep = RecipeHandler.recipebinmap.get(prog, None)
+                            if not progdep:
+                                progdep = progmap.get(prog, None)
                             if progdep:
                                 deps.append(progdep)
-                            else:
+                            elif progdep is None:
                                 if not prog.startswith('$'):
                                     unmapped.append(prog)
             elif keyword == 'AC_CHECK_LIB':
-- 
2.5.0

[PATCH v2 3/8] recipetool: create: match *LICENSE* as a license file

[Paul Eggleton]

For example, this picks up a file named MIT-LICENSE.txt.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index a77c191..def2eea 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -722,7 +722,7 @@ def guess_license(srctree):
     md5sums = get_license_md5sums(tinfoil.config_data)
 
     licenses = []
-    licspecs = ['LICENSE*', 'COPYING*', '*[Ll]icense*', 'LICENCE*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
+    licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
     licfiles = []
     for root, dirs, files in os.walk(srctree):
         for fn in files:
-- 
2.5.0

[PATCH v2 4/8] recipetool: create: add license file crunching

[Paul Eggleton]

Matching license texts directly to md5sums only goes so far. Some
licenses make the copyright statement an intrinsic part of the license
statement (e.g. MIT) which of course varies between projects. Also,
people often seem to take standard license texts such as GPLv2 and
reformat them cosmetically - re-wrapping lines at a different width or
changing quoting styles are seemingly popular examples. In order to
match license files to their actual licenses more effectively, "crunch"
out these elements before comparing to an md5sum. (The existing plain
md5sum matching has been left in since it's a shortcut, and our list of
crunched md5sums isn't a complete replacement for it.)

As always, this code isn't providing any guarantees (legal or otherwise)
that it will always get the license correct - as indicated by the
accompanying comments the LICENSE values it writes out to the recipe are
indicative and you should verify them yourself by looking at the
documentation supplied from upstream for the software being built if you
have any concerns.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create.py | 77 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index def2eea..718f2aa 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -25,6 +25,7 @@ import json
 import logging
 import scriptutils
 import urlparse
+import hashlib
 
 logger = logging.getLogger('recipetool')
 
@@ -717,6 +718,76 @@ def get_license_md5sums(d, static_only=False):
     md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause'
     return md5sums
 
+def crunch_license(licfile):
+    '''
+    Remove non-material text from a license file and then check
+    its md5sum against a known list. This works well for licenses
+    which contain a copyright statement, but is also a useful way
+    to handle people's insistence upon reformatting the license text
+    slightly (with no material difference to the text of the
+    license).
+    '''
+
+    import oe.utils
+
+    # Note: these are carefully constructed!
+    license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$')
+    license_statement_re = re.compile('^This (project|software) is( free software)? released under the .{1,10} [Ll]icen[sc]e:?$')
+    copyright_re = re.compile('^(#+)? *Copyright .*$')
+
+    crunched_md5sums = {}
+    # The following two were gleaned from the "forever" npm package
+    crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
+    crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT'
+    # https://github.com/vasi/pixz/blob/master/LICENSE
+    crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause'
+    # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
+    crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause'
+    # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
+    crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2'
+    # https://github.com/datto/dattobd/blob/master/COPYING
+    # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT
+    crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2'
+    # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
+    # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD
+    crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2'
+    # https://github.com/gkos/nrf24/blob/master/COPYING
+    crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2'
+    # https://github.com/josch09/resetusb/blob/master/COPYING
+    crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3'
+    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
+    crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1'
+    # unixODBC-2.3.4 COPYING
+    crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1'
+    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
+    crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3'
+    lictext = []
+    with open(licfile, 'r') as f:
+        for line in f:
+            # Drop opening statements
+            if copyright_re.match(line):
+                continue
+            elif license_title_re.match(line):
+                continue
+            elif license_statement_re.match(line):
+                continue
+            # Squash spaces, and replace smart quotes, double quotes
+            # and backticks with single quotes
+            line = oe.utils.squashspaces(line.strip()).decode("utf-8")
+            line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
+            if line:
+                lictext.append(line)
+
+    m = hashlib.md5()
+    try:
+        m.update(' '.join(lictext))
+        md5val = m.hexdigest()
+    except UnicodeEncodeError:
+        md5val = None
+        lictext = ''
+    license = crunched_md5sums.get(md5val, None)
+    return license, md5val, lictext
+
 def guess_license(srctree):
     import bb
     md5sums = get_license_md5sums(tinfoil.config_data)
@@ -733,7 +804,11 @@ def guess_license(srctree):
                         licfiles.append(fullpath)
     for licfile in licfiles:
         md5value = bb.utils.md5_file(licfile)
-        license = md5sums.get(md5value, 'Unknown')
+        license = md5sums.get(md5value, None)
+        if not license:
+            license, crunched_md5, lictext = crunch_license(licfile)
+            if not license:
+                license = 'Unknown'
         licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
     # FIXME should we grab at least one source file with a license header and add that too?
-- 
2.5.0

[PATCH v2 5/8] recipetool: create: split npm module dependencies into packages

[Paul Eggleton]

Rather than rolling all of an npm module's dependencies into the same
package, split them into one module per package, setting the SUMMARY and
PKGV values from the package.json file for each package. Additionally,
mark each package with the appropriate license using the license
scanning we already do, falling back to the license stated in the
package.json file for the module if unknown. All of this is mostly in
aid of ensuring all modules and their licenses now show up in the
manifests for the image.

Additionally we set the main LICENSE value more concretely once we've
calculated the per-package licenses, since we have more information at
that point.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/classes/npm.bbclass             | 20 +++++++++++++
 meta/lib/oe/package.py               | 32 ++++++++++++++++++++
 scripts/lib/recipetool/create.py     | 28 ++++++++++++++++++
 scripts/lib/recipetool/create_npm.py | 57 ++++++++++++++++++++++++++++++++++--
 4 files changed, 134 insertions(+), 3 deletions(-)

diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
index be76056..b5db99d 100644
--- a/meta/classes/npm.bbclass
+++ b/meta/classes/npm.bbclass
@@ -18,6 +18,26 @@ npm_do_install() {
 	cp -a ${S}/* ${D}${libdir}/node_modules/${PN}/ --no-preserve=ownership
 }
 
+python populate_packages_prepend () {
+    instdir = d.expand('${D}${libdir}/node_modules/${PN}')
+    extrapackages = oe.package.npm_split_package_dirs(instdir)
+    pkgnames = extrapackages.keys()
+    d.prependVar('PACKAGES', '%s ' % ' '.join(pkgnames))
+    for pkgname in pkgnames:
+        pkgrelpath, pdata = extrapackages[pkgname]
+        pkgpath = '${libdir}/node_modules/${PN}/' + pkgrelpath
+        expanded_pkgname = d.expand(pkgname)
+        d.setVar('FILES_%s' % expanded_pkgname, pkgpath)
+        if pdata:
+            version = pdata.get('version', None)
+            if version:
+                d.setVar('PKGV_%s' % expanded_pkgname, version.encode("utf8"))
+            description = pdata.get('description', None)
+            if description:
+                d.setVar('SUMMARY_%s' % expanded_pkgname, description.replace(u"\u2018", "'").replace(u"\u2019", "'").encode("utf8"))
+    d.appendVar('RDEPENDS_%s' % d.getVar('PN', True), ' %s' % ' '.join(pkgnames))
+}
+
 FILES_${PN} += " \
     ${libdir}/node_modules/${PN} \
 "
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index f176446..dea443d 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -123,3 +123,35 @@ def read_shlib_providers(d):
                         shlib_provider[s[0]] = {}
                     shlib_provider[s[0]][s[1]] = (dep_pkg, s[2])
     return shlib_provider
+
+
+def npm_split_package_dirs(pkgdir):
+    """
+    Work out the packages fetched and unpacked by BitBake's npm fetcher
+    Returns a dict of packagename -> (relpath, package.json) ordered
+    such that it is suitable for use in PACKAGES and FILES
+    """
+    from collections import OrderedDict
+    import json
+    packages = {}
+    for root, dirs, files in os.walk(pkgdir):
+        if os.path.basename(root) == 'node_modules':
+            for dn in dirs:
+                relpth = os.path.relpath(os.path.join(root, dn), pkgdir)
+                pkgitems = ['${PN}']
+                for pathitem in relpth.split('/'):
+                    if pathitem == 'node_modules':
+                        continue
+                    pkgitems.append(pathitem)
+                pkgname = '-'.join(pkgitems)
+                pkgfile = os.path.join(root, dn, 'package.json')
+                data = None
+                if os.path.exists(pkgfile):
+                    with open(pkgfile, 'r') as f:
+                        data = json.loads(f.read())
+                packages[pkgname] = (relpth, data)
+    # We want the main package for a module sorted *after* its subpackages
+    # (so that it doesn't otherwise steal the files for the subpackage), so
+    # this is a cheap way to do that whilst still having an otherwise
+    # alphabetical sort
+    return OrderedDict((key, packages[key]) for key in sorted(packages, key=lambda pkg: pkg + '~'))
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 718f2aa..43c0784 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -544,6 +544,7 @@ def create_recipe(args):
 
     # Apply the handlers
     handled = []
+    handled.append(('license', licvalues))
 
     if args.binary:
         classes.append('bin_package')
@@ -815,6 +816,33 @@ def guess_license(srctree):
 
     return licenses
 
+def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'):
+    """
+    Given a list of (license, path, md5sum) as returned by guess_license(),
+    a dict of package name to path mappings, write out a set of
+    package-specific LICENSE values.
+    """
+    pkglicenses = {pn: []}
+    for license, licpath, _ in licvalues:
+        for pkgname, pkgpath in packages.iteritems():
+            if licpath.startswith(pkgpath + '/'):
+                if pkgname in pkglicenses:
+                    pkglicenses[pkgname].append(license)
+                else:
+                    pkglicenses[pkgname] = [license]
+                break
+        else:
+            # Accumulate on the main package
+            pkglicenses[pn].append(license)
+    outlicenses = {}
+    for pkgname in packages:
+        license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown']))))
+        if license == 'Unknown' and pkgname in fallback_licenses:
+            license = fallback_licenses[pkgname]
+        outlines.append('LICENSE_%s = "%s"' % (pkgname, license))
+        outlicenses[pkgname] = license.split()
+    return outlicenses
+
 def read_pkgconfig_provides(d):
     pkgdatadir = d.getVar('PKGDATA_DIR', True)
     pkgmap = {}
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index 0e33cc9..4bf6cae 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -17,20 +17,37 @@
 
 import logging
 import json
-from recipetool.create import RecipeHandler
+from recipetool.create import RecipeHandler, split_pkg_licenses
 
 logger = logging.getLogger('recipetool')
 
 
 class NpmRecipeHandler(RecipeHandler):
+    def _handle_license(self, data):
+        '''
+        Handle the license value from an npm package.json file
+        '''
+        license = None
+        if 'license' in data:
+            license = data['license']
+            if isinstance(license, dict):
+                license = license.get('type', None)
+        return None
+
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        import oe
+        from collections import OrderedDict
+
         if 'buildsystem' in handled:
             return False
 
+        def read_package_json(fn):
+            with open(fn, 'r') as f:
+                return json.loads(f.read())
+
         files = RecipeHandler.checkfiles(srctree, ['package.json'])
         if files:
-            with open(files[0], 'r') as f:
-                data = json.loads(f.read())
+            data = read_package_json(files[0])
             if 'name' in data and 'version' in data:
                 extravalues['PN'] = data['name']
                 extravalues['PV'] = data['version']
@@ -40,6 +57,40 @@ class NpmRecipeHandler(RecipeHandler):
                     lines_before.append('SUMMARY = "%s"' % data['description'])
                 if 'homepage' in data:
                     lines_before.append('HOMEPAGE = "%s"' % data['homepage'])
+
+                # Split each npm module out to is own package
+                npmpackages = oe.package.npm_split_package_dirs(srctree)
+                for item in handled:
+                    if isinstance(item, tuple):
+                        if item[0] == 'license':
+                            licvalues = item[1]
+                            break
+                if licvalues:
+                    # Augment the license list with information we have in the packages
+                    licenses = {}
+                    license = self._handle_license(data)
+                    if license:
+                        licenses['${PN}'] = license
+                    for pkgname, pkgitem in npmpackages.iteritems():
+                        _, pdata = pkgitem
+                        license = self._handle_license(pdata)
+                        if license:
+                            licenses[pkgname] = license
+                    # Now write out the package-specific license values
+                    # We need to strip out the json data dicts for this since split_pkg_licenses
+                    # isn't expecting it
+                    packages = OrderedDict((x,y[0]) for x,y in npmpackages.iteritems())
+                    packages['${PN}'] = ''
+                    pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
+                    all_licenses = list(set([item for pkglicense in pkglicenses.values() for item in pkglicense]))
+                    # Go back and update the LICENSE value since we have a bit more
+                    # information than when that was written out (and we know all apply
+                    # vs. there being a choice, so we can join them with &)
+                    for i, line in enumerate(lines_before):
+                        if line.startswith('LICENSE = '):
+                            lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses)
+                            break
+
                 return True
 
         return False
-- 
2.5.0

[PATCH v2 6/8] recipetool: create: check if npm available if npm:// URL specified

[Paul Eggleton]

If the user specifies an npm:// URL then the fetcher needs npm to be
available to run, so check if it's available early rather than failing
later.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 43c0784..1d48e36 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -355,6 +355,12 @@ def create_recipe(args):
             srcuri = rev_re.sub('', srcuri)
         tempsrc = tempfile.mkdtemp(prefix='recipetool-')
         srctree = tempsrc
+        if fetchuri.startswith('npm://'):
+            # Check if npm is available
+            npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm')
+            if not npm:
+                logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager')
+                sys.exit(1)
         logger.info('Fetching %s...' % srcuri)
         try:
             checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev)
-- 
2.5.0

[PATCH v2 7/8] recipetool: create: support creation of additional files by plugins

[Paul Eggleton]

Allow plugins to create additional files to go alongside the recipe. The
plugins don't know what the output filename is going to be, so they need
to put the files in a temporary location and add them to an "extrafiles"
dict within extravalues where the destination filename is the key and
the temporary path is the value.

devtool add was also extended to ensure these files get moved in and
preserved upon reset if they've been edited by the user.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/devtool/standard.py  |  6 +++++-
 scripts/lib/recipetool/create.py | 11 +++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 5f83a91..b344001 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -189,6 +189,9 @@ def add(args, config, basepath, workspace):
                     raise DevtoolError('Couldn\'t find source tree created by recipetool')
             bb.utils.mkdirhier(recipedir)
             shutil.move(recipes[0], recipefile)
+            # Move any additional files created by recipetool
+            for fn in os.listdir(tempdir):
+                shutil.move(os.path.join(tempdir, fn), recipedir)
         else:
             raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
         attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
@@ -199,7 +202,8 @@ def add(args, config, basepath, workspace):
             shutil.rmtree(tmpsrcdir)
         shutil.rmtree(tempdir)
 
-    _add_md5(config, recipename, recipefile)
+    for fn in os.listdir(recipedir):
+        _add_md5(config, recipename, os.path.join(recipedir, fn))
 
     if args.fetch and not args.no_git:
         setup_git_repo(srctree, args.version, 'devtool')
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 1d48e36..1649e40 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -560,6 +560,8 @@ def create_recipe(args):
     for handler in handlers:
         handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
 
+    extrafiles = extravalues.pop('extrafiles', {})
+
     if not realpv:
         realpv = extravalues.get('PV', None)
         if realpv:
@@ -601,6 +603,15 @@ def create_recipe(args):
                 logger.error('Output file %s already exists' % outfile)
                 sys.exit(1)
 
+    # Move any extra files the plugins created to a directory next to the recipe
+    if outfile == '-':
+        extraoutdir = pn
+    else:
+        extraoutdir = os.path.join(os.path.dirname(outfile), pn)
+    bb.utils.mkdirhier(extraoutdir)
+    for destfn, extrafile in extrafiles.iteritems():
+        shutil.move(extrafile, os.path.join(extraoutdir, destfn))
+
     lines = lines_before
     lines_before = []
     skipblank = True
-- 
2.5.0

[PATCH v2 8/8] recipetool: create: shrinkwrap and lockdown npm modules

[Paul Eggleton]

"npm shrinkwrap" creates a file that ensures that the exact same
versions get fetched the next time the recipe is built. lockdown is
similar but also includes sha1sums of the modules thus validating they
haven't changed between builds. These ensure that the build is
reproducible.

Fixes [YOCTO #9225].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 scripts/lib/recipetool/create_npm.py | 57 ++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index 4bf6cae..b3ffcdb 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -15,14 +15,27 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import os
 import logging
+import subprocess
+import tempfile
+import shutil
 import json
 from recipetool.create import RecipeHandler, split_pkg_licenses
 
 logger = logging.getLogger('recipetool')
 
 
+tinfoil = None
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+
 class NpmRecipeHandler(RecipeHandler):
+    lockdownpath = None
+
     def _handle_license(self, data):
         '''
         Handle the license value from an npm package.json file
@@ -34,7 +47,44 @@ class NpmRecipeHandler(RecipeHandler):
                 license = license.get('type', None)
         return None
 
+    def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before):
+        try:
+            runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True))
+            bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+        except bb.process.ExecutionError as e:
+            logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
+            return
+
+        tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
+        shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
+        extravalues.setdefault('extrafiles', {})
+        extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
+        lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
+
+    def _lockdown(self, srctree, localfilesdir, extravalues, lines_before):
+        runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True))
+        if not NpmRecipeHandler.lockdownpath:
+            NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
+            bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
+                           cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+        relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
+        if not os.path.exists(relockbin):
+            logger.warn('Could not find relock.js within lockdown directory; skipping lockdown')
+            return
+        try:
+            bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+        except bb.process.ExecutionError as e:
+            logger.warn('lockdown-relock failed:\n%s' % e.stdout)
+            return
+
+        tmpfile = os.path.join(localfilesdir, 'lockdown.json')
+        shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
+        extravalues.setdefault('extrafiles', {})
+        extravalues['extrafiles']['lockdown.json'] = tmpfile
+        lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
+
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        import bb.utils
         import oe
         from collections import OrderedDict
 
@@ -58,6 +108,13 @@ class NpmRecipeHandler(RecipeHandler):
                 if 'homepage' in data:
                     lines_before.append('HOMEPAGE = "%s"' % data['homepage'])
 
+                # Shrinkwrap
+                localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
+                self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before)
+
+                # Lockdown
+                self._lockdown(srctree, localfilesdir, extravalues, lines_before)
+
                 # Split each npm module out to is own package
                 npmpackages = oe.package.npm_split_package_dirs(srctree)
                 for item in handled:
-- 
2.5.0