[PATCH 1/5] recipetool-create: add ensure_native_cmd function

[PATCH 1/5] recipetool-create: add ensure_native_cmd function

[Stefan Herbrechtsmeier]

From: Lukas Funke <lukas.funke@weidmueller.com>

Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
---

 scripts/lib/recipetool/create.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 824ac6350d..efcb82173e 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -1375,6 +1375,29 @@ def convert_rpm_xml(xmlfile):
                     values[varname] = child[0].text
     return values
 
+def ensure_native_cmd(tinfoil, cmd):
+    """Check if the command is available in the recipes"""
+    if not tinfoil.recipes_parsed:
+        tinfoil.parse_recipes()
+
+    try:
+        d = tinfoil.parse_recipe("%s-native" % cmd)
+    except bb.providers.NoProvider:
+        bb.error("Nothing provides '%s-native' which is required for the build" % cmd)
+        bb.note("You will likely need to add a layer that provides %s" % cmd)
+        sys.exit(14)
+
+    bindir = d.getVar("STAGING_BINDIR_NATIVE")
+    cmdpath = os.path.join(bindir, cmd)
+
+    if not os.path.exists(cmdpath):
+        tinfoil.build_targets("%s-native" % cmd, "addto_recipe_sysroot")
+
+        if not os.path.exists(cmdpath):
+            bb.error("Failed to add '%s' to sysroot" % cmd)
+            sys.exit(14)
+
+    return bindir
 
 def register_commands(subparsers):
     parser_create = subparsers.add_parser('create',
-- 
2.30.2

[PATCH 2/5] create_npm: reuse ensure_native_cmd from create.py

[Stefan Herbrechtsmeier]

From: Lukas Funke <lukas.funke@weidmueller.com>

Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
---

 scripts/lib/recipetool/create_npm.py | 28 ++--------------------------
 1 file changed, 2 insertions(+), 26 deletions(-)

diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index 3394a89970..3aed59252c 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -18,6 +18,7 @@ from recipetool.create import RecipeHandler
 from recipetool.create import get_license_md5sums
 from recipetool.create import guess_license
 from recipetool.create import split_pkg_licenses
+from recipetool.create import ensure_native_cmd
 logger = logging.getLogger('recipetool')
 
 TINFOIL = None
@@ -54,31 +55,6 @@ class NpmRecipeHandler(RecipeHandler):
 
         return registry
 
-    @staticmethod
-    def _ensure_npm():
-        """Check if the 'npm' command is available in the recipes"""
-        if not TINFOIL.recipes_parsed:
-            TINFOIL.parse_recipes()
-
-        try:
-            d = TINFOIL.parse_recipe("nodejs-native")
-        except bb.providers.NoProvider:
-            bb.error("Nothing provides 'nodejs-native' which is required for the build")
-            bb.note("You will likely need to add a layer that provides nodejs")
-            sys.exit(14)
-
-        bindir = d.getVar("STAGING_BINDIR_NATIVE")
-        npmpath = os.path.join(bindir, "npm")
-
-        if not os.path.exists(npmpath):
-            TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")
-
-            if not os.path.exists(npmpath):
-                bb.error("Failed to add 'npm' to sysroot")
-                sys.exit(14)
-
-        return bindir
-
     @staticmethod
     def _npm_global_configs(dev):
         """Get the npm global configuration"""
@@ -190,7 +166,7 @@ class NpmRecipeHandler(RecipeHandler):
         # npm version is high enough to ensure an efficient dependency tree
         # resolution and avoid issue with the shrinkwrap file format.
         # Moreover the native npm is mandatory for the build.
-        bindir = self._ensure_npm()
+        bindir = ensure_native_cmd(TINFOIL, "npm")
 
         d = bb.data.createCopy(TINFOIL.config_data)
         d.prependVar("PATH", bindir + ":")
-- 
2.30.2

[PATCH 3/5] poky-meta: add go vendor class for offline builds

[Stefan Herbrechtsmeier]

From: Lukas Funke <lukas.funke@weidmueller.com>

Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
---

 meta/classes/go-vendor.bbclass | 68 ++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 meta/classes/go-vendor.bbclass

diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
new file mode 100644
index 0000000000..a1a740a1fc
--- /dev/null
+++ b/meta/classes/go-vendor.bbclass
@@ -0,0 +1,68 @@
+# Copyright 2022 (C) Weidmueller GmbH & Co KG
+# Author: Lukas Funke <lukas.funke@weidmueller.com>
+#
+# Handle Go vendor support for offline builds
+#
+# When importing Go modules, Go downloads the imported module using
+# a network (proxy) connection ahead of the compile stage. This contradicts 
+# the yocto build concept of fetching every source ahead of build-time
+# and supporting offline builds.
+#
+# To support offline builds, we use Go 'vendoring': module dependencies are 
+# downloaded during the fetch-phase and unpacked into the modules 'vendor'
+# folder. Additinally a manifest file is generated for the 'vendor' folder
+# 
+
+inherit go-mod
+
+def go_src_uri(repo, path=None, subdir=None, vcs='git', destsuffix_prefix = 'git/src/import/vendor.fetch'):
+    module_path = repo if not path else path
+    src_uri = "{}://{};name={};destsuffix={}/{}".format(vcs, repo, \
+                                    module_path.replace('/', '.'), \
+                                    destsuffix_prefix, module_path)
+
+    src_uri += ";subdir={}".format(subdir) if subdir else ""
+    src_uri += ";nobranch=1;protocol=https" if vcs == "git" else ""
+
+    return src_uri
+
+def go_generate_vendor_manifest(d):
+
+    vendor_dir = os.path.join(os.path.basename(d.getVar('S')),
+                                        'src', d.getVar('GO_IMPORT'), "vendor")
+    dst = os.path.join(vendor_dir, "modules.txt")
+
+    go_modules = d.getVarFlags("GO_MODULE_PATH")
+    with open(dst, "w") as manifest:
+        for go_module in go_modules:
+            module_path = d.getVarFlag("GO_MODULE_PATH", go_module)
+            module_version = d.getVarFlag("GO_MODULE_VERSION", go_module)
+            if module_path and module_version:
+                manifest.write("# %s %s\n" % (module_path, module_version))
+                manifest.write("## explicit\n")
+                exclude = set(['vendor'])
+                for subdir, dirs, files in os.walk(os.path.join(vendor_dir, module_path), topdown=True):
+                    dirs[:] = [d for d in dirs if d not in exclude]
+                    for file in files:
+                        if file.endswith(".go"):
+                            manifest.write(subdir[len(vendor_dir)+1:] + "\n")
+                            break
+
+python go_do_unpack:append() {
+    src_uri = (d.getVar('SRC_URI') or "").split()
+    if len(src_uri) == 0:
+        return
+
+    try:
+        fetcher = bb.fetch2.Fetch(src_uri, d)
+        src_folder = os.path.join(os.path.basename(d.getVar('S')),
+                                        'src', d.getVar('GO_IMPORT'))
+        vendor_src = os.path.join(src_folder, "vendor")
+        vendor_dst = os.path.join(d.getVar('S'), "src", "import", "vendor.fetch")
+
+        os.symlink(os.path.relpath(vendor_dst, src_folder), vendor_src)
+        go_generate_vendor_manifest(d)
+
+    except bb.fetch2.BBFetchException as e:
+        raise bb.build.FuncFailed(e)
+}
-- 
2.30.2

[PATCH 4/5] recipetool: add go recipe generator

[Stefan Herbrechtsmeier]

From: Lukas Funke <lukas.funke@weidmueller.com>

Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
---

 scripts/lib/recipetool/create_go.py | 394 ++++++++++++++++++++++++++++
 1 file changed, 394 insertions(+)
 create mode 100644 scripts/lib/recipetool/create_go.py

diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
new file mode 100644
index 0000000000..4552e9b470
--- /dev/null
+++ b/scripts/lib/recipetool/create_go.py
@@ -0,0 +1,394 @@
+# Recipe creation tool - go support plugin
+#
+# Copyright (C) 2022 Weidmueller GmbH & Co KG
+# Author: Lukas Funke <lukas.funke@weidmueller.com>
+#
+# Copyright (c) 2009 The Go Authors. All rights reserved.
+#
+#  SPDX-License-Identifier: BSD-3-Clause AND GPL-2.0-only
+#
+import bb.utils
+from collections import namedtuple
+from enum import Enum
+from html.parser import HTMLParser
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import shutil
+from urllib.error import URLError, HTTPError
+import urllib.parse
+import urllib.request
+
+from recipetool.create import RecipeHandler, handle_license_vars, ensure_native_cmd
+
+GoImport = namedtuple('GoImport', 'reporoot vcs repourl suffix')
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+re_pseudo_semver = re.compile(r"v([0-9]+)\.([0-9]+).([0-9]+|\([0-9]+\+1\))-(pre\.[0-9]+\.)?([0-9]+\.)?(?P<utc>[0-9]+)-(?P<sha1_abbrev>[0-9Aa-zA-Z]+)")
+re_semver = re.compile(r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+class GoRecipeHandler(RecipeHandler):
+
+    def _resolve_repository_static(self, modulepath):
+        _rootpath = None
+        _vcs = None
+        _repourl = None
+        _suffix = None
+
+        host, _, path = modulepath.partition('/')
+
+        class vcs(Enum):
+            pathprefix = "pathprefix"
+            regexp = "regexp"
+            vcs = "vcs"
+            repo = "repo"
+            check = "check"
+            schemelessRepo = "schemelessRepo"
+
+        # GitHub
+        vcsGitHub = {}
+        vcsGitHub[vcs.pathprefix] = "github.com"
+        vcsGitHub[vcs.regexp] = re.compile(r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsGitHub[vcs.vcs] = "git"
+        vcsGitHub[vcs.repo] = "https://\g<root>"
+
+        # Bitbucket
+        vcsBitbucket = {}
+        vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
+        vcsBitbucket[vcs.regexp] = re.compile(r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsBitbucket[vcs.vcs] = "git"
+        vcsBitbucket[vcs.repo] = "https://\g<root>"
+
+        # IBM DevOps Services (JazzHub)
+        vcsIBMDevOps = {}
+        vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
+        vcsIBMDevOps[vcs.regexp] = re.compile(r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsIBMDevOps[vcs.vcs] = "git"
+        vcsIBMDevOps[vcs.repo] = "https://\g<root>"
+
+        # Git at Apache
+        vcsApacheGit = {}
+        vcsApacheGit[vcs.pathprefix] = "git.apache.org"
+        vcsApacheGit[vcs.regexp] = re.compile(r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsApacheGit[vcs.vcs] = "git"
+        vcsApacheGit[vcs.repo] = "https://\g<root>"
+
+        # Git at OpenStack
+        vcsOpenStackGit = {}
+        vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
+        vcsOpenStackGit[vcs.regexp] = re.compile(r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsOpenStackGit[vcs.vcs] = "git"
+        vcsOpenStackGit[vcs.repo] = "https://\g<root>"
+
+        # chiselapp.com for fossil
+        vcsChiselapp = {}
+        vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
+        vcsChiselapp[vcs.regexp] = re.compile(r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
+        vcsChiselapp[vcs.vcs] = "fossil"
+        vcsChiselapp[vcs.repo] = "https://\g<root>"
+
+        # General syntax for any server.
+        # Must be last.
+        vcsGeneralServer = {}
+        vcsGeneralServer[vcs.regexp] = re.compile("(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\-]+))*$")
+        vcsGeneralServer[vcs.schemelessRepo] = True
+
+        vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, vcsApacheGit, vcsOpenStackGit, vcsChiselapp, vcsGeneralServer]
+
+        if modulepath.startswith("example.net") or modulepath == "rsc.io":
+            logger.warning("Suspicious module path %s" % modulepath)
+            return None
+        if modulepath.startswith("http:") or modulepath.startswith("https:"):
+            logger.warning("Import path should not start with %s %s" % ("http", "https"))
+            return None
+
+        for srv in vcsPaths:
+            m = srv[vcs.regexp].match(modulepath)
+            if vcs.pathprefix in srv:
+                if host == srv[vcs.pathprefix]:
+                    _rootpath = m.group('root')
+                    _vcs = srv[vcs.vcs]
+                    _repourl = m.expand(srv[vcs.repo])
+                    _suffix = m.group('suffix')
+                    break
+            elif m and srv[vcs.schemelessRepo]:
+                _rootpath = m.group('root')
+                _vcs = m[vcs.vcs]
+                _repourl = m[vcs.repo]
+                _suffix = m.group('suffix')
+                break
+
+        return GoImport(_rootpath, _vcs, _repourl, _suffix)
+
+    def _resolve_repository_dynamic(self, modulepath):
+
+        url = urllib.parse.urlparse("https://" + modulepath)
+
+        class GoImportHTMLParser(HTMLParser):
+
+            def __init__(self):
+                super().__init__()
+                self.__srv = []
+
+            def handle_starttag(self, tag, attrs):
+                if tag == 'meta' and list(filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)):
+                    content = list(filter(lambda a: (a[0] == 'content'), attrs))
+                    if content:
+                        self.__srv = content[0][1].split()
+
+            @property
+            def rootpath(self):
+                return self.__srv[0]
+
+            @property
+            def vcs(self):
+                return self.__srv[1]
+
+            @property
+            def repourl(self):
+                return self.__srv[2]
+
+        req = urllib.request.Request(url.geturl() + "?go-get=1")
+
+        try:
+            resp = urllib.request.urlopen(req)
+        except URLError as url_err:
+            logger.error("Error while fetching redirect page: %s", str(url_err))
+            return None
+        except HTTPError as http_err:
+            logger.error("Error while fetching redirect page: %s", str(http_err))
+            return None
+
+        parser = GoImportHTMLParser()
+        parser.feed(resp.read().decode('utf-8'))
+        parser.close()
+
+        return GoImport(parser.rootpath, parser.vcs, parser.repourl, None)
+
+    def _resolve_repository(self, modulepath):
+        """
+        Resolves src uri from go module-path
+        """
+        repodata = self._resolve_repository_static(modulepath)
+        if not repodata.repourl:
+            repodata = self._resolve_repository_dynamic(modulepath)
+
+        if repodata:
+            logger.info("Resolved download path for import '%s' => %s", modulepath, repodata.repourl)
+
+        return repodata
+
+    def _resolve_pseudo_semver(self, d, repo, module_version):
+        hash = None
+
+        def vcs_fetch_all():
+            tmpdir = tempfile.mkdtemp()
+            clone_cmd = "%s clone --bare %s %s" % ('git', repo, tmpdir)
+            bb.process.run(clone_cmd)
+            log_cmd = "git log --all --pretty='%H %d' --decorate=short"
+            output, errors = bb.process.run(log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir)
+            bb.utils.prunedir(tmpdir)
+            return output.strip().split('\n')
+
+        def vcs_fetch_remote(search=""):
+            ls_remote_cmd = "git ls-remote --tags {} {}".format(repo, search)
+            output, errors = bb.process.run(ls_remote_cmd)
+            return output.strip().split('\n')
+
+        m_pseudo_semver = re_pseudo_semver.match(module_version)
+        if m_pseudo_semver:
+            remote_refs = vcs_fetch_all()
+            short_commit = m_pseudo_semver.group('sha1_abbrev')
+            for l in remote_refs:
+                r = l.split(maxsplit=1)
+                sha1 = r[0] if len(r) else None
+                if not sha1:
+                    logger.error("Ups: could not resolve abbref commit for %s" % short_commit)
+
+                elif sha1.startswith(short_commit):
+                    hash = sha1
+                    break
+        else:
+            m_semver = re_semver.match(module_version)
+            if m_semver:
+
+                def get_sha1_remote(re, groupId):
+                    for l in remote_refs:
+                        r = l.split(maxsplit=1)
+                        sha1 = r[0] if len(r) else None
+                        ref = r[1] if len(r) == 2 else None
+                        if ref:
+                            m = re.match(ref)
+                            if m and semver_tag in m.group(groupId).split(','):
+                                return sha1
+                    return None
+
+                re_tags_remote = re.compile(r"refs/tags/(?P<tag>[0-9A-Za-z-_\.]+)")
+                re_tags_all = re.compile(r"\((HEAD -> (.*), )?tag: *((?:([0-9A-Za-z-_\.]+),? *)+)\)")
+                semver_tag = "v" + m_semver.group('major') + "."\
+                                +m_semver.group('minor') + "."\
+                                +m_semver.group('patch') \
+                                +(("-" + m_semver.group('prerelease')) if m_semver.group('prerelease') else "")
+                remote_refs = vcs_fetch_remote(semver_tag)
+                # probe tag using 'ls-remote', which is faster than fetching complete history
+                sha1 = get_sha1_remote(re_tags_remote, 'tag')
+                if sha1:
+                    hash = sha1
+                else:
+                    # backup: fetch complete history
+                    remote_refs = vcs_fetch_all()
+                    hash = get_sha1_remote(re_tags_all, 3)
+        return hash
+
+    def _handle_dependencies(self, d, srctree, go_mod):
+        runenv = dict(os.environ, PATH=d.getVar('PATH'))
+        src_uris = []
+        src_revs = []
+        for require in go_mod['Require']:
+            module_path = require['Path']
+            module_version = require['Version']
+
+            repodata = self._resolve_repository(module_path)
+            commit_id = self._resolve_pseudo_semver(d, repodata.repourl, module_version)
+            url = urllib.parse.urlparse(repodata.repourl)
+            repo_url = url.netloc + url.path
+            inline_fcn = "${@go_src_uri("
+            inline_fcn += "'{}'".format(repo_url)
+            if repo_url != module_path:
+                inline_fcn += ",path='{}'".format(module_path)
+            if repodata.suffix and not re.match("v[0-9]+", repodata.suffix):
+                inline_fcn += ",subdir='{}'".format(repodata.suffix)
+            if repodata.vcs != 'git':
+                inline_fcn += ",vcs='{}'".format(repodata.vcs)
+            inline_fcn += ")}"
+
+            src_uris.append(inline_fcn)
+            flat_module_path = module_path.replace('/', '.')
+            src_rev = "# %s@%s => %s\n" % (module_path, module_version, commit_id)
+            src_rev += "SRCREV_%s = \"%s\"\n" % (flat_module_path, commit_id)
+            src_rev += "GO_MODULE_PATH[%s] = \"%s\"\n" % (flat_module_path, module_path)
+            src_rev += "GO_MODULE_VERSION[%s] = \"%s\"" % (flat_module_path, module_version)
+            src_revs.append(src_rev)
+
+        return src_uris, src_revs
+
+    def _go_mod_patch(self, patchfile, go_import, srctree, localfilesdir, extravalues, d):
+        runenv = dict(os.environ, PATH=d.getVar('PATH'))
+        # first remove go.mod and go.sum, otherwise 'go mod init' will fail
+        bb.utils.remove(os.path.join(srctree, "go.mod"))
+        bb.utils.remove(os.path.join(srctree, "go.sum"))
+        bb.process.run("go mod init %s" % go_import, stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
+        bb.process.run("go mod tidy", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
+        output, _ = bb.process.run("go mod edit -json", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
+        bb.process.run("git diff go.mod > %s" % (patchfile), stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
+        bb.process.run("git checkout HEAD go.mod go.sum;", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
+        go_mod = json.loads(output)
+        tmpfile = os.path.join(localfilesdir, patchfile)
+        shutil.move(os.path.join(srctree, patchfile), tmpfile)
+        extravalues.setdefault('extrafiles', {})
+        extravalues['extrafiles'][patchfile] = tmpfile
+
+        return go_mod
+
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+
+        if 'buildsystem' in handled:
+            return False
+
+        files = RecipeHandler.checkfiles(srctree, ['go.mod'])
+        if not files:
+            return False
+
+        go_bindir = ensure_native_cmd(tinfoil, "go")
+
+        d = bb.data.createCopy(tinfoil.config_data)
+        d.prependVar('PATH', '%s:' % go_bindir)
+        handled.append('buildsystem')
+        classes.append("go-vendor")
+
+        runenv = dict(os.environ, PATH=d.getVar('PATH'))
+        output, _ = bb.process.run("go mod edit -json", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
+        go_mod = json.loads(output)
+
+        go_import = go_mod['Module']['Path']
+        go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go'])
+        go_version_major = int(go_version_match.group(1))
+        go_version_minor = int(go_version_match.group(2))
+        src_uris = []
+        if go_version_major == 1 and go_version_minor < 17:
+            logger.warning("go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.")
+            patchfile = "go.mod.patch"
+            localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
+            go_mod = self._go_mod_patch(patchfile, go_import, srctree, localfilesdir, extravalues, d)
+            src_uris.append("file://%s;patchdir=src/${GO_IMPORT}" % (patchfile))
+
+        if not os.path.exists(os.path.join(srctree, "vendor")):
+            dep_src_uris, src_revs = self._handle_dependencies(d, srctree, go_mod)
+            src_uris.extend(dep_src_uris)
+            lines_after.append("#TODO: Subdirectories are heuristically derived from " \
+                              "the import path and might be incorrect.")
+            for src_rev in src_revs:
+                lines_after.append(src_rev)
+
+        self._rewrite_src_uri(src_uris, lines_before)
+
+        handle_license_vars(srctree, lines_before, handled, extravalues, d)
+        self._rewrite_lic_uri(lines_before)
+
+        lines_before.append("GO_IMPORT = \"{}\"".format(go_import))
+        lines_before.append("SRCREV_FORMAT = \"${PN}\"")
+
+    def _update_lines_before(self, updated, newlines, lines_before):
+        if updated:
+            del lines_before[:]
+            for line in newlines:
+                # Hack to avoid newlines that edit_metadata inserts
+                if line.endswith('\n'):
+                    line = line[:-1]
+                lines_before.append(line)
+        return updated
+
+    def _rewrite_lic_uri(self, lines_before):
+
+        def varfunc(varname, origvalue, op, newlines):
+            if varname == 'LIC_FILES_CHKSUM':
+                new_licenses = []
+                licenses = origvalue.split()
+
+                for license in licenses:
+                    uri, chksum = license.split(';', 1)
+                    url = urllib.parse.urlparse(uri)
+                    new_uri = os.path.join(url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
+                    new_licenses.append(new_uri)
+
+                return new_licenses, None, -1, True
+            return origvalue, None, 0, True
+
+        updated, newlines = bb.utils.edit_metadata(lines_before, ['LIC_FILES_CHKSUM'], varfunc)
+        return self._update_lines_before(updated, newlines, lines_before)
+
+    def _rewrite_src_uri(self, src_uris_deps, lines_before):
+
+        def varfunc(varname, origvalue, op, newlines):
+            if varname == 'SRC_URI':
+                src_uri = []
+                src_uri.append("git://${GO_IMPORT};nobranch=1;name=${PN};protocol=https")
+                src_uri.extend(src_uris_deps)
+                return src_uri, None, -1, True
+            return origvalue, None, 0, True
+
+        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
+        return self._update_lines_before(updated, newlines, lines_before)
+
+def register_recipe_handlers(handlers):
+    handlers.append((GoRecipeHandler(), 60))
-- 
2.30.2

[PATCH 5/5] oe-selftest: add go recipe create selftest

[Stefan Herbrechtsmeier]

From: Lukas Funke <lukas.funke@weidmueller.com>

Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>

Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
---

 meta/lib/oeqa/selftest/cases/recipetool.py | 88 ++++++++++++++++++++++
 1 file changed, 88 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py b/meta/lib/oeqa/selftest/cases/recipetool.py
index 510dae6bad..bb36e7b6d5 100644
--- a/meta/lib/oeqa/selftest/cases/recipetool.py
+++ b/meta/lib/oeqa/selftest/cases/recipetool.py
@@ -530,6 +530,94 @@ class RecipetoolTests(RecipetoolBase):
         libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'recipetool')
         sys.path.insert(0, libpath)
 
+    def test_recipetool_create_go(self):
+        # Basic test to check go recipe generation
+        def urifiy(url, version, path = None, subdir = None):
+            path = ",path='%s'" % path if path else ''
+            subdir = ",subdir='%s'" % subdir if subdir else ''
+            return "${@go_src_uri('%s','%s'%s%s)}" % (url, path, subdir)
+
+        temprecipe = os.path.join(self.tempdir, 'recipe')
+        os.makedirs(temprecipe)
+        recipefile = os.path.join(temprecipe, 'edgex-go_git.bb')
+        srcuri = 'https://github.com/edgexfoundry/edgex-go.git'
+        srcrev = "v2.2.0-dev.54"
+        result = runCmd('recipetool create -o %s %s -S %s' % (temprecipe, srcuri, srcrev))
+        self.assertTrue(os.path.isfile(recipefile))
+        checkvars = {}
+        src_uri = ['git://${GO_IMPORT};nobranch=1;name=${BPN}']
+        checkvars['LIC_FILES_CHKSUM'] = set(['file://src/${GO_IMPORT}/LICENSE;md5=71a6955f3cd81a809549da266346dc59'])
+        checkvars['GO_IMPORT'] = "github.com/edgexfoundry/edgex-go"
+        inherits = ['go-vendor']
+        dependencies = \
+            [
+                ('bitbucket.org/bertimus9/systemstat'),
+                ('github.com/edgexfoundry/go-mod-bootstrap','github.com/edgexfoundry/go-mod-bootstrap/v2'),
+                ('github.com/edgexfoundry/go-mod-core-contracts''github.com/edgexfoundry/go-mod-core-contracts/v2'),
+                ('github.com/edgexfoundry/go-mod-messaging','github.com/edgexfoundry/go-mod-messaging/v2'),
+                ('github.com/edgexfoundry/go-mod-registry','github.com/edgexfoundry/go-mod-registry/v2'),
+                ('github.com/edgexfoundry/go-mod-secrets','github.com/edgexfoundry/go-mod-secrets/v2'),
+                ('github.com/fxamacker/cbor','github.com/fxamacker/cbor/v2'),
+                ('github.com/golang-jwt/jwt','github.com/golang-jwt/jwt/v4'),
+                ('github.com/gomodule/redigo'),
+                ('github.com/google/uuid'),
+                ('github.com/gorilla/mux'),
+                ('github.com/lib/pq'),
+                ('github.com/pelletier/go-toml'),
+                ('github.com/spiffe/go-spiffe','github.com/spiffe/go-spiffe/v2'),
+                ('github.com/stretchr/testify'),
+                ('go.googlesource.com/crypto','golang.org/x/crypto'),
+                ('gopkg.in/eapache/queue.v1'),
+                ('gopkg.in/yaml.v3'),
+                ('github.com/armon/go-metrics'),
+                ('github.com/cenkalti/backoff'),
+                ('github.com/davecgh/go-spew'),
+                ('github.com/eclipse/paho.mqtt.golang'),
+                ('github.com/edgexfoundry/go-mod-configuration','github.com/edgexfoundry/go-mod-configuration/v2'),
+                ('github.com/fatih/color'),
+                ('github.com/go-kit/log'),
+                ('github.com/go-logfmt/logfmt'),
+                ('github.com/go-playground/locales'),
+                ('github.com/go-playground/universal-translator'),
+                ('github.com/go-playground/validator','github.com/go-playground/validator/v10'),
+                ('github.com/go-redis/redis','github.com/go-redis/redis/v7'),
+                ('github.com/golang/protobuf'),
+                ('github.com/gorilla/websocket'),
+                ('github.com/hashicorp/consul','api'),
+                ('github.com/hashicorp/errwrap'),
+                ('github.com/hashicorp/go-cleanhttp'),
+                ('github.com/hashicorp/go-hclog'),
+                ('github.com/hashicorp/go-immutable-radix'),
+                ('github.com/hashicorp/go-multierror'),
+                ('github.com/hashicorp/go-rootcerts'),
+                ('github.com/hashicorp/golang-lru'),
+                ('github.com/hashicorp/serf'),
+                ('github.com/leodido/go-urn'),
+                ('github.com/mattn/go-colorable'),
+                ('github.com/mattn/go-isatty'),
+                ('github.com/mitchellh/consulstructure'),
+                ('github.com/mitchellh/copystructure'),
+                ('github.com/mitchellh/go-homedir'),
+                ('github.com/mitchellh/mapstructure'),
+                ('github.com/mitchellh/reflectwalk'),
+                ('github.com/pebbe/zmq4'),
+                ('github.com/pmezard/go-difflib'),
+                ('github.com/stretchr/objx'),
+                ('github.com/x448/float16'),
+                ('github.com/zeebo/errs'),
+                ('go.googlesource.com/net','golang.org/x/net'),
+                ('go.googlesource.com/sys','golang.org/x/sys'),
+                ('go.googlesource.com/text','golang.org/x/text'),
+                ('github.com/googleapis/go-genproto','google.golang.org/genproto'),
+                ('github.com/grpc/grpc-go','google.golang.org/grpc'),
+                ('go.googlesource.com/protobuf','google.golang.org/protobuf'),
+                ('gopkg.in/square/go-jose.v2'),
+            ]
+        src_uri = {urifiy(*d) for d in dependencies}
+        checkvars['SRC_URI'] = set(src_uri)
+        self.maxDiff = None
+        self._test_recipe_contents(recipefile, checkvars, inherits)
+        
     def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths):
         dstdir = basedstdir
         self.assertTrue(os.path.exists(dstdir))
-- 
2.30.2

Re: [OE-core] [PATCH 1/5] recipetool-create: add ensure_native_cmd function

[Alexander Kanavin]

I think the terminology is a bit confused here, instead of 'command'
it should say 'recipe'. Also:
"""Check if the command is available in the recipes""" is not
everything it does, it also builds the recipe.

And please write a commit message that explains what and why.

Alex

On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
<stefan.herbrechtsmeier-oss@weidmueller.com> wrote:
>
> From: Lukas Funke <lukas.funke@weidmueller.com>
>
> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
> ---
>
>  scripts/lib/recipetool/create.py | 23 +++++++++++++++++++++++
>  1 file changed, 23 insertions(+)
>
> diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
> index 824ac6350d..efcb82173e 100644
> --- a/scripts/lib/recipetool/create.py
> +++ b/scripts/lib/recipetool/create.py
> @@ -1375,6 +1375,29 @@ def convert_rpm_xml(xmlfile):
>                      values[varname] = child[0].text
>      return values
>
> +def ensure_native_cmd(tinfoil, cmd):
> +    """Check if the command is available in the recipes"""
> +    if not tinfoil.recipes_parsed:
> +        tinfoil.parse_recipes()
> +
> +    try:
> +        d = tinfoil.parse_recipe("%s-native" % cmd)
> +    except bb.providers.NoProvider:
> +        bb.error("Nothing provides '%s-native' which is required for the build" % cmd)
> +        bb.note("You will likely need to add a layer that provides %s" % cmd)
> +        sys.exit(14)
> +
> +    bindir = d.getVar("STAGING_BINDIR_NATIVE")
> +    cmdpath = os.path.join(bindir, cmd)
> +
> +    if not os.path.exists(cmdpath):
> +        tinfoil.build_targets("%s-native" % cmd, "addto_recipe_sysroot")
> +
> +        if not os.path.exists(cmdpath):
> +            bb.error("Failed to add '%s' to sysroot" % cmd)
> +            sys.exit(14)
> +
> +    return bindir
>
>  def register_commands(subparsers):
>      parser_create = subparsers.add_parser('create',
> --
> 2.30.2
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#165323): https://lists.openembedded.org/g/openembedded-core/message/165323
> Mute This Topic: https://lists.openembedded.org/mt/90928682/1686489
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>

Re: [OE-core] [PATCH 4/5] recipetool: add go recipe generator

[Alexander Kanavin]

This is a lot of code. Can you add some documentation for it, what it
does and how it works? If someone would want to understand it, how
would they go about it?

Alex

On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
<stefan.herbrechtsmeier-oss@weidmueller.com> wrote:
>
> From: Lukas Funke <lukas.funke@weidmueller.com>
>
> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
> ---
>
>  scripts/lib/recipetool/create_go.py | 394 ++++++++++++++++++++++++++++
>  1 file changed, 394 insertions(+)
>  create mode 100644 scripts/lib/recipetool/create_go.py
>
> diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
> new file mode 100644
> index 0000000000..4552e9b470
> --- /dev/null
> +++ b/scripts/lib/recipetool/create_go.py
> @@ -0,0 +1,394 @@
> +# Recipe creation tool - go support plugin
> +#
> +# Copyright (C) 2022 Weidmueller GmbH & Co KG
> +# Author: Lukas Funke <lukas.funke@weidmueller.com>
> +#
> +# Copyright (c) 2009 The Go Authors. All rights reserved.
> +#
> +#  SPDX-License-Identifier: BSD-3-Clause AND GPL-2.0-only
> +#
> +import bb.utils
> +from collections import namedtuple
> +from enum import Enum
> +from html.parser import HTMLParser
> +import json
> +import logging
> +import os
> +import re
> +import subprocess
> +import sys
> +import tempfile
> +import shutil
> +from urllib.error import URLError, HTTPError
> +import urllib.parse
> +import urllib.request
> +
> +from recipetool.create import RecipeHandler, handle_license_vars, ensure_native_cmd
> +
> +GoImport = namedtuple('GoImport', 'reporoot vcs repourl suffix')
> +logger = logging.getLogger('recipetool')
> +
> +tinfoil = None
> +
> +re_pseudo_semver = re.compile(r"v([0-9]+)\.([0-9]+).([0-9]+|\([0-9]+\+1\))-(pre\.[0-9]+\.)?([0-9]+\.)?(?P<utc>[0-9]+)-(?P<sha1_abbrev>[0-9Aa-zA-Z]+)")
> +re_semver = re.compile(r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
> +
> +def tinfoil_init(instance):
> +    global tinfoil
> +    tinfoil = instance
> +
> +class GoRecipeHandler(RecipeHandler):
> +
> +    def _resolve_repository_static(self, modulepath):
> +        _rootpath = None
> +        _vcs = None
> +        _repourl = None
> +        _suffix = None
> +
> +        host, _, path = modulepath.partition('/')
> +
> +        class vcs(Enum):
> +            pathprefix = "pathprefix"
> +            regexp = "regexp"
> +            vcs = "vcs"
> +            repo = "repo"
> +            check = "check"
> +            schemelessRepo = "schemelessRepo"
> +
> +        # GitHub
> +        vcsGitHub = {}
> +        vcsGitHub[vcs.pathprefix] = "github.com"
> +        vcsGitHub[vcs.regexp] = re.compile(r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
> +        vcsGitHub[vcs.vcs] = "git"
> +        vcsGitHub[vcs.repo] = "https://\g<root>"
> +
> +        # Bitbucket
> +        vcsBitbucket = {}
> +        vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
> +        vcsBitbucket[vcs.regexp] = re.compile(r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
> +        vcsBitbucket[vcs.vcs] = "git"
> +        vcsBitbucket[vcs.repo] = "https://\g<root>"
> +
> +        # IBM DevOps Services (JazzHub)
> +        vcsIBMDevOps = {}
> +        vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
> +        vcsIBMDevOps[vcs.regexp] = re.compile(r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
> +        vcsIBMDevOps[vcs.vcs] = "git"
> +        vcsIBMDevOps[vcs.repo] = "https://\g<root>"
> +
> +        # Git at Apache
> +        vcsApacheGit = {}
> +        vcsApacheGit[vcs.pathprefix] = "git.apache.org"
> +        vcsApacheGit[vcs.regexp] = re.compile(r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
> +        vcsApacheGit[vcs.vcs] = "git"
> +        vcsApacheGit[vcs.repo] = "https://\g<root>"
> +
> +        # Git at OpenStack
> +        vcsOpenStackGit = {}
> +        vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
> +        vcsOpenStackGit[vcs.regexp] = re.compile(r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
> +        vcsOpenStackGit[vcs.vcs] = "git"
> +        vcsOpenStackGit[vcs.repo] = "https://\g<root>"
> +
> +        # chiselapp.com for fossil
> +        vcsChiselapp = {}
> +        vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
> +        vcsChiselapp[vcs.regexp] = re.compile(r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
> +        vcsChiselapp[vcs.vcs] = "fossil"
> +        vcsChiselapp[vcs.repo] = "https://\g<root>"
> +
> +        # General syntax for any server.
> +        # Must be last.
> +        vcsGeneralServer = {}
> +        vcsGeneralServer[vcs.regexp] = re.compile("(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\-]+))*$")
> +        vcsGeneralServer[vcs.schemelessRepo] = True
> +
> +        vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, vcsApacheGit, vcsOpenStackGit, vcsChiselapp, vcsGeneralServer]
> +
> +        if modulepath.startswith("example.net") or modulepath == "rsc.io":
> +            logger.warning("Suspicious module path %s" % modulepath)
> +            return None
> +        if modulepath.startswith("http:") or modulepath.startswith("https:"):
> +            logger.warning("Import path should not start with %s %s" % ("http", "https"))
> +            return None
> +
> +        for srv in vcsPaths:
> +            m = srv[vcs.regexp].match(modulepath)
> +            if vcs.pathprefix in srv:
> +                if host == srv[vcs.pathprefix]:
> +                    _rootpath = m.group('root')
> +                    _vcs = srv[vcs.vcs]
> +                    _repourl = m.expand(srv[vcs.repo])
> +                    _suffix = m.group('suffix')
> +                    break
> +            elif m and srv[vcs.schemelessRepo]:
> +                _rootpath = m.group('root')
> +                _vcs = m[vcs.vcs]
> +                _repourl = m[vcs.repo]
> +                _suffix = m.group('suffix')
> +                break
> +
> +        return GoImport(_rootpath, _vcs, _repourl, _suffix)
> +
> +    def _resolve_repository_dynamic(self, modulepath):
> +
> +        url = urllib.parse.urlparse("https://" + modulepath)
> +
> +        class GoImportHTMLParser(HTMLParser):
> +
> +            def __init__(self):
> +                super().__init__()
> +                self.__srv = []
> +
> +            def handle_starttag(self, tag, attrs):
> +                if tag == 'meta' and list(filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)):
> +                    content = list(filter(lambda a: (a[0] == 'content'), attrs))
> +                    if content:
> +                        self.__srv = content[0][1].split()
> +
> +            @property
> +            def rootpath(self):
> +                return self.__srv[0]
> +
> +            @property
> +            def vcs(self):
> +                return self.__srv[1]
> +
> +            @property
> +            def repourl(self):
> +                return self.__srv[2]
> +
> +        req = urllib.request.Request(url.geturl() + "?go-get=1")
> +
> +        try:
> +            resp = urllib.request.urlopen(req)
> +        except URLError as url_err:
> +            logger.error("Error while fetching redirect page: %s", str(url_err))
> +            return None
> +        except HTTPError as http_err:
> +            logger.error("Error while fetching redirect page: %s", str(http_err))
> +            return None
> +
> +        parser = GoImportHTMLParser()
> +        parser.feed(resp.read().decode('utf-8'))
> +        parser.close()
> +
> +        return GoImport(parser.rootpath, parser.vcs, parser.repourl, None)
> +
> +    def _resolve_repository(self, modulepath):
> +        """
> +        Resolves src uri from go module-path
> +        """
> +        repodata = self._resolve_repository_static(modulepath)
> +        if not repodata.repourl:
> +            repodata = self._resolve_repository_dynamic(modulepath)
> +
> +        if repodata:
> +            logger.info("Resolved download path for import '%s' => %s", modulepath, repodata.repourl)
> +
> +        return repodata
> +
> +    def _resolve_pseudo_semver(self, d, repo, module_version):
> +        hash = None
> +
> +        def vcs_fetch_all():
> +            tmpdir = tempfile.mkdtemp()
> +            clone_cmd = "%s clone --bare %s %s" % ('git', repo, tmpdir)
> +            bb.process.run(clone_cmd)
> +            log_cmd = "git log --all --pretty='%H %d' --decorate=short"
> +            output, errors = bb.process.run(log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir)
> +            bb.utils.prunedir(tmpdir)
> +            return output.strip().split('\n')
> +
> +        def vcs_fetch_remote(search=""):
> +            ls_remote_cmd = "git ls-remote --tags {} {}".format(repo, search)
> +            output, errors = bb.process.run(ls_remote_cmd)
> +            return output.strip().split('\n')
> +
> +        m_pseudo_semver = re_pseudo_semver.match(module_version)
> +        if m_pseudo_semver:
> +            remote_refs = vcs_fetch_all()
> +            short_commit = m_pseudo_semver.group('sha1_abbrev')
> +            for l in remote_refs:
> +                r = l.split(maxsplit=1)
> +                sha1 = r[0] if len(r) else None
> +                if not sha1:
> +                    logger.error("Ups: could not resolve abbref commit for %s" % short_commit)
> +
> +                elif sha1.startswith(short_commit):
> +                    hash = sha1
> +                    break
> +        else:
> +            m_semver = re_semver.match(module_version)
> +            if m_semver:
> +
> +                def get_sha1_remote(re, groupId):
> +                    for l in remote_refs:
> +                        r = l.split(maxsplit=1)
> +                        sha1 = r[0] if len(r) else None
> +                        ref = r[1] if len(r) == 2 else None
> +                        if ref:
> +                            m = re.match(ref)
> +                            if m and semver_tag in m.group(groupId).split(','):
> +                                return sha1
> +                    return None
> +
> +                re_tags_remote = re.compile(r"refs/tags/(?P<tag>[0-9A-Za-z-_\.]+)")
> +                re_tags_all = re.compile(r"\((HEAD -> (.*), )?tag: *((?:([0-9A-Za-z-_\.]+),? *)+)\)")
> +                semver_tag = "v" + m_semver.group('major') + "."\
> +                                +m_semver.group('minor') + "."\
> +                                +m_semver.group('patch') \
> +                                +(("-" + m_semver.group('prerelease')) if m_semver.group('prerelease') else "")
> +                remote_refs = vcs_fetch_remote(semver_tag)
> +                # probe tag using 'ls-remote', which is faster than fetching complete history
> +                sha1 = get_sha1_remote(re_tags_remote, 'tag')
> +                if sha1:
> +                    hash = sha1
> +                else:
> +                    # backup: fetch complete history
> +                    remote_refs = vcs_fetch_all()
> +                    hash = get_sha1_remote(re_tags_all, 3)
> +        return hash
> +
> +    def _handle_dependencies(self, d, srctree, go_mod):
> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
> +        src_uris = []
> +        src_revs = []
> +        for require in go_mod['Require']:
> +            module_path = require['Path']
> +            module_version = require['Version']
> +
> +            repodata = self._resolve_repository(module_path)
> +            commit_id = self._resolve_pseudo_semver(d, repodata.repourl, module_version)
> +            url = urllib.parse.urlparse(repodata.repourl)
> +            repo_url = url.netloc + url.path
> +            inline_fcn = "${@go_src_uri("
> +            inline_fcn += "'{}'".format(repo_url)
> +            if repo_url != module_path:
> +                inline_fcn += ",path='{}'".format(module_path)
> +            if repodata.suffix and not re.match("v[0-9]+", repodata.suffix):
> +                inline_fcn += ",subdir='{}'".format(repodata.suffix)
> +            if repodata.vcs != 'git':
> +                inline_fcn += ",vcs='{}'".format(repodata.vcs)
> +            inline_fcn += ")}"
> +
> +            src_uris.append(inline_fcn)
> +            flat_module_path = module_path.replace('/', '.')
> +            src_rev = "# %s@%s => %s\n" % (module_path, module_version, commit_id)
> +            src_rev += "SRCREV_%s = \"%s\"\n" % (flat_module_path, commit_id)
> +            src_rev += "GO_MODULE_PATH[%s] = \"%s\"\n" % (flat_module_path, module_path)
> +            src_rev += "GO_MODULE_VERSION[%s] = \"%s\"" % (flat_module_path, module_version)
> +            src_revs.append(src_rev)
> +
> +        return src_uris, src_revs
> +
> +    def _go_mod_patch(self, patchfile, go_import, srctree, localfilesdir, extravalues, d):
> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
> +        # first remove go.mod and go.sum, otherwise 'go mod init' will fail
> +        bb.utils.remove(os.path.join(srctree, "go.mod"))
> +        bb.utils.remove(os.path.join(srctree, "go.sum"))
> +        bb.process.run("go mod init %s" % go_import, stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
> +        bb.process.run("go mod tidy", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
> +        output, _ = bb.process.run("go mod edit -json", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
> +        bb.process.run("git diff go.mod > %s" % (patchfile), stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
> +        bb.process.run("git checkout HEAD go.mod go.sum;", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
> +        go_mod = json.loads(output)
> +        tmpfile = os.path.join(localfilesdir, patchfile)
> +        shutil.move(os.path.join(srctree, patchfile), tmpfile)
> +        extravalues.setdefault('extrafiles', {})
> +        extravalues['extrafiles'][patchfile] = tmpfile
> +
> +        return go_mod
> +
> +    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
> +
> +        if 'buildsystem' in handled:
> +            return False
> +
> +        files = RecipeHandler.checkfiles(srctree, ['go.mod'])
> +        if not files:
> +            return False
> +
> +        go_bindir = ensure_native_cmd(tinfoil, "go")
> +
> +        d = bb.data.createCopy(tinfoil.config_data)
> +        d.prependVar('PATH', '%s:' % go_bindir)
> +        handled.append('buildsystem')
> +        classes.append("go-vendor")
> +
> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
> +        output, _ = bb.process.run("go mod edit -json", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
> +        go_mod = json.loads(output)
> +
> +        go_import = go_mod['Module']['Path']
> +        go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go'])
> +        go_version_major = int(go_version_match.group(1))
> +        go_version_minor = int(go_version_match.group(2))
> +        src_uris = []
> +        if go_version_major == 1 and go_version_minor < 17:
> +            logger.warning("go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.")
> +            patchfile = "go.mod.patch"
> +            localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
> +            go_mod = self._go_mod_patch(patchfile, go_import, srctree, localfilesdir, extravalues, d)
> +            src_uris.append("file://%s;patchdir=src/${GO_IMPORT}" % (patchfile))
> +
> +        if not os.path.exists(os.path.join(srctree, "vendor")):
> +            dep_src_uris, src_revs = self._handle_dependencies(d, srctree, go_mod)
> +            src_uris.extend(dep_src_uris)
> +            lines_after.append("#TODO: Subdirectories are heuristically derived from " \
> +                              "the import path and might be incorrect.")
> +            for src_rev in src_revs:
> +                lines_after.append(src_rev)
> +
> +        self._rewrite_src_uri(src_uris, lines_before)
> +
> +        handle_license_vars(srctree, lines_before, handled, extravalues, d)
> +        self._rewrite_lic_uri(lines_before)
> +
> +        lines_before.append("GO_IMPORT = \"{}\"".format(go_import))
> +        lines_before.append("SRCREV_FORMAT = \"${PN}\"")
> +
> +    def _update_lines_before(self, updated, newlines, lines_before):
> +        if updated:
> +            del lines_before[:]
> +            for line in newlines:
> +                # Hack to avoid newlines that edit_metadata inserts
> +                if line.endswith('\n'):
> +                    line = line[:-1]
> +                lines_before.append(line)
> +        return updated
> +
> +    def _rewrite_lic_uri(self, lines_before):
> +
> +        def varfunc(varname, origvalue, op, newlines):
> +            if varname == 'LIC_FILES_CHKSUM':
> +                new_licenses = []
> +                licenses = origvalue.split()
> +
> +                for license in licenses:
> +                    uri, chksum = license.split(';', 1)
> +                    url = urllib.parse.urlparse(uri)
> +                    new_uri = os.path.join(url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
> +                    new_licenses.append(new_uri)
> +
> +                return new_licenses, None, -1, True
> +            return origvalue, None, 0, True
> +
> +        updated, newlines = bb.utils.edit_metadata(lines_before, ['LIC_FILES_CHKSUM'], varfunc)
> +        return self._update_lines_before(updated, newlines, lines_before)
> +
> +    def _rewrite_src_uri(self, src_uris_deps, lines_before):
> +
> +        def varfunc(varname, origvalue, op, newlines):
> +            if varname == 'SRC_URI':
> +                src_uri = []
> +                src_uri.append("git://${GO_IMPORT};nobranch=1;name=${PN};protocol=https")
> +                src_uri.extend(src_uris_deps)
> +                return src_uri, None, -1, True
> +            return origvalue, None, 0, True
> +
> +        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
> +        return self._update_lines_before(updated, newlines, lines_before)
> +
> +def register_recipe_handlers(handlers):
> +    handlers.append((GoRecipeHandler(), 60))
> --
> 2.30.2
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#165326): https://lists.openembedded.org/g/openembedded-core/message/165326
> Mute This Topic: https://lists.openembedded.org/mt/90928688/1686489
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>

Re: [OE-core] [PATCH 5/5] oe-selftest: add go recipe create selftest

[Alexander Kanavin]

Also here an explanation of what the test does would be useful.

Please keep in mind: writing code is only half the job. You need to
explain it to non-specialists too, we're already struggling with items
which are understood by few people, or in some cases only one person.

Alex

On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
<stefan.herbrechtsmeier-oss@weidmueller.com> wrote:
>
> From: Lukas Funke <lukas.funke@weidmueller.com>
>
> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
>
> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
> ---
>
>  meta/lib/oeqa/selftest/cases/recipetool.py | 88 ++++++++++++++++++++++
>  1 file changed, 88 insertions(+)
>
> diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py b/meta/lib/oeqa/selftest/cases/recipetool.py
> index 510dae6bad..bb36e7b6d5 100644
> --- a/meta/lib/oeqa/selftest/cases/recipetool.py
> +++ b/meta/lib/oeqa/selftest/cases/recipetool.py
> @@ -530,6 +530,94 @@ class RecipetoolTests(RecipetoolBase):
>          libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'recipetool')
>          sys.path.insert(0, libpath)
>
> +    def test_recipetool_create_go(self):
> +        # Basic test to check go recipe generation
> +        def urifiy(url, version, path = None, subdir = None):
> +            path = ",path='%s'" % path if path else ''
> +            subdir = ",subdir='%s'" % subdir if subdir else ''
> +            return "${@go_src_uri('%s','%s'%s%s)}" % (url, path, subdir)
> +
> +        temprecipe = os.path.join(self.tempdir, 'recipe')
> +        os.makedirs(temprecipe)
> +        recipefile = os.path.join(temprecipe, 'edgex-go_git.bb')
> +        srcuri = 'https://github.com/edgexfoundry/edgex-go.git'
> +        srcrev = "v2.2.0-dev.54"
> +        result = runCmd('recipetool create -o %s %s -S %s' % (temprecipe, srcuri, srcrev))
> +        self.assertTrue(os.path.isfile(recipefile))
> +        checkvars = {}
> +        src_uri = ['git://${GO_IMPORT};nobranch=1;name=${BPN}']
> +        checkvars['LIC_FILES_CHKSUM'] = set(['file://src/${GO_IMPORT}/LICENSE;md5=71a6955f3cd81a809549da266346dc59'])
> +        checkvars['GO_IMPORT'] = "github.com/edgexfoundry/edgex-go"
> +        inherits = ['go-vendor']
> +        dependencies = \
> +            [
> +                ('bitbucket.org/bertimus9/systemstat'),
> +                ('github.com/edgexfoundry/go-mod-bootstrap','github.com/edgexfoundry/go-mod-bootstrap/v2'),
> +                ('github.com/edgexfoundry/go-mod-core-contracts''github.com/edgexfoundry/go-mod-core-contracts/v2'),
> +                ('github.com/edgexfoundry/go-mod-messaging','github.com/edgexfoundry/go-mod-messaging/v2'),
> +                ('github.com/edgexfoundry/go-mod-registry','github.com/edgexfoundry/go-mod-registry/v2'),
> +                ('github.com/edgexfoundry/go-mod-secrets','github.com/edgexfoundry/go-mod-secrets/v2'),
> +                ('github.com/fxamacker/cbor','github.com/fxamacker/cbor/v2'),
> +                ('github.com/golang-jwt/jwt','github.com/golang-jwt/jwt/v4'),
> +                ('github.com/gomodule/redigo'),
> +                ('github.com/google/uuid'),
> +                ('github.com/gorilla/mux'),
> +                ('github.com/lib/pq'),
> +                ('github.com/pelletier/go-toml'),
> +                ('github.com/spiffe/go-spiffe','github.com/spiffe/go-spiffe/v2'),
> +                ('github.com/stretchr/testify'),
> +                ('go.googlesource.com/crypto','golang.org/x/crypto'),
> +                ('gopkg.in/eapache/queue.v1'),
> +                ('gopkg.in/yaml.v3'),
> +                ('github.com/armon/go-metrics'),
> +                ('github.com/cenkalti/backoff'),
> +                ('github.com/davecgh/go-spew'),
> +                ('github.com/eclipse/paho.mqtt.golang'),
> +                ('github.com/edgexfoundry/go-mod-configuration','github.com/edgexfoundry/go-mod-configuration/v2'),
> +                ('github.com/fatih/color'),
> +                ('github.com/go-kit/log'),
> +                ('github.com/go-logfmt/logfmt'),
> +                ('github.com/go-playground/locales'),
> +                ('github.com/go-playground/universal-translator'),
> +                ('github.com/go-playground/validator','github.com/go-playground/validator/v10'),
> +                ('github.com/go-redis/redis','github.com/go-redis/redis/v7'),
> +                ('github.com/golang/protobuf'),
> +                ('github.com/gorilla/websocket'),
> +                ('github.com/hashicorp/consul','api'),
> +                ('github.com/hashicorp/errwrap'),
> +                ('github.com/hashicorp/go-cleanhttp'),
> +                ('github.com/hashicorp/go-hclog'),
> +                ('github.com/hashicorp/go-immutable-radix'),
> +                ('github.com/hashicorp/go-multierror'),
> +                ('github.com/hashicorp/go-rootcerts'),
> +                ('github.com/hashicorp/golang-lru'),
> +                ('github.com/hashicorp/serf'),
> +                ('github.com/leodido/go-urn'),
> +                ('github.com/mattn/go-colorable'),
> +                ('github.com/mattn/go-isatty'),
> +                ('github.com/mitchellh/consulstructure'),
> +                ('github.com/mitchellh/copystructure'),
> +                ('github.com/mitchellh/go-homedir'),
> +                ('github.com/mitchellh/mapstructure'),
> +                ('github.com/mitchellh/reflectwalk'),
> +                ('github.com/pebbe/zmq4'),
> +                ('github.com/pmezard/go-difflib'),
> +                ('github.com/stretchr/objx'),
> +                ('github.com/x448/float16'),
> +                ('github.com/zeebo/errs'),
> +                ('go.googlesource.com/net','golang.org/x/net'),
> +                ('go.googlesource.com/sys','golang.org/x/sys'),
> +                ('go.googlesource.com/text','golang.org/x/text'),
> +                ('github.com/googleapis/go-genproto','google.golang.org/genproto'),
> +                ('github.com/grpc/grpc-go','google.golang.org/grpc'),
> +                ('go.googlesource.com/protobuf','google.golang.org/protobuf'),
> +                ('gopkg.in/square/go-jose.v2'),
> +            ]
> +        src_uri = {urifiy(*d) for d in dependencies}
> +        checkvars['SRC_URI'] = set(src_uri)
> +        self.maxDiff = None
> +        self._test_recipe_contents(recipefile, checkvars, inherits)
> +
>      def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths):
>          dstdir = basedstdir
>          self.assertTrue(os.path.exists(dstdir))
> --
> 2.30.2
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#165327): https://lists.openembedded.org/g/openembedded-core/message/165327
> Mute This Topic: https://lists.openembedded.org/mt/90928692/1686489
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>

Re: [OE-core] [PATCH 1/5] recipetool-create: add ensure_native_cmd function

[Luca Ceresoli]

Hello Stefan,

Il giorno Fri,  6 May 2022 08:59:13 +0200
"Stefan Herbrechtsmeier" <stefan.herbrechtsmeier-oss@weidmueller.com>
ha scritto:

> From: Lukas Funke <lukas.funke@weidmueller.com>
> 
> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
> Signed-off-by: Stefan Herbrechtsmeier
> <stefan.herbrechtsmeier@weidmueller.com>

Testing builds with your series trigger many build failures related to
recipetool. Can you check these logs?

https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/3557/steps/15/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/3506/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/3527/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/3566/steps/14/logs/stdio



-- 
Luca Ceresoli, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

Re: [OE-core] [PATCH 1/5] recipetool-create: add ensure_native_cmd function

[Stefan Herbrechtsmeier]

Hi Luca,

Lukas will check the logs and fix the problem.

Regards
   Stefan

Am 09.05.2022 um 23:46 schrieb Luca Ceresoli:
> Hello Stefan,
> 
> Il giorno Fri,  6 May 2022 08:59:13 +0200
> "Stefan Herbrechtsmeier" <stefan.herbrechtsmeier-oss@weidmueller.com>
> ha scritto:
> 
>> From: Lukas Funke <lukas.funke@weidmueller.com>
>>
>> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
>> Signed-off-by: Stefan Herbrechtsmeier
>> <stefan.herbrechtsmeier@weidmueller.com>
> 
> Testing builds with your series trigger many build failures related to
> recipetool. Can you check these logs?
> 
> https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/3557/steps/15/logs/stdio
> https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/3506/steps/14/logs/stdio
> https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/3527/steps/14/logs/stdio
> https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/3566/steps/14/logs/stdio
> 
> 
> 

Re: [OE-core] [PATCH 1/5] recipetool-create: add ensure_native_cmd function

[Mark Asselstine]

On 2022-05-06 03:09, Alexander Kanavin wrote:
> I think the terminology is a bit confused here, instead of 'command'
> it should say 'recipe'. Also:
> """Check if the command is available in the recipes""" is not
> everything it does, it also builds the recipe.
> 
> And please write a commit message that explains what and why.

Additionally, I would squash commits 1/5 and 2/5. Most people, including 
myself, will review the commits in order, in doing so commit 1/5 is 
reviewed as original work, only to find in 2/5 that it was an existing 
function, just made generic. Combining these also makes it easier when 
mining history.

MarkA


> 
> Alex
> 
> On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
> <stefan.herbrechtsmeier-oss@weidmueller.com> wrote:
>>
>> From: Lukas Funke <lukas.funke@weidmueller.com>
>>
>> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
>> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
>> ---
>>
>>   scripts/lib/recipetool/create.py | 23 +++++++++++++++++++++++
>>   1 file changed, 23 insertions(+)
>>
>> diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
>> index 824ac6350d..efcb82173e 100644
>> --- a/scripts/lib/recipetool/create.py
>> +++ b/scripts/lib/recipetool/create.py
>> @@ -1375,6 +1375,29 @@ def convert_rpm_xml(xmlfile):
>>                       values[varname] = child[0].text
>>       return values
>>
>> +def ensure_native_cmd(tinfoil, cmd):
>> +    """Check if the command is available in the recipes"""
>> +    if not tinfoil.recipes_parsed:
>> +        tinfoil.parse_recipes()
>> +
>> +    try:
>> +        d = tinfoil.parse_recipe("%s-native" % cmd)
>> +    except bb.providers.NoProvider:
>> +        bb.error("Nothing provides '%s-native' which is required for the build" % cmd)
>> +        bb.note("You will likely need to add a layer that provides %s" % cmd)
>> +        sys.exit(14)
>> +
>> +    bindir = d.getVar("STAGING_BINDIR_NATIVE")
>> +    cmdpath = os.path.join(bindir, cmd)
>> +
>> +    if not os.path.exists(cmdpath):
>> +        tinfoil.build_targets("%s-native" % cmd, "addto_recipe_sysroot")
>> +
>> +        if not os.path.exists(cmdpath):
>> +            bb.error("Failed to add '%s' to sysroot" % cmd)
>> +            sys.exit(14)
>> +
>> +    return bindir
>>
>>   def register_commands(subparsers):
>>       parser_create = subparsers.add_parser('create',
>> --
>> 2.30.2
>>
>>
>>
>>
>>
>>
>> -=-=-=-=-=-=-=-=-=-=-=-
>> Links: You receive all messages sent to this group.
>> View/Reply Online (#165330): https://lists.openembedded.org/g/openembedded-core/message/165330
>> Mute This Topic: https://lists.openembedded.org/mt/90928682/3616946
>> Group Owner: openembedded-core+owner@lists.openembedded.org
>> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [mark.asselstine@windriver.com]
>> -=-=-=-=-=-=-=-=-=-=-=-
>>

Re: [OE-core] [PATCH 1/5] recipetool-create: add ensure_native_cmd function

[Mark Asselstine]

On 2022-05-11 15:45, Mark Asselstine wrote:
> 
> 
> On 2022-05-06 03:09, Alexander Kanavin wrote:
>> I think the terminology is a bit confused here, instead of 'command'
>> it should say 'recipe'. Also:
>> """Check if the command is available in the recipes""" is not
>> everything it does, it also builds the recipe.
>>
>> And please write a commit message that explains what and why.
> 
> Additionally, I would squash commits 1/5 and 2/5. Most people, including 
> myself, will review the commits in order, in doing so commit 1/5 is 
> reviewed as original work, only to find in 2/5 that it was an existing 
> function, just made generic. Combining these also makes it easier when 
> mining history.
> 

I suspect this will need rework as the assumption that a command and 
package name are 1:1 might have only been valid with NPM but definitely 
won't be the case for many commands.

MarkA


> MarkA
> 
> 
>>
>> Alex
>>
>> On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
>> <stefan.herbrechtsmeier-oss@weidmueller.com> wrote:
>>>
>>> From: Lukas Funke <lukas.funke@weidmueller.com>
>>>
>>> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
>>> Signed-off-by: Stefan Herbrechtsmeier 
>>> <stefan.herbrechtsmeier@weidmueller.com>
>>> ---
>>>
>>>   scripts/lib/recipetool/create.py | 23 +++++++++++++++++++++++
>>>   1 file changed, 23 insertions(+)
>>>
>>> diff --git a/scripts/lib/recipetool/create.py 
>>> b/scripts/lib/recipetool/create.py
>>> index 824ac6350d..efcb82173e 100644
>>> --- a/scripts/lib/recipetool/create.py
>>> +++ b/scripts/lib/recipetool/create.py
>>> @@ -1375,6 +1375,29 @@ def convert_rpm_xml(xmlfile):
>>>                       values[varname] = child[0].text
>>>       return values
>>>
>>> +def ensure_native_cmd(tinfoil, cmd):
>>> +    """Check if the command is available in the recipes"""
>>> +    if not tinfoil.recipes_parsed:
>>> +        tinfoil.parse_recipes()
>>> +
>>> +    try:
>>> +        d = tinfoil.parse_recipe("%s-native" % cmd)
>>> +    except bb.providers.NoProvider:
>>> +        bb.error("Nothing provides '%s-native' which is required for 
>>> the build" % cmd)
>>> +        bb.note("You will likely need to add a layer that provides 
>>> %s" % cmd)
>>> +        sys.exit(14)
>>> +
>>> +    bindir = d.getVar("STAGING_BINDIR_NATIVE")
>>> +    cmdpath = os.path.join(bindir, cmd)
>>> +
>>> +    if not os.path.exists(cmdpath):
>>> +        tinfoil.build_targets("%s-native" % cmd, 
>>> "addto_recipe_sysroot")
>>> +
>>> +        if not os.path.exists(cmdpath):
>>> +            bb.error("Failed to add '%s' to sysroot" % cmd)
>>> +            sys.exit(14)
>>> +
>>> +    return bindir
>>>
>>>   def register_commands(subparsers):
>>>       parser_create = subparsers.add_parser('create',
>>> -- 
>>> 2.30.2
>>>
>>>
>>>
>>>
>>>
>>>
>>> -=-=-=-=-=-=-=-=-=-=-=-
>>> Links: You receive all messages sent to this group.
>>> View/Reply Online (#165330): 
>>> https://lists.openembedded.org/g/openembedded-core/message/165330
>>> Mute This Topic: https://lists.openembedded.org/mt/90928682/3616946
>>> Group Owner: openembedded-core+owner@lists.openembedded.org
>>> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
>>> [mark.asselstine@windriver.com]
>>> -=-=-=-=-=-=-=-=-=-=-=-
>>>

Re: [OE-core] [PATCH 5/5] oe-selftest: add go recipe create selftest

[Mark Asselstine]

On 2022-05-06 03:16, Alexander Kanavin wrote:
> Also here an explanation of what the test does would be useful.
> 
> Please keep in mind: writing code is only half the job. You need to
> explain it to non-specialists too, we're already struggling with items
> which are understood by few people, or in some cases only one person.
> 
> Alex
> 
> On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
> <stefan.herbrechtsmeier-oss@weidmueller.com> wrote:
>>
>> From: Lukas Funke <lukas.funke@weidmueller.com>
>>
>> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
>>
>> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
>> ---
>>
>>   meta/lib/oeqa/selftest/cases/recipetool.py | 88 ++++++++++++++++++++++
>>   1 file changed, 88 insertions(+)
>>
>> diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py b/meta/lib/oeqa/selftest/cases/recipetool.py
>> index 510dae6bad..bb36e7b6d5 100644
>> --- a/meta/lib/oeqa/selftest/cases/recipetool.py
>> +++ b/meta/lib/oeqa/selftest/cases/recipetool.py
>> @@ -530,6 +530,94 @@ class RecipetoolTests(RecipetoolBase):
>>           libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'recipetool')
>>           sys.path.insert(0, libpath)
>>
>> +    def test_recipetool_create_go(self):
>> +        # Basic test to check go recipe generation
>> +        def urifiy(url, version, path = None, subdir = None):
>> +            path = ",path='%s'" % path if path else ''
>> +            subdir = ",subdir='%s'" % subdir if subdir else ''
>> +            return "${@go_src_uri('%s','%s'%s%s)}" % (url, path, subdir)
>> +
>> +        temprecipe = os.path.join(self.tempdir, 'recipe')
>> +        os.makedirs(temprecipe)

I hate anything that has the potential to generate a python splat 
existing. At any rate you have matched what the other tests do so no 
change required (maybe the test framework catches splats, I am not 
familiar enough with the test framework to know). Typically I would want 
to see this call in a try block.

>> +        recipefile = os.path.join(temprecipe, 'edgex-go_git.bb')
>> +        srcuri = 'https://github.com/edgexfoundry/edgex-go.git'
>> +        srcrev = "v2.2.0-dev.54"
>> +        result = runCmd('recipetool create -o %s %s -S %s' % (temprecipe, srcuri, srcrev))
>> +        self.assertTrue(os.path.isfile(recipefile))
>> +        checkvars = {}
>> +        src_uri = ['git://${GO_IMPORT};nobranch=1;name=${BPN}']
>> +        checkvars['LIC_FILES_CHKSUM'] = set(['file://src/${GO_IMPORT}/LICENSE;md5=71a6955f3cd81a809549da266346dc59'])
>> +        checkvars['GO_IMPORT'] = "github.com/edgexfoundry/edgex-go"
>> +        inherits = ['go-vendor']
>> +        dependencies = \
>> +            [
>> +                ('bitbucket.org/bertimus9/systemstat'),
>> +                ('github.com/edgexfoundry/go-mod-bootstrap','github.com/edgexfoundry/go-mod-bootstrap/v2'),
>> +                ('github.com/edgexfoundry/go-mod-core-contracts''github.com/edgexfoundry/go-mod-core-contracts/v2'),
>> +                ('github.com/edgexfoundry/go-mod-messaging','github.com/edgexfoundry/go-mod-messaging/v2'),
>> +                ('github.com/edgexfoundry/go-mod-registry','github.com/edgexfoundry/go-mod-registry/v2'),
>> +                ('github.com/edgexfoundry/go-mod-secrets','github.com/edgexfoundry/go-mod-secrets/v2'),
>> +                ('github.com/fxamacker/cbor','github.com/fxamacker/cbor/v2'),
>> +                ('github.com/golang-jwt/jwt','github.com/golang-jwt/jwt/v4'),
>> +                ('github.com/gomodule/redigo'),
>> +                ('github.com/google/uuid'),
>> +                ('github.com/gorilla/mux'),
>> +                ('github.com/lib/pq'),
>> +                ('github.com/pelletier/go-toml'),
>> +                ('github.com/spiffe/go-spiffe','github.com/spiffe/go-spiffe/v2'),
>> +                ('github.com/stretchr/testify'),
>> +                ('go.googlesource.com/crypto','golang.org/x/crypto'),
>> +                ('gopkg.in/eapache/queue.v1'),
>> +                ('gopkg.in/yaml.v3'),
>> +                ('github.com/armon/go-metrics'),
>> +                ('github.com/cenkalti/backoff'),
>> +                ('github.com/davecgh/go-spew'),
>> +                ('github.com/eclipse/paho.mqtt.golang'),
>> +                ('github.com/edgexfoundry/go-mod-configuration','github.com/edgexfoundry/go-mod-configuration/v2'),
>> +                ('github.com/fatih/color'),
>> +                ('github.com/go-kit/log'),
>> +                ('github.com/go-logfmt/logfmt'),
>> +                ('github.com/go-playground/locales'),
>> +                ('github.com/go-playground/universal-translator'),
>> +                ('github.com/go-playground/validator','github.com/go-playground/validator/v10'),
>> +                ('github.com/go-redis/redis','github.com/go-redis/redis/v7'),
>> +                ('github.com/golang/protobuf'),
>> +                ('github.com/gorilla/websocket'),
>> +                ('github.com/hashicorp/consul','api'),
>> +                ('github.com/hashicorp/errwrap'),
>> +                ('github.com/hashicorp/go-cleanhttp'),
>> +                ('github.com/hashicorp/go-hclog'),
>> +                ('github.com/hashicorp/go-immutable-radix'),
>> +                ('github.com/hashicorp/go-multierror'),
>> +                ('github.com/hashicorp/go-rootcerts'),
>> +                ('github.com/hashicorp/golang-lru'),
>> +                ('github.com/hashicorp/serf'),
>> +                ('github.com/leodido/go-urn'),
>> +                ('github.com/mattn/go-colorable'),
>> +                ('github.com/mattn/go-isatty'),
>> +                ('github.com/mitchellh/consulstructure'),
>> +                ('github.com/mitchellh/copystructure'),
>> +                ('github.com/mitchellh/go-homedir'),
>> +                ('github.com/mitchellh/mapstructure'),
>> +                ('github.com/mitchellh/reflectwalk'),
>> +                ('github.com/pebbe/zmq4'),
>> +                ('github.com/pmezard/go-difflib'),
>> +                ('github.com/stretchr/objx'),
>> +                ('github.com/x448/float16'),
>> +                ('github.com/zeebo/errs'),
>> +                ('go.googlesource.com/net','golang.org/x/net'),
>> +                ('go.googlesource.com/sys','golang.org/x/sys'),
>> +                ('go.googlesource.com/text','golang.org/x/text'),
>> +                ('github.com/googleapis/go-genproto','google.golang.org/genproto'),
>> +                ('github.com/grpc/grpc-go','google.golang.org/grpc'),
>> +                ('go.googlesource.com/protobuf','google.golang.org/protobuf'),
>> +                ('gopkg.in/square/go-jose.v2'),
>> +            ]
>> +        src_uri = {urifiy(*d) for d in dependencies}
>> +        checkvars['SRC_URI'] = set(src_uri)
>> +        self.maxDiff = None
>> +        self._test_recipe_contents(recipefile, checkvars, inherits)
>> +

Trailing whitespace.

MarkA

>>       def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths):
>>           dstdir = basedstdir
>>           self.assertTrue(os.path.exists(dstdir))
>> --
>> 2.30.2
>>
>>
>>
>>
>>
>>
>> -=-=-=-=-=-=-=-=-=-=-=-
>> Links: You receive all messages sent to this group.
>> View/Reply Online (#165332): https://lists.openembedded.org/g/openembedded-core/message/165332
>> Mute This Topic: https://lists.openembedded.org/mt/90928692/3616946
>> Group Owner: openembedded-core+owner@lists.openembedded.org
>> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [mark.asselstine@windriver.com]
>> -=-=-=-=-=-=-=-=-=-=-=-
>>

Re: [OE-core] [PATCH 4/5] recipetool: add go recipe generator

[Mark Asselstine]

On 2022-05-06 03:15, Alexander Kanavin wrote:
> This is a lot of code. Can you add some documentation for it, what it
> does and how it works? If someone would want to understand it, how
> would they go about it?
> 
> Alex
> 
> On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
> <stefan.herbrechtsmeier-oss@weidmueller.com> wrote:
>>
>> From: Lukas Funke <lukas.funke@weidmueller.com>
>>
>> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
>> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
>> ---
>>
>>   scripts/lib/recipetool/create_go.py | 394 ++++++++++++++++++++++++++++
>>   1 file changed, 394 insertions(+)
>>   create mode 100644 scripts/lib/recipetool/create_go.py
>>
>> diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
>> new file mode 100644
>> index 0000000000..4552e9b470
>> --- /dev/null
>> +++ b/scripts/lib/recipetool/create_go.py
>> @@ -0,0 +1,394 @@
>> +# Recipe creation tool - go support plugin
>> +#
>> +# Copyright (C) 2022 Weidmueller GmbH & Co KG
>> +# Author: Lukas Funke <lukas.funke@weidmueller.com>
>> +#
>> +# Copyright (c) 2009 The Go Authors. All rights reserved.
>> +#
>> +#  SPDX-License-Identifier: BSD-3-Clause AND GPL-2.0-only
>> +#
>> +import bb.utils
>> +from collections import namedtuple
>> +from enum import Enum
>> +from html.parser import HTMLParser
>> +import json
>> +import logging
>> +import os
>> +import re
>> +import subprocess
>> +import sys
>> +import tempfile
>> +import shutil
>> +from urllib.error import URLError, HTTPError
>> +import urllib.parse
>> +import urllib.request
>> +
>> +from recipetool.create import RecipeHandler, handle_license_vars, ensure_native_cmd
>> +
>> +GoImport = namedtuple('GoImport', 'reporoot vcs repourl suffix')
>> +logger = logging.getLogger('recipetool')
>> +
>> +tinfoil = None
>> +
>> +re_pseudo_semver = re.compile(r"v([0-9]+)\.([0-9]+).([0-9]+|\([0-9]+\+1\))-(pre\.[0-9]+\.)?([0-9]+\.)?(?P<utc>[0-9]+)-(?P<sha1_abbrev>[0-9Aa-zA-Z]+)")
>> +re_semver = re.compile(r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
>> +
>> +def tinfoil_init(instance):
>> +    global tinfoil
>> +    tinfoil = instance
>> +
>> +class GoRecipeHandler(RecipeHandler):
>> +
>> +    def _resolve_repository_static(self, modulepath):
>> +        _rootpath = None
>> +        _vcs = None
>> +        _repourl = None
>> +        _suffix = None
>> +
>> +        host, _, path = modulepath.partition('/')
>> +
>> +        class vcs(Enum):
>> +            pathprefix = "pathprefix"
>> +            regexp = "regexp"
>> +            vcs = "vcs"
>> +            repo = "repo"
>> +            check = "check"
>> +            schemelessRepo = "schemelessRepo"
>> +
>> +        # GitHub
>> +        vcsGitHub = {}
>> +        vcsGitHub[vcs.pathprefix] = "github.com"
>> +        vcsGitHub[vcs.regexp] = re.compile(r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsGitHub[vcs.vcs] = "git"
>> +        vcsGitHub[vcs.repo] = "https://\g<root>"
>> +
>> +        # Bitbucket
>> +        vcsBitbucket = {}
>> +        vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
>> +        vcsBitbucket[vcs.regexp] = re.compile(r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsBitbucket[vcs.vcs] = "git"
>> +        vcsBitbucket[vcs.repo] = "https://\g<root>"
>> +
>> +        # IBM DevOps Services (JazzHub)
>> +        vcsIBMDevOps = {}
>> +        vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
>> +        vcsIBMDevOps[vcs.regexp] = re.compile(r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsIBMDevOps[vcs.vcs] = "git"
>> +        vcsIBMDevOps[vcs.repo] = "https://\g<root>"
>> +
>> +        # Git at Apache
>> +        vcsApacheGit = {}
>> +        vcsApacheGit[vcs.pathprefix] = "git.apache.org"
>> +        vcsApacheGit[vcs.regexp] = re.compile(r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsApacheGit[vcs.vcs] = "git"
>> +        vcsApacheGit[vcs.repo] = "https://\g<root>"
>> +
>> +        # Git at OpenStack
>> +        vcsOpenStackGit = {}
>> +        vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
>> +        vcsOpenStackGit[vcs.regexp] = re.compile(r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsOpenStackGit[vcs.vcs] = "git"
>> +        vcsOpenStackGit[vcs.repo] = "https://\g<root>"
>> +
>> +        # chiselapp.com for fossil
>> +        vcsChiselapp = {}
>> +        vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
>> +        vcsChiselapp[vcs.regexp] = re.compile(r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
>> +        vcsChiselapp[vcs.vcs] = "fossil"
>> +        vcsChiselapp[vcs.repo] = "https://\g<root>"
>> +
>> +        # General syntax for any server.
>> +        # Must be last.
>> +        vcsGeneralServer = {}
>> +        vcsGeneralServer[vcs.regexp] = re.compile("(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\-]+))*$")
>> +        vcsGeneralServer[vcs.schemelessRepo] = True
>> +
>> +        vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, vcsApacheGit, vcsOpenStackGit, vcsChiselapp, vcsGeneralServer]
>> +
>> +        if modulepath.startswith("example.net") or modulepath == "rsc.io":
>> +            logger.warning("Suspicious module path %s" % modulepath)
>> +            return None
>> +        if modulepath.startswith("http:") or modulepath.startswith("https:"):
>> +            logger.warning("Import path should not start with %s %s" % ("http", "https"))
>> +            return None
>> +
>> +        for srv in vcsPaths:
>> +            m = srv[vcs.regexp].match(modulepath)
>> +            if vcs.pathprefix in srv:
>> +                if host == srv[vcs.pathprefix]:
>> +                    _rootpath = m.group('root')
>> +                    _vcs = srv[vcs.vcs]
>> +                    _repourl = m.expand(srv[vcs.repo])
>> +                    _suffix = m.group('suffix')
>> +                    break
>> +            elif m and srv[vcs.schemelessRepo]:
>> +                _rootpath = m.group('root')
>> +                _vcs = m[vcs.vcs]
>> +                _repourl = m[vcs.repo]
>> +                _suffix = m.group('suffix')
>> +                break
>> +
>> +        return GoImport(_rootpath, _vcs, _repourl, _suffix)
>> +
>> +    def _resolve_repository_dynamic(self, modulepath):
>> +
>> +        url = urllib.parse.urlparse("https://" + modulepath)
>> +
>> +        class GoImportHTMLParser(HTMLParser):
>> +
>> +            def __init__(self):
>> +                super().__init__()
>> +                self.__srv = []
>> +
>> +            def handle_starttag(self, tag, attrs):
>> +                if tag == 'meta' and list(filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)):
>> +                    content = list(filter(lambda a: (a[0] == 'content'), attrs))
>> +                    if content:
>> +                        self.__srv = content[0][1].split()
>> +
>> +            @property
>> +            def rootpath(self):
>> +                return self.__srv[0]
>> +
>> +            @property
>> +            def vcs(self):
>> +                return self.__srv[1]
>> +
>> +            @property
>> +            def repourl(self):
>> +                return self.__srv[2]
>> +
>> +        req = urllib.request.Request(url.geturl() + "?go-get=1")
>> +
>> +        try:
>> +            resp = urllib.request.urlopen(req)
>> +        except URLError as url_err:
>> +            logger.error("Error while fetching redirect page: %s", str(url_err))
>> +            return None
>> +        except HTTPError as http_err:
>> +            logger.error("Error while fetching redirect page: %s", str(http_err))
>> +            return None
>> +
>> +        parser = GoImportHTMLParser()
>> +        parser.feed(resp.read().decode('utf-8'))
>> +        parser.close()
>> +
>> +        return GoImport(parser.rootpath, parser.vcs, parser.repourl, None)

Attempting this on minio client

recipetool create -o ./maa https://github.com/minio/mc \
   -S b5a0640899f8f8653bcacd19791c92ca22066ba3

I am seeing

   File "/home/mark/git/poky/scripts/lib/recipetool/create_go.py", line 
151, in rootpath
     return self.__srv[0]
IndexError: list index out of range

Is more error checking required?

MarkA

>> +
>> +    def _resolve_repository(self, modulepath):
>> +        """
>> +        Resolves src uri from go module-path
>> +        """
>> +        repodata = self._resolve_repository_static(modulepath)
>> +        if not repodata.repourl:
>> +            repodata = self._resolve_repository_dynamic(modulepath)
>> +
>> +        if repodata:
>> +            logger.info("Resolved download path for import '%s' => %s", modulepath, repodata.repourl)
>> +
>> +        return repodata
>> +
>> +    def _resolve_pseudo_semver(self, d, repo, module_version):
>> +        hash = None
>> +
>> +        def vcs_fetch_all():
>> +            tmpdir = tempfile.mkdtemp()
>> +            clone_cmd = "%s clone --bare %s %s" % ('git', repo, tmpdir)
>> +            bb.process.run(clone_cmd)
>> +            log_cmd = "git log --all --pretty='%H %d' --decorate=short"
>> +            output, errors = bb.process.run(log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir)
>> +            bb.utils.prunedir(tmpdir)
>> +            return output.strip().split('\n')
>> +
>> +        def vcs_fetch_remote(search=""):
>> +            ls_remote_cmd = "git ls-remote --tags {} {}".format(repo, search)
>> +            output, errors = bb.process.run(ls_remote_cmd)
>> +            return output.strip().split('\n')
>> +
>> +        m_pseudo_semver = re_pseudo_semver.match(module_version)
>> +        if m_pseudo_semver:
>> +            remote_refs = vcs_fetch_all()
>> +            short_commit = m_pseudo_semver.group('sha1_abbrev')
>> +            for l in remote_refs:
>> +                r = l.split(maxsplit=1)
>> +                sha1 = r[0] if len(r) else None
>> +                if not sha1:
>> +                    logger.error("Ups: could not resolve abbref commit for %s" % short_commit)
>> +
>> +                elif sha1.startswith(short_commit):
>> +                    hash = sha1
>> +                    break
>> +        else:
>> +            m_semver = re_semver.match(module_version)
>> +            if m_semver:
>> +
>> +                def get_sha1_remote(re, groupId):
>> +                    for l in remote_refs:
>> +                        r = l.split(maxsplit=1)
>> +                        sha1 = r[0] if len(r) else None
>> +                        ref = r[1] if len(r) == 2 else None
>> +                        if ref:
>> +                            m = re.match(ref)
>> +                            if m and semver_tag in m.group(groupId).split(','):
>> +                                return sha1
>> +                    return None
>> +
>> +                re_tags_remote = re.compile(r"refs/tags/(?P<tag>[0-9A-Za-z-_\.]+)")
>> +                re_tags_all = re.compile(r"\((HEAD -> (.*), )?tag: *((?:([0-9A-Za-z-_\.]+),? *)+)\)")
>> +                semver_tag = "v" + m_semver.group('major') + "."\
>> +                                +m_semver.group('minor') + "."\
>> +                                +m_semver.group('patch') \
>> +                                +(("-" + m_semver.group('prerelease')) if m_semver.group('prerelease') else "")
>> +                remote_refs = vcs_fetch_remote(semver_tag)
>> +                # probe tag using 'ls-remote', which is faster than fetching complete history
>> +                sha1 = get_sha1_remote(re_tags_remote, 'tag')
>> +                if sha1:
>> +                    hash = sha1
>> +                else:
>> +                    # backup: fetch complete history
>> +                    remote_refs = vcs_fetch_all()
>> +                    hash = get_sha1_remote(re_tags_all, 3)
>> +        return hash
>> +
>> +    def _handle_dependencies(self, d, srctree, go_mod):
>> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
>> +        src_uris = []
>> +        src_revs = []
>> +        for require in go_mod['Require']:
>> +            module_path = require['Path']
>> +            module_version = require['Version']
>> +
>> +            repodata = self._resolve_repository(module_path)
>> +            commit_id = self._resolve_pseudo_semver(d, repodata.repourl, module_version)
>> +            url = urllib.parse.urlparse(repodata.repourl)
>> +            repo_url = url.netloc + url.path
>> +            inline_fcn = "${@go_src_uri("
>> +            inline_fcn += "'{}'".format(repo_url)
>> +            if repo_url != module_path:
>> +                inline_fcn += ",path='{}'".format(module_path)
>> +            if repodata.suffix and not re.match("v[0-9]+", repodata.suffix):
>> +                inline_fcn += ",subdir='{}'".format(repodata.suffix)
>> +            if repodata.vcs != 'git':
>> +                inline_fcn += ",vcs='{}'".format(repodata.vcs)
>> +            inline_fcn += ")}"
>> +
>> +            src_uris.append(inline_fcn)
>> +            flat_module_path = module_path.replace('/', '.')
>> +            src_rev = "# %s@%s => %s\n" % (module_path, module_version, commit_id)
>> +            src_rev += "SRCREV_%s = \"%s\"\n" % (flat_module_path, commit_id)
>> +            src_rev += "GO_MODULE_PATH[%s] = \"%s\"\n" % (flat_module_path, module_path)
>> +            src_rev += "GO_MODULE_VERSION[%s] = \"%s\"" % (flat_module_path, module_version)
>> +            src_revs.append(src_rev)
>> +
>> +        return src_uris, src_revs
>> +
>> +    def _go_mod_patch(self, patchfile, go_import, srctree, localfilesdir, extravalues, d):
>> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
>> +        # first remove go.mod and go.sum, otherwise 'go mod init' will fail
>> +        bb.utils.remove(os.path.join(srctree, "go.mod"))
>> +        bb.utils.remove(os.path.join(srctree, "go.sum"))
>> +        bb.process.run("go mod init %s" % go_import, stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        bb.process.run("go mod tidy", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        output, _ = bb.process.run("go mod edit -json", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        bb.process.run("git diff go.mod > %s" % (patchfile), stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        bb.process.run("git checkout HEAD go.mod go.sum;", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        go_mod = json.loads(output)
>> +        tmpfile = os.path.join(localfilesdir, patchfile)
>> +        shutil.move(os.path.join(srctree, patchfile), tmpfile)
>> +        extravalues.setdefault('extrafiles', {})
>> +        extravalues['extrafiles'][patchfile] = tmpfile
>> +
>> +        return go_mod
>> +
>> +    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
>> +
>> +        if 'buildsystem' in handled:
>> +            return False
>> +
>> +        files = RecipeHandler.checkfiles(srctree, ['go.mod'])
>> +        if not files:
>> +            return False
>> +
>> +        go_bindir = ensure_native_cmd(tinfoil, "go")
>> +
>> +        d = bb.data.createCopy(tinfoil.config_data)
>> +        d.prependVar('PATH', '%s:' % go_bindir)
>> +        handled.append('buildsystem')
>> +        classes.append("go-vendor")
>> +
>> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
>> +        output, _ = bb.process.run("go mod edit -json", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        go_mod = json.loads(output)
>> +
>> +        go_import = go_mod['Module']['Path']
>> +        go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go'])
>> +        go_version_major = int(go_version_match.group(1))
>> +        go_version_minor = int(go_version_match.group(2))
>> +        src_uris = []
>> +        if go_version_major == 1 and go_version_minor < 17:
>> +            logger.warning("go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.")
>> +            patchfile = "go.mod.patch"
>> +            localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
>> +            go_mod = self._go_mod_patch(patchfile, go_import, srctree, localfilesdir, extravalues, d)
>> +            src_uris.append("file://%s;patchdir=src/${GO_IMPORT}" % (patchfile))
>> +
>> +        if not os.path.exists(os.path.join(srctree, "vendor")):
>> +            dep_src_uris, src_revs = self._handle_dependencies(d, srctree, go_mod)
>> +            src_uris.extend(dep_src_uris)
>> +            lines_after.append("#TODO: Subdirectories are heuristically derived from " \
>> +                              "the import path and might be incorrect.")
>> +            for src_rev in src_revs:
>> +                lines_after.append(src_rev)
>> +
>> +        self._rewrite_src_uri(src_uris, lines_before)
>> +
>> +        handle_license_vars(srctree, lines_before, handled, extravalues, d)
>> +        self._rewrite_lic_uri(lines_before)
>> +
>> +        lines_before.append("GO_IMPORT = \"{}\"".format(go_import))
>> +        lines_before.append("SRCREV_FORMAT = \"${PN}\"")
>> +
>> +    def _update_lines_before(self, updated, newlines, lines_before):
>> +        if updated:
>> +            del lines_before[:]
>> +            for line in newlines:
>> +                # Hack to avoid newlines that edit_metadata inserts
>> +                if line.endswith('\n'):
>> +                    line = line[:-1]
>> +                lines_before.append(line)
>> +        return updated
>> +
>> +    def _rewrite_lic_uri(self, lines_before):
>> +
>> +        def varfunc(varname, origvalue, op, newlines):
>> +            if varname == 'LIC_FILES_CHKSUM':
>> +                new_licenses = []
>> +                licenses = origvalue.split()
>> +
>> +                for license in licenses:
>> +                    uri, chksum = license.split(';', 1)
>> +                    url = urllib.parse.urlparse(uri)
>> +                    new_uri = os.path.join(url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
>> +                    new_licenses.append(new_uri)
>> +
>> +                return new_licenses, None, -1, True
>> +            return origvalue, None, 0, True
>> +
>> +        updated, newlines = bb.utils.edit_metadata(lines_before, ['LIC_FILES_CHKSUM'], varfunc)
>> +        return self._update_lines_before(updated, newlines, lines_before)
>> +
>> +    def _rewrite_src_uri(self, src_uris_deps, lines_before):
>> +
>> +        def varfunc(varname, origvalue, op, newlines):
>> +            if varname == 'SRC_URI':
>> +                src_uri = []
>> +                src_uri.append("git://${GO_IMPORT};nobranch=1;name=${PN};protocol=https")
>> +                src_uri.extend(src_uris_deps)
>> +                return src_uri, None, -1, True
>> +            return origvalue, None, 0, True
>> +
>> +        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
>> +        return self._update_lines_before(updated, newlines, lines_before)
>> +
>> +def register_recipe_handlers(handlers):
>> +    handlers.append((GoRecipeHandler(), 60))
>> --
>> 2.30.2
>>
>>
>>
>>
>>
>>
>> -=-=-=-=-=-=-=-=-=-=-=-
>> Links: You receive all messages sent to this group.
>> View/Reply Online (#165331): https://lists.openembedded.org/g/openembedded-core/message/165331
>> Mute This Topic: https://lists.openembedded.org/mt/90928688/3616946
>> Group Owner: openembedded-core+owner@lists.openembedded.org
>> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [mark.asselstine@windriver.com]
>> -=-=-=-=-=-=-=-=-=-=-=-
>>

Re: [OE-core] [PATCH 3/5] poky-meta: add go vendor class for offline builds

[Mark Asselstine]

On 2022-05-06 02:59, Stefan Herbrechtsmeier wrote:
> From: Lukas Funke <lukas.funke@weidmueller.com>
> 
> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
> ---
> 
>   meta/classes/go-vendor.bbclass | 68 ++++++++++++++++++++++++++++++++++
>   1 file changed, 68 insertions(+)
>   create mode 100644 meta/classes/go-vendor.bbclass
> 
> diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
> new file mode 100644
> index 0000000000..a1a740a1fc
> --- /dev/null
> +++ b/meta/classes/go-vendor.bbclass
> @@ -0,0 +1,68 @@
> +# Copyright 2022 (C) Weidmueller GmbH & Co KG
> +# Author: Lukas Funke <lukas.funke@weidmueller.com>
> +#
> +# Handle Go vendor support for offline builds
> +#
> +# When importing Go modules, Go downloads the imported module using
> +# a network (proxy) connection ahead of the compile stage. This contradicts
> +# the yocto build concept of fetching every source ahead of build-time
> +# and supporting offline builds.
> +#
> +# To support offline builds, we use Go 'vendoring': module dependencies are
> +# downloaded during the fetch-phase and unpacked into the modules 'vendor'
> +# folder. Additinally a manifest file is generated for the 'vendor' folder
> +#
> +

Several instances of trailing whitespace in the file header.

> +inherit go-mod
> +
> +def go_src_uri(repo, path=None, subdir=None, vcs='git', destsuffix_prefix = 'git/src/import/vendor.fetch'):
> +    module_path = repo if not path else path
> +    src_uri = "{}://{};name={};destsuffix={}/{}".format(vcs, repo, \
> +                                    module_path.replace('/', '.'), \
> +                                    destsuffix_prefix, module_path)
> +
> +    src_uri += ";subdir={}".format(subdir) if subdir else ""
> +    src_uri += ";nobranch=1;protocol=https" if vcs == "git" else ""
> +
> +    return src_uri
> +
> +def go_generate_vendor_manifest(d):
> +
> +    vendor_dir = os.path.join(os.path.basename(d.getVar('S')),
> +                                        'src', d.getVar('GO_IMPORT'), "vendor")
> +    dst = os.path.join(vendor_dir, "modules.txt")
> +
> +    go_modules = d.getVarFlags("GO_MODULE_PATH")
> +    with open(dst, "w") as manifest:
> +        for go_module in go_modules:
> +            module_path = d.getVarFlag("GO_MODULE_PATH", go_module)
> +            module_version = d.getVarFlag("GO_MODULE_VERSION", go_module)
> +            if module_path and module_version:
> +                manifest.write("# %s %s\n" % (module_path, module_version))
> +                manifest.write("## explicit\n")
> +                exclude = set(['vendor'])
> +                for subdir, dirs, files in os.walk(os.path.join(vendor_dir, module_path), topdown=True):
> +                    dirs[:] = [d for d in dirs if d not in exclude]
> +                    for file in files:

I am not sure what other folks think but I tend not to like to see 
python builtins reused as variable names. Too easy to screw up and 
messes with sytax highlighting in editors.

MarkA

> +                        if file.endswith(".go"):
> +                            manifest.write(subdir[len(vendor_dir)+1:] + "\n")
> +                            break
> +
> +python go_do_unpack:append() {
> +    src_uri = (d.getVar('SRC_URI') or "").split()
> +    if len(src_uri) == 0:
> +        return
> +
> +    try:
> +        fetcher = bb.fetch2.Fetch(src_uri, d)
> +        src_folder = os.path.join(os.path.basename(d.getVar('S')),
> +                                        'src', d.getVar('GO_IMPORT'))
> +        vendor_src = os.path.join(src_folder, "vendor")
> +        vendor_dst = os.path.join(d.getVar('S'), "src", "import", "vendor.fetch")
> +
> +        os.symlink(os.path.relpath(vendor_dst, src_folder), vendor_src)
> +        go_generate_vendor_manifest(d)
> +
> +    except bb.fetch2.BBFetchException as e:
> +        raise bb.build.FuncFailed(e)
> +}
> 
> 
> 
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#165324): https://lists.openembedded.org/g/openembedded-core/message/165324
> Mute This Topic: https://lists.openembedded.org/mt/90928683/3616946
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [mark.asselstine@windriver.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>