[PATCH v2 1/9] recipetool: Don't fail on local go modules

[PATCH v2 1/9] recipetool: Don't fail on local go modules

[Vyacheslav Yurkov]

Local modules are usually referenced with a 'replace' directive in
go.mod file. If that's the case, remove them from populating SRC_URI.

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 scripts/lib/recipetool/create_go.py | 36 ++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
index 21dcb41271..668a24b8dd 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -504,7 +504,7 @@ class GoRecipeHandler(RecipeHandler):
 
         return inline_fcn, commit
 
-    def __go_handle_dependencies(self, go_mod, localfilesdir, extravalues, d):
+    def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d):
 
         src_uris = []
         src_revs = []
@@ -525,6 +525,27 @@ class GoRecipeHandler(RecipeHandler):
 
             return src_rev
 
+        # we first go over replacement list, because we are essentialy
+        # interested only in the replaced path
+        if go_mod['Replace']:
+            for replacement in go_mod['Replace']:
+                oldpath = replacement['Old']['Path']
+                path = replacement['New']['Path']
+                version = ''
+                if 'Version' in replacement['New']:
+                    version = replacement['New']['Version']
+
+                if os.path.exists(os.path.join(srctree, path)):
+                    # the module refers to the local path, remove it from requirement list
+                    # because it's a local module
+                    go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath]
+                else:
+                    # Replace the path and the version, so we don't iterate replacement list anymore
+                    for require in go_mod['Require']:
+                        if require['Path'] == oldpath:
+                            require.update({'Path': path, 'Version': version})
+                            break
+
         for require in go_mod['Require']:
             path = require['Path']
             version = require['Version']
@@ -534,17 +555,6 @@ class GoRecipeHandler(RecipeHandler):
             src_uris.append(inline_fcn)
             src_revs.append(generate_src_rev(path, version, commithash))
 
-        if go_mod['Replace']:
-            for replacement in go_mod['Replace']:
-                oldpath = replacement['Old']['Path']
-                path = replacement['New']['Path']
-                version = replacement['New']['Version']
-
-                inline_fcn, commithash = self.__generate_srcuri_inline_fcn(
-                    path, version, oldpath)
-                src_uris.append(inline_fcn)
-                src_revs.append(generate_src_rev(path, version, commithash))
-
         pn, _ = determine_from_url(go_mod['Module']['Path'])
         go_mods_basename = "%s-modules.inc" % pn
 
@@ -693,7 +703,7 @@ class GoRecipeHandler(RecipeHandler):
 
             self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
 
-            self.__go_handle_dependencies(go_mod, localfilesdir, extravalues, d)
+            self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d)
             lines_before.append("require ${BPN}-modules.inc")
 
         # Do generic license handling
-- 
2.35.1

[PATCH v2 2/9] classes: go-vendor: Reference local modules

[Vyacheslav Yurkov]

Create symlinks for local modules, which are usually not referenced in
the SRC_URI, but still expected to be found in the vendor directory
during the build.

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/classes/go-vendor.bbclass | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
index 2426bddfba..b965428dd1 100644
--- a/meta/classes/go-vendor.bbclass
+++ b/meta/classes/go-vendor.bbclass
@@ -169,6 +169,7 @@ python do_go_vendor() {
     fetched_paths.remove('.')
 
     vendored_paths = set()
+    replaced_paths = dict()
     with open(modules_txt_src) as f:
         for line in f:
             if not line.startswith("#"):
@@ -182,6 +183,15 @@ python do_go_vendor() {
                         vendored_paths.add(topdir)
 
                     topdir = os.path.dirname(topdir)
+            else:
+                replaced_module = line.split("=>")
+                if len(replaced_module) > 1:
+                    # This module has been replaced, use a local path
+                    # we parse the line that has a pattern "# module-name [module-version] => local-path
+                    actual_path = replaced_module[1].strip()
+                    vendored_name = replaced_module[0].split()[1]
+                    bb.debug(1, "added vendored name %s for actual path %s" % (vendored_name, actual_path))
+                    replaced_paths[vendored_name] = actual_path
 
     for path in fetched_paths:
         if path not in vendored_paths:
@@ -189,7 +199,13 @@ python do_go_vendor() {
             if os.path.exists(realpath):
                 shutil.rmtree(realpath)
 
-    # Create a symlink the the actual directory
+    for vendored_name, replaced_path in replaced_paths.items():
+        symlink_target = os.path.join(source_dir, *['src', go_import, replaced_path])
+        symlink_name = os.path.join(vendor_dir, vendored_name)
+        bb.debug(1, "vendored name %s, symlink name %s" % (vendored_name, symlink_name))
+        os.symlink(symlink_target, symlink_name)
+
+    # Create a symlink to the actual directory
     os.symlink(vendor_dir, linkname)
 }
 
-- 
2.35.1

[PATCH v2 3/9] classes: go-vendor: Handle modules from the same repo

[Vyacheslav Yurkov]

Take into account module version when populating vendor directory,
because a module with the same URL but with a different version tag
could be used as an indirect dependency.

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/classes/go-vendor.bbclass | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
index b965428dd1..48f4d1041c 100644
--- a/meta/classes/go-vendor.bbclass
+++ b/meta/classes/go-vendor.bbclass
@@ -103,18 +103,17 @@ python do_go_vendor() {
         pathMajor = fetcher.ud[url].parm.get('go_pathmajor')
         pathMajor = None if not pathMajor else pathMajor.strip('/')
 
-        if not repo in modules:
-            modules[repo] =   { "version": version,
+        if not (repo, version) in modules:
+            modules[(repo, version)] =   {
                                 "repo_path": os.path.join(import_dir, p),
                                 "module_path": module_path,
                                 "subdir": subdir,
                                 "pathMajor": pathMajor }
 
-    for module_key in sorted(modules):
+    for module_key, module in modules.items():
 
         # only take the version which is explicitly listed
         # as a dependency in the go.mod
-        module = modules[module_key]
         module_path = module['module_path']
         rootdir = module['repo_path']
         subdir = module['subdir']
@@ -139,7 +138,7 @@ python do_go_vendor() {
         dst = os.path.join(vendor_dir, module_path)
 
         bb.debug(1, "cp %s --> %s" % (src, dst))
-        shutil.copytree(src, dst, symlinks=True, \
+        shutil.copytree(src, dst, symlinks=True, dirs_exist_ok=True, \
             ignore=shutil.ignore_patterns(".git", \
                                             "vendor", \
                                             "*._test.go"))
-- 
2.35.1

[PATCH v2 4/9] classes: go-vendor: Unlink vendor dir later

[Vyacheslav Yurkov]

Vendor directory might still be required during install stage, so defer
the removal until later stage.

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/classes/go-vendor.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
index 48f4d1041c..1bbb99ac79 100644
--- a/meta/classes/go-vendor.bbclass
+++ b/meta/classes/go-vendor.bbclass
@@ -48,7 +48,7 @@ python do_vendor_unlink() {
     os.unlink(linkname)
 }
 
-addtask vendor_unlink before do_install after do_compile
+addtask vendor_unlink before do_package after do_install
 
 python do_go_vendor() {
     import shutil
-- 
2.35.1

[PATCH v2 5/9] recipetool: Proceed even with a missing license file

[Vyacheslav Yurkov]

Whenever the recipe uses a CLOSED license, the list is going to be
empty. It's a discouraged practice not to have a license, but proceed
anyway to finish recipe generation.

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 scripts/lib/recipetool/create_go.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
index 668a24b8dd..035ab9f7c1 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -730,6 +730,12 @@ class GoRecipeHandler(RecipeHandler):
                 new_licenses = []
                 licenses = origvalue.split('\\')
                 for license in licenses:
+                    if not license:
+                        logger.warning("No license file was detected for the main module!")
+                        # the license list of the main recipe must be empty
+                        # this can happen for example in case of CLOSED license
+                        # Fall through to complete recipe generation
+                        continue
                     license = license.strip()
                     uri, chksum = license.split(';', 1)
                     url = urllib.parse.urlparse(uri)
-- 
2.35.1

[PATCH v2 6/9] recipetool: Disregard version in URL for replaced modules

[Vyacheslav Yurkov]

Major module version is a part of name, but not necessary part of the
actual URL (See https://go.dev/ref/mod#module-path).

Nevertheless, name detection function can't handle that suffix, so get
rid of it to determine component name.

For replaced modules that name might be different that the actual module
name defined in go.mod file.

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 scripts/lib/recipetool/create_go.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
index 035ab9f7c1..c560831442 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -506,6 +506,7 @@ class GoRecipeHandler(RecipeHandler):
 
     def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d):
 
+        import re
         src_uris = []
         src_revs = []
 
@@ -555,7 +556,9 @@ class GoRecipeHandler(RecipeHandler):
             src_uris.append(inline_fcn)
             src_revs.append(generate_src_rev(path, version, commithash))
 
-        pn, _ = determine_from_url(go_mod['Module']['Path'])
+        # strip version part from module URL /vXX
+        baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+        pn, _ = determine_from_url(baseurl)
         go_mods_basename = "%s-modules.inc" % pn
 
         go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
@@ -636,7 +639,9 @@ class GoRecipeHandler(RecipeHandler):
                 lic_files_chksum.append(
                     'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2]))
 
-        pn, _ = determine_from_url(go_mod['Module']['Path'])
+        # strip version part from module URL /vXX
+        baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+        pn, _ = determine_from_url(baseurl)
         licenses_basename = "%s-licenses.inc" % pn
 
         licenses_filename = os.path.join(localfilesdir, licenses_basename)
@@ -682,6 +687,13 @@ class GoRecipeHandler(RecipeHandler):
 
         localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
         extravalues.setdefault('extrafiles', {})
+
+        # Use an explicit name determined from the module name because it
+        # might differ from the actual URL for replaced modules
+        # strip version part from module URL /vXX
+        baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+        pn, _ = determine_from_url(baseurl)
+
         # go.mod files with version < 1.17 may not include all indirect
         # dependencies. Thus, we have to upgrade the go version.
         if go_version_major == 1 and go_version_minor < 17:
@@ -699,18 +711,18 @@ class GoRecipeHandler(RecipeHandler):
             # Write additional $BPN-modules.inc file
             self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d)
             lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"")
-            lines_before.append("require ${BPN}-licenses.inc")
+            lines_before.append("require %s-licenses.inc" % (pn))
 
             self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
 
             self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d)
-            lines_before.append("require ${BPN}-modules.inc")
+            lines_before.append("require %s-modules.inc" % (pn))
 
         # Do generic license handling
         handle_license_vars(srctree, lines_before, handled, extravalues, d)
         self.__rewrite_lic_uri(lines_before)
 
-        lines_before.append("GO_IMPORT = \"{}\"".format(go_import))
+        lines_before.append("GO_IMPORT = \"{}\"".format(baseurl))
         lines_before.append("SRCREV_FORMAT = \"${BPN}\"")
 
     def __update_lines_before(self, updated, newlines, lines_before):
-- 
2.35.1

[PATCH v2 7/9] oeqa/selftest/recipetool: Move create_go test to a proper class

[Vyacheslav Yurkov]

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/lib/oeqa/selftest/cases/recipetool.py | 24 ++++++++++------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py b/meta/lib/oeqa/selftest/cases/recipetool.py
index df15c80069..356bb4a746 100644
--- a/meta/lib/oeqa/selftest/cases/recipetool.py
+++ b/meta/lib/oeqa/selftest/cases/recipetool.py
@@ -744,19 +744,6 @@ class RecipetoolCreateTests(RecipetoolBase):
     def test_recipetool_create_git_srcbranch(self):
         self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;protocol=https', 'matchbox-keyboard-0-1')
 
-
-class RecipetoolTests(RecipetoolBase):
-
-    @classmethod
-    def setUpClass(cls):
-        import sys
-
-        super(RecipetoolTests, cls).setUpClass()
-        bb_vars = get_bb_vars(['BBPATH'])
-        cls.bbpath = bb_vars['BBPATH']
-        libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'recipetool')
-        sys.path.insert(0, libpath)
-
     def test_recipetool_create_go(self):
         # Basic test to check go recipe generation
         def urifiy(url, version, modulepath = None, pathmajor = None, subdir = None):
@@ -941,6 +928,17 @@ class RecipetoolTests(RecipetoolBase):
         self._test_recipe_contents(deps_require_file, checkvars, [])
 
 
+class RecipetoolTests(RecipetoolBase):
+
+    @classmethod
+    def setUpClass(cls):
+        import sys
+
+        super(RecipetoolTests, cls).setUpClass()
+        bb_vars = get_bb_vars(['BBPATH'])
+        cls.bbpath = bb_vars['BBPATH']
+        libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'recipetool')
+        sys.path.insert(0, libpath)
 
     def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths):
         dstdir = basedstdir
-- 
2.35.1

[PATCH v2 8/9] oeqa/selftest/recipetool: Move helper function to the class scope

[Vyacheslav Yurkov]

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/lib/oeqa/selftest/cases/recipetool.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py b/meta/lib/oeqa/selftest/cases/recipetool.py
index 356bb4a746..83848d6170 100644
--- a/meta/lib/oeqa/selftest/cases/recipetool.py
+++ b/meta/lib/oeqa/selftest/cases/recipetool.py
@@ -744,14 +744,14 @@ class RecipetoolCreateTests(RecipetoolBase):
     def test_recipetool_create_git_srcbranch(self):
         self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;protocol=https', 'matchbox-keyboard-0-1')
 
+    def _go_urifiy(self, url, version, modulepath = None, pathmajor = None, subdir = None):
+        modulepath = ",path='%s'" % modulepath if len(modulepath) else ''
+        pathmajor = ",pathmajor='%s'" % pathmajor if len(pathmajor) else ''
+        subdir = ",subdir='%s'" % subdir if len(subdir) else ''
+        return "${@go_src_uri('%s','%s'%s%s%s)}" % (url, version, modulepath, pathmajor, subdir)
+
     def test_recipetool_create_go(self):
         # Basic test to check go recipe generation
-        def urifiy(url, version, modulepath = None, pathmajor = None, subdir = None):
-            modulepath = ",path='%s'" % modulepath if len(modulepath) else ''
-            pathmajor = ",pathmajor='%s'" % pathmajor if len(pathmajor) else ''
-            subdir = ",subdir='%s'" % subdir if len(subdir) else ''
-            return "${@go_src_uri('%s','%s'%s%s%s)}" % (url, version, modulepath, pathmajor, subdir)
-
         temprecipe = os.path.join(self.tempdir, 'recipe')
         os.makedirs(temprecipe)
 
@@ -919,7 +919,7 @@ class RecipetoolCreateTests(RecipetoolBase):
 
         src_uri = set()
         for d in dependencies:
-            src_uri.add(urifiy(*d))
+            src_uri.add(self._go_urifiy(*d))
 
         checkvars = {}
         checkvars['GO_DEPENDENCIES_SRC_URI'] = src_uri
-- 
2.35.1

[PATCH v2 9/9] oeqa/selftest/recipetool: Add test coverage for local go modules

[Vyacheslav Yurkov]

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/lib/oeqa/selftest/cases/recipetool.py | 55 ++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py b/meta/lib/oeqa/selftest/cases/recipetool.py
index 83848d6170..0c296875b2 100644
--- a/meta/lib/oeqa/selftest/cases/recipetool.py
+++ b/meta/lib/oeqa/selftest/cases/recipetool.py
@@ -927,6 +927,61 @@ class RecipetoolCreateTests(RecipetoolBase):
         self.assertTrue(os.path.isfile(deps_require_file))
         self._test_recipe_contents(deps_require_file, checkvars, [])
 
+    def test_recipetool_create_go_replace_modules(self):
+        # Check handling of replaced modules
+        temprecipe = os.path.join(self.tempdir, 'recipe')
+        os.makedirs(temprecipe)
+
+        recipefile = os.path.join(temprecipe, 'openapi-generator_git.bb')
+        deps_require_file = os.path.join(temprecipe, 'openapi-generator', 'go-modules.inc')
+        lics_require_file = os.path.join(temprecipe, 'openapi-generator', 'go-licenses.inc')
+        modules_txt_file = os.path.join(temprecipe, 'openapi-generator', 'modules.txt')
+
+        srcuri = 'https://github.com/OpenAPITools/openapi-generator.git'
+        srcrev = "v7.2.0"
+        srcbranch = "master"
+        srcsubdir = "samples/openapi3/client/petstore/go"
+
+        result = runCmd('recipetool create -o %s %s -S %s -B %s --src-subdir %s' % (temprecipe, srcuri, srcrev, srcbranch, srcsubdir))
+
+        self.maxDiff = None
+        inherits = ['go-vendor']
+
+        checkvars = {}
+        checkvars['GO_IMPORT'] = "github.com/OpenAPITools/openapi-generator/samples/openapi3/client/petstore/go"
+        checkvars['SRC_URI'] = {'git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https',
+                                'file://modules.txt'}
+
+        self.assertNotIn('Traceback', result.output)
+        self.assertIn('No license file was detected for the main module', result.output)
+        self.assertTrue(os.path.isfile(recipefile))
+        self._test_recipe_contents(recipefile, checkvars, inherits)
+
+        # make sure that dependencies don't mention local directory ./go-petstore
+        dependencies = \
+            [   ('github.com/stretchr/testify','v1.8.4', '', '', ''),
+                ('go.googlesource.com/oauth2','v0.10.0','golang.org/x/oauth2', '', ''),
+                ('github.com/davecgh/go-spew','v1.1.1', '', '', ''),
+                ('github.com/golang/protobuf','v1.5.3', '', '', ''),
+                ('github.com/kr/pretty','v0.3.0', '', '', ''),
+                ('github.com/pmezard/go-difflib','v1.0.0', '', '', ''),
+                ('github.com/rogpeppe/go-internal','v1.9.0', '', '', ''),
+                ('go.googlesource.com/net','v0.12.0','golang.org/x/net', '', ''),
+                ('github.com/golang/appengine','v1.6.7','google.golang.org/appengine', '', ''),
+                ('go.googlesource.com/protobuf','v1.31.0','google.golang.org/protobuf', '', ''),
+                ('gopkg.in/check.v1','v1.0.0-20201130134442-10cb98267c6c', '', '', ''),
+                ('gopkg.in/yaml.v3','v3.0.1', '', '', ''),
+            ]
+
+        src_uri = set()
+        for d in dependencies:
+            src_uri.add(self._go_urifiy(*d))
+
+        checkvars = {}
+        checkvars['GO_DEPENDENCIES_SRC_URI'] = src_uri
+
+        self.assertTrue(os.path.isfile(deps_require_file))
+        self._test_recipe_contents(deps_require_file, checkvars, [])
 
 class RecipetoolTests(RecipetoolBase):
 
-- 
2.35.1