[PATCH v2 0/2] Add symlinks for git (shallow) tarballs when name differs due to mirror rewrite rules

[PATCH v2 0/2] Add symlinks for git (shallow) tarballs when name differs due to mirror rewrite rules

[Urs Fässler]

v2: Check if fullmirror and shallow property exists before creating symlinks for
them. This prevents errors with fetchers that don't support tarball creation.

The patches are based on master-next (5463c16e3619d324aed137f47f93f0997a227d29).

The git (shallow) tarballs are not found when certain mirror rewrite rules are
in use. This is due to the fact that the tarball is named according to the url
where it is downloaded from. The error occurs in the unpack step where BitBake
looks for a tarball named after the url as specified in the recipe. By adding
the symlinks BitBake finds the tarballs in the unpack step.

Regards
Urs

Urs Fässler (2):
  fetch2: add symlinks for mirror tarballs
  fetch2: add symlink for shallow mirror tarballs

 lib/bb/fetch2/__init__.py |   6 +++
 lib/bb/tests/fetch.py     | 102 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 107 insertions(+), 1 deletion(-)

-- 
2.19.1

[PATCH v2 1/2] fetch2: add symlinks for mirror tarballs

[Urs Fässler]

In the unpack step the mirror tarball is not found when the mirrored
tarball is named differently than the original tarball. This happens
due to mirror rewrite rules. To solve the problem we add symlinks for
the differently named tarballs.

Signed-off-by: Urs Fässler <urs.fassler@bbv.ch>
Signed-off-by: Pascal Bach <pascal.bach@siemens.com>
---
 lib/bb/fetch2/__init__.py |  4 +++
 lib/bb/tests/fetch.py     | 58 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 2b62b416..84d2af73 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1019,6 +1019,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
             return origud.localpath
         # Otherwise the result is a local file:// and we symlink to it
         ensure_symlink(ud.localpath, origud.localpath)
+
+        if hasattr(ud, "fullmirror") and os.path.exists(ud.fullmirror):
+            ensure_symlink(ud.fullmirror, origud.fullmirror)
+
         update_stamp(origud, ld)
         return ud.localpath
 
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 17909ec6..0c6382a2 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -29,6 +29,20 @@ from bb.fetch2 import URI
 from bb.fetch2 import FetchMethod
 import bb
 
+
+def get_broken_symlinks(directory):
+    result = []
+
+    dir = os.listdir(directory)
+    for filename in dir:
+        fullname = directory + '/' + filename
+        broken = (not os.path.exists(fullname)) and os.path.islink(fullname)
+        if broken:
+            result += [filename]
+
+    return result
+
+
 def skipIfNoNetwork():
     if os.environ.get("BB_SKIP_NETTESTS") == "yes":
         return unittest.skip("Network tests being skipped")
@@ -536,7 +550,7 @@ class TarballNamingTest(FetcherTest):
         self.assertIn(self.recipe_tarball, dir)
 
     @skipIfNoNetwork()
-    def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
+    def test_that_tarball_for_mirrored_url_and_recipe_url_exists_when_mirroring_is_used(self):
         self.setup_mirror_rewrite()
         fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
 
@@ -544,6 +558,7 @@ class TarballNamingTest(FetcherTest):
 
         dir = os.listdir(self.dldir)
         self.assertIn(self.mirror_tarball, dir)
+        self.assertIn(self.recipe_tarball, dir)
 
 
 class GitShallowTarballNamingTest(FetcherTest):
@@ -647,6 +662,11 @@ class FetcherLocalTest(FetcherTest):
         with self.assertRaises(bb.fetch2.UnpackError):
             self.fetchUnpack(['file://a;subdir=/bin/sh'])
 
+    def test_local_mirror_of_http(self):
+        self.d.setVar("PREMIRRORS", " http://.*/.* file://" + self.localsrcdir + " \n ")
+        tree = self.fetchUnpack(["http://downloads.yoctoproject.org/releases/a"])
+        self.assertEqual(tree, ['a'])
+
 class FetcherNoNetworkTest(FetcherTest):
     def setUp(self):
         super().setUp()
@@ -881,6 +901,12 @@ class FetcherNetworkTest(FetcherTest):
         self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
         self.gitfetcher(recipeurl, recipeurl)
 
+    @skipIfNoNetwork()
+    def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_mirror(self):
+        recipeurl = "git://git.wrong.org/bitbake"
+        self.d.setVar("MIRRORS", "git://git.wrong.org/bitbake git://git.openembedded.org/bitbake \n")
+        self.gitfetcher(recipeurl, recipeurl)
+
     @skipIfNoNetwork()
     def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
         realurl = "git://git.openembedded.org/bitbake"
@@ -891,6 +917,36 @@ class FetcherNetworkTest(FetcherTest):
         self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
         self.gitfetcher(recipeurl, recipeurl)
 
+    @skipIfNoNetwork()
+    def test_gitfetch_uses_the_git_tarball_when_using_mirror_rewrite_rules_and_the_git_clone_does_not_exist(self):
+        self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
+        self.d.setVar('PREMIRRORS', 'git://git.yoctoproject.org/.* git://git.yoctoproject.org/git/PATH;protocol=https \n')
+        fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
+        fetcher.download()
+        bb.utils.prunedir(self.dldir + '/git2')
+        bb.utils.prunedir(self.unpackdir)
+        self.d.setVar('PREMIRRORS', '')
+        self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
+
+        fetcher.download()
+
+        fetcher.unpack(self.unpackdir)
+        dir = os.listdir(self.unpackdir + "/git/")
+        self.assertIn("fstests.doap", dir)
+
+    @skipIfNoNetwork()
+    def test_that_no_broken_symlinks_exist_when_using_mirror_rewrite_rules_and_no_tarball(self):
+        self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
+        self.d.setVar('PREMIRRORS', 'git://git.yoctoproject.org/.* git://git.yoctoproject.org/git/PATH;protocol=https \n')
+        fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
+
+        fetcher.download()
+
+        broken_symlinks = get_broken_symlinks(self.dldir)
+        self.assertEqual([], broken_symlinks)
+
     @skipIfNoNetwork()
     def test_git_submodule(self):
         fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
-- 
2.19.1

[PATCH v2 2/2] fetch2: add symlink for shallow mirror tarballs

[Urs Fässler]

In the unpack step the shallow mirror tarball is not found when the
mirrored tarball is named differently than the original tarball. This
happens due to mirror rewrite rules. To solve the problem we add
symlinks for the differently named tarballs.

Signed-off-by: Urs Fässler <urs.fassler@bbv.ch>
Signed-off-by: Pascal Bach <pascal.bach@siemens.com>
---
 lib/bb/fetch2/__init__.py |  2 ++
 lib/bb/tests/fetch.py     | 44 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+)

diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 84d2af73..99520e82 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1022,6 +1022,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
 
         if hasattr(ud, "fullmirror") and os.path.exists(ud.fullmirror):
             ensure_symlink(ud.fullmirror, origud.fullmirror)
+        if hasattr(ud, "shallow") and ud.shallow and os.path.exists(ud.fullshallow):
+            ensure_symlink(ud.fullshallow, origud.fullshallow)
 
         update_stamp(origud, ld)
         return ud.localpath
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 0c6382a2..fc15ee47 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -595,6 +595,21 @@ class GitShallowTarballNamingTest(FetcherTest):
         dir = os.listdir(self.dldir)
         self.assertIn(self.mirror_tarball, dir)
 
+    @skipIfNoNetwork()
+    def test_that_tarball_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used_and_the_mirrored_tarball_already_exists(self):
+        self.setup_mirror_rewrite()
+        fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
+        fetcher.download()
+        bb.utils.prunedir(self.dldir + '/git2')
+        bb.utils.prunedir(self.unpackdir)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        dir = os.listdir(self.dldir)
+        self.assertIn(self.mirror_tarball, dir)
+        self.assertIn(self.recipe_tarball, dir)
+
 
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
@@ -1781,3 +1796,32 @@ class GitShallowTest(FetcherTest):
 
         dir = os.listdir(self.unpackdir + "/git/")
         self.assertIn("fstests.doap", dir)
+
+    @skipIfNoNetwork()
+    def test_that_unpack_uses_the_git_shallow_tarball_when_using_mirror_rewrite_rules_and_the_git_clone_does_not_exist(self):
+        self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
+        self.d.setVar('BB_GIT_SHALLOW', '1')
+        self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
+        self.d.setVar('PREMIRRORS', 'git://git.yoctoproject.org/.* git://git.yoctoproject.org/git/PATH;protocol=https \n')
+        fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
+        fetcher.download()
+        bb.utils.remove(self.dldir + '/git2', recurse=True)
+
+        fetcher.unpack(self.unpackdir)
+
+        dir = os.listdir(self.unpackdir + "/git/")
+        self.assertIn("fstests.doap", dir)
+
+    @skipIfNoNetwork()
+    def test_that_no_broken_symlinks_exist_when_using_mirror_rewrite_rules_and_mirror_tarball_is_configured(self):
+        self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
+        self.d.setVar('BB_GIT_SHALLOW', '1')
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
+        self.d.delVar('BB_GENERATE_SHALLOW_TARBALLS')
+        self.d.setVar('PREMIRRORS', 'git://git.yoctoproject.org/.* git://git.yoctoproject.org/git/PATH;protocol=https \n')
+        fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
+
+        fetcher.download()
+
+        broken_symlinks = get_broken_symlinks(self.dldir)
+        self.assertEqual([], broken_symlinks)
-- 
2.19.1