[PATCH v3 1/3] fetch/git: add support for disabling shared clones on unpack

[PATCH v3 1/3] fetch/git: add support for disabling shared clones on unpack

[henning.schild]

By default the unpacker will create a "shared" clone when cloning from
the DL_DIR to the WORKDIR. This patch introduces an option to control
that behaviour.

Imagine some recipe steps are executed in a namespace that is different
from the one your downloader and unpacker ran in. (chroot) Because a
"shared" clone has an absolute reference to its "alternate" you now
have to make that "alternate" visible in that new namespace (chroot) at
the exact place.

With this patch you can unpack "noshared" and get a stand-alone copy.
This copy will also work if the "alternate" is not visible or existant.

The switch is a global bitbake switch and will affect all git urls.
Build systems that need "noshared" most likely need it for everything
they do with git.

Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
 lib/bb/fetch2/git.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index e3ba80a3f52a..3e25b4b6047c 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -168,7 +168,11 @@ class Git(FetchMethod):
         if len(branches) != len(ud.names):
             raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
 
-        ud.cloneflags = "-s -n"
+        ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
+
+        ud.cloneflags = "-n"
+        if not ud.noshared:
+            ud.cloneflags += " -s"
         if ud.bareclone:
             ud.cloneflags += " --mirror"
 
-- 
2.26.3

[PATCH v3 2/3] tests/fetch: deduplicate local git testing code

[henning.schild]

Purely cosmetic change that probably improves the code.

Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
 lib/bb/tests/fetch.py | 34 ++++++----------------------------
 1 file changed, 6 insertions(+), 28 deletions(-)

diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index b921a952e493..ab02bc74ac15 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -674,7 +674,7 @@ class FetcherLocalTest(FetcherTest):
         with self.assertRaises(bb.fetch2.UnpackError):
             self.fetchUnpack(['file://a;subdir=/bin/sh'])
 
-    def test_local_gitfetch_usehead(self):
+    def dummyGitTest(self, suffix):
         # Create dummy local Git repo
         src_dir = tempfile.mkdtemp(dir=self.tempdir,
                                    prefix='gitfetch_localusehead_')
@@ -693,7 +693,7 @@ class FetcherLocalTest(FetcherTest):
 
         # Fetch and check revision
         self.d.setVar("SRCREV", "AUTOINC")
-        url = "git://" + src_dir + ";protocol=file;usehead=1"
+        url = "git://" + src_dir + ";protocol=file;" + suffix
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
@@ -702,33 +702,11 @@ class FetcherLocalTest(FetcherTest):
         unpack_rev = stdout[0].strip()
         self.assertEqual(orig_rev, unpack_rev)
 
-    def test_local_gitfetch_usehead_withname(self):
-        # Create dummy local Git repo
-        src_dir = tempfile.mkdtemp(dir=self.tempdir,
-                                   prefix='gitfetch_localusehead_')
-        src_dir = os.path.abspath(src_dir)
-        bb.process.run("git init", cwd=src_dir)
-        bb.process.run("git config user.email 'you@example.com'", cwd=src_dir)
-        bb.process.run("git config user.name 'Your Name'", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit'",
-                       cwd=src_dir)
-        # Use other branch than master
-        bb.process.run("git checkout -b my-devel", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
-                       cwd=src_dir)
-        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
-        orig_rev = stdout[0].strip()
+    def test_local_gitfetch_usehead(self):
+        self.dummyGitTest("usehead=1")
 
-        # Fetch and check revision
-        self.d.setVar("SRCREV", "AUTOINC")
-        url = "git://" + src_dir + ";protocol=file;usehead=1;name=newName"
-        fetcher = bb.fetch.Fetch([url], self.d)
-        fetcher.download()
-        fetcher.unpack(self.unpackdir)
-        stdout = bb.process.run("git rev-parse HEAD",
-                                cwd=os.path.join(self.unpackdir, 'git'))
-        unpack_rev = stdout[0].strip()
-        self.assertEqual(orig_rev, unpack_rev)
+    def test_local_gitfetch_usehead_withname(self):
+        self.dummyGitTest("usehead=1;name=newName")
 
 class FetcherNoNetworkTest(FetcherTest):
     def setUp(self):
-- 
2.26.3

[PATCH v3 3/3] tests/fetch: add tests for local and remote "noshared" git fetching

[henning.schild]

Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
 lib/bb/tests/fetch.py | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index ab02bc74ac15..0f7585e119fd 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -708,6 +708,18 @@ class FetcherLocalTest(FetcherTest):
     def test_local_gitfetch_usehead_withname(self):
         self.dummyGitTest("usehead=1;name=newName")
 
+    def test_local_gitfetch_shared(self):
+        self.dummyGitTest("usehead=1;name=sharedName")
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertTrue(os.path.exists(alt))
+
+    def test_local_gitfetch_noshared(self):
+        self.d.setVar('BB_GIT_NOSHARED', '1')
+        self.unpackdir += '_noshared'
+        self.dummyGitTest("usehead=1;name=noSharedName")
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertFalse(os.path.exists(alt))
+
 class FetcherNoNetworkTest(FetcherTest):
     def setUp(self):
         super().setUp()
@@ -2628,3 +2640,29 @@ class NPMTest(FetcherTest):
         fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
+
+class GitSharedTest(FetcherTest):
+    def setUp(self):
+        super(GitSharedTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    @skipIfNoNetwork()
+    def test_shared_unpack(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertTrue(os.path.exists(alt))
+
+    @skipIfNoNetwork()
+    def test_noshared_unpack(self):
+        self.d.setVar('BB_GIT_NOSHARED', '1')
+        self.unpackdir += '_noshared'
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertFalse(os.path.exists(alt))
-- 
2.26.3

Re: [PATCH v3 1/3] fetch/git: add support for disabling shared clones on unpack

[henning.schild]

Note mainly to isar-users, this is now merged in upstream bitbake. Once
we update our bitbake we can set "BB_GIT_NOSHARED = 1" in our
bitbake.conf and drop the git adjust task. That should allow use of git
outside the chroot and hopefully does not have a too negative impact on
unpack performance ... for big things like the kernel

Henning

Am Wed, 14 Apr 2021 08:32:39 +0200
schrieb Henning Schild <henning.schild@siemens.com>:

> By default the unpacker will create a "shared" clone when cloning from
> the DL_DIR to the WORKDIR. This patch introduces an option to control
> that behaviour.
> 
> Imagine some recipe steps are executed in a namespace that is
> different from the one your downloader and unpacker ran in. (chroot)
> Because a "shared" clone has an absolute reference to its "alternate"
> you now have to make that "alternate" visible in that new namespace
> (chroot) at the exact place.
> 
> With this patch you can unpack "noshared" and get a stand-alone copy.
> This copy will also work if the "alternate" is not visible or
> existant.
> 
> The switch is a global bitbake switch and will affect all git urls.
> Build systems that need "noshared" most likely need it for everything
> they do with git.
> 
> Signed-off-by: Henning Schild <henning.schild@siemens.com>
> ---
>  lib/bb/fetch2/git.py | 6 +++++-
>  1 file changed, 5 insertions(+), 1 deletion(-)
> 
> diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
> index e3ba80a3f52a..3e25b4b6047c 100644
> --- a/lib/bb/fetch2/git.py
> +++ b/lib/bb/fetch2/git.py
> @@ -168,7 +168,11 @@ class Git(FetchMethod):
>          if len(branches) != len(ud.names):
>              raise bb.fetch2.ParameterError("The number of name and
> branch parameters is not balanced", ud.url) 
> -        ud.cloneflags = "-s -n"
> +        ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
> +
> +        ud.cloneflags = "-n"
> +        if not ud.noshared:
> +            ud.cloneflags += " -s"
>          if ud.bareclone:
>              ud.cloneflags += " --mirror"
>