All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH]gitsm fetcher: Fix recurse submodule error
@ 2019-01-27 16:57 Mark Hatle
  2019-01-27 16:57 ` [PATCH] gitsm: The fetcher did not process some recursive submodules properly Mark Hatle
  0 siblings, 1 reply; 3+ messages in thread
From: Mark Hatle @ 2019-01-27 16:57 UTC (permalink / raw)
  To: bitbake-devel; +Cc: liu.ming50

Due to an error in some optimization work, recursive submodules may not
always unpack.  Resolve this issue and add a specific test case to catch
any new regressions.

(already in master-next, but I wanted to make sure others saw the patch
as well..)

Mark Hatle (1):
  gitsm: The fetcher did not process some recursive submodules properly.

 lib/bb/fetch2/gitsm.py |  4 +++-
 lib/bb/tests/fetch.py  | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

-- 
1.8.3.1



^ permalink raw reply	[flat|nested] 3+ messages in thread
* [PATCH] gitsm: The fetcher did not process some recursive submodules properly.
  2019-01-27 16:57 [PATCH]gitsm fetcher: Fix recurse submodule error Mark Hatle
@ 2019-01-27 16:57 ` Mark Hatle
  2019-01-28 19:38   ` Stefan Agner
  0 siblings, 1 reply; 3+ messages in thread
From: Mark Hatle @ 2019-01-27 16:57 UTC (permalink / raw)
  To: bitbake-devel; +Cc: liu.ming50

Move the submodule processing outside of the if statement to avoid any
optimizations that may happen.

Update the test cases to include the additional case, and split the other
test cases into individual tests to make it easier to figure out what
the failure may be.

Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
 lib/bb/fetch2/gitsm.py |  4 +++-
 lib/bb/tests/fetch.py  | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index f45546b..b21fed2 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -198,6 +198,8 @@ class GitSM(Git):
 
         Git.unpack(self, ud, destdir, d)
 
-        if not ud.bareclone and self.process_submodules(ud, ud.destdir, unpack_submodules, d):
+        ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
+
+        if not ud.bareclone and ret:
             # Run submodule update, this sets up the directories -- without touching the config
             runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index e9ad807..522d202 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -912,6 +912,7 @@ class FetcherNetworkTest(FetcherTest):
         if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
             self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
 
+    def test_git_submodule_dbus_broker(self):
         # The following external repositories have show failures in fetch and unpack operations
         # We want to avoid regressions!
         url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"
@@ -928,16 +929,35 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
 
+    def test_git_submodule_CLI11(self):
         url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
         os.chdir(os.path.dirname(self.unpackdir))
         fetcher.unpack(self.unpackdir)
+
+        repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
 
+    def test_git_submodule_aktualizr(self):
+        url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+        # Previous cwd has been deleted
+        os.chdir(os.path.dirname(self.unpackdir))
+        fetcher.unpack(self.unpackdir)
+
+        repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
+        self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
+        self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
+        self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
+        self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
+        self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
+        self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
+
 class TrustedNetworksTest(FetcherTest):
     def test_trusted_network(self):
         # Ensure trusted_network returns False when the host IS in the list.
-- 
1.8.3.1



^ permalink raw reply related	[flat|nested] 3+ messages in thread
* Re: [PATCH] gitsm: The fetcher did not process some recursive submodules properly.
  2019-01-27 16:57 ` [PATCH] gitsm: The fetcher did not process some recursive submodules properly Mark Hatle
@ 2019-01-28 19:38   ` Stefan Agner
  0 siblings, 0 replies; 3+ messages in thread
From: Stefan Agner @ 2019-01-28 19:38 UTC (permalink / raw)
  To: Mark Hatle; +Cc: bitbake-devel, liu.ming50

On 27.01.2019 17:57, Mark Hatle wrote:
> Move the submodule processing outside of the if statement to avoid any
> optimizations that may happen.
> 
> Update the test cases to include the additional case, and split the other
> test cases into individual tests to make it easier to figure out what
> the failure may be.
> 
> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>

I can confirm that this resolves the issue we have been seeing:

Tested-by: Stefan Agner <stefan.agner@toradex.com>

Thanks Mark for looking into this!

--
Stefan

> ---
>  lib/bb/fetch2/gitsm.py |  4 +++-
>  lib/bb/tests/fetch.py  | 20 ++++++++++++++++++++
>  2 files changed, 23 insertions(+), 1 deletion(-)
> 
> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> index f45546b..b21fed2 100644
> --- a/lib/bb/fetch2/gitsm.py
> +++ b/lib/bb/fetch2/gitsm.py
> @@ -198,6 +198,8 @@ class GitSM(Git):
>  
>          Git.unpack(self, ud, destdir, d)
>  
> -        if not ud.bareclone and self.process_submodules(ud,
> ud.destdir, unpack_submodules, d):
> +        ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
> +
> +        if not ud.bareclone and ret:
>              # Run submodule update, this sets up the directories --
> without touching the config
>              runfetchcmd("%s submodule update --recursive --no-fetch"
> % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
> diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
> index e9ad807..522d202 100644
> --- a/lib/bb/tests/fetch.py
> +++ b/lib/bb/tests/fetch.py
> @@ -912,6 +912,7 @@ class FetcherNetworkTest(FetcherTest):
>          if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
>              self.assertTrue(os.path.exists(os.path.join(repo_path,
> 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule
> missing')
>  
> +    def test_git_submodule_dbus_broker(self):
>          # The following external repositories have show failures in
> fetch and unpack operations
>          # We want to avoid regressions!
>          url =
> "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"
> @@ -928,16 +929,35 @@ class FetcherNetworkTest(FetcherTest):
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule
> config "subprojects/c-sundry"')
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule
> config "subprojects/c-utf8"')
>  
> +    def test_git_submodule_CLI11(self):
>          url =
> "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
>          fetcher = bb.fetch.Fetch([url], self.d)
>          fetcher.download()
>          # Previous cwd has been deleted
>          os.chdir(os.path.dirname(self.unpackdir))
>          fetcher.unpack(self.unpackdir)
> +
> +        repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/extern/googletest/config')), msg='Missing submodule
> config "extern/googletest"')
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/extern/json/config')), msg='Missing submodule config
> "extern/json"')
>          self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/extern/sanitizers/config')), msg='Missing submodule
> config "extern/sanitizers"')
>  
> +    def test_git_submodule_aktualizr(self):
> +        url =
> "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
> +        fetcher = bb.fetch.Fetch([url], self.d)
> +        fetcher.download()
> +        # Previous cwd has been deleted
> +        os.chdir(os.path.dirname(self.unpackdir))
> +        fetcher.unpack(self.unpackdir)
> +
> +        repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule
> config "partial/extern/isotp-c/config"')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')),
> msg='Missing submodule config
> "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of
> submodule isotp-c did not unpack properly")
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule
> config "tests/tuf-test-vectors/config"')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/third_party/googletest/config')), msg='Missing submodule
> config "third_party/googletest/config"')
> +        self.assertTrue(os.path.exists(os.path.join(repo_path,
> '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing
> submodule config "third_party/HdrHistogram_c/config"')
> +
>  class TrustedNetworksTest(FetcherTest):
>      def test_trusted_network(self):
>          # Ensure trusted_network returns False when the host IS in the list.


^ permalink raw reply	[flat|nested] 3+ messages in thread
end of thread, other threads:[~2019-01-28 19:38 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-01-27 16:57 [PATCH]gitsm fetcher: Fix recurse submodule error Mark Hatle
2019-01-27 16:57 ` [PATCH] gitsm: The fetcher did not process some recursive submodules properly Mark Hatle
2019-01-28 19:38   ` Stefan Agner

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.