* [PATCH v2] fetch2/gitsm: Unpack shallow mirror tarballs
@ 2020-03-09 13:42 Paul Barker
2020-03-09 13:45 ` Paul Barker
2020-07-21 9:03 ` (EXT) [bitbake-devel] " Matthias Schiffer
0 siblings, 2 replies; 3+ messages in thread
From: Paul Barker @ 2020-03-09 13:42 UTC (permalink / raw)
To: bitbake-devel
When a shallow mirror tarball is used to satisfy a gitsm URI it needs to
be unpacked temporarily so that the .gitmodules file can be examined.
Signed-off-by: Paul Barker <pbarker@konsulko.com>
---
lib/bb/fetch2/gitsm.py | 21 ++++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index aa121cbe..e7083001 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -20,6 +20,8 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r
import os
import bb
import copy
+import shutil
+import tempfile
from bb.fetch2.git import Git
from bb.fetch2 import runfetchcmd
from bb.fetch2 import logger
@@ -130,7 +132,7 @@ class GitSM(Git):
ld.setVar('SRCPV', d.getVar('SRCPV'))
ld.setVar('SRCREV_FORMAT', module)
- function(ud, url, module, paths[module], ld)
+ function(ud, url, module, paths[module], workdir, ld)
return submodules != []
@@ -152,7 +154,7 @@ class GitSM(Git):
return False
def download(self, ud, d):
- def download_submodule(ud, url, module, modpath, d):
+ def download_submodule(ud, url, module, modpath, workdir, d):
url += ";bareclone=1;nobranch=1"
# Is the following still needed?
@@ -163,16 +165,25 @@ class GitSM(Git):
newfetch.download()
# Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
runfetchcmd("%s config --add bitbake.srcrev %s" % \
- (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
+ (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=workdir)
except Exception as e:
logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
raise
Git.download(self, ud, d)
- self.process_submodules(ud, ud.clonedir, download_submodule, d)
+
+ # If we're using a shallow mirror tarball it needs to be unpacked
+ # temporarily so that we can examine the .gitmodules file
+ if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
+ tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+ runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+ self.process_submodules(ud, tmpdir, download_submodule, d)
+ shutil.rmtree(tmpdir)
+ else:
+ self.process_submodules(ud, ud.clonedir, download_submodule, d)
def unpack(self, ud, destdir, d):
- def unpack_submodules(ud, url, module, modpath, d):
+ def unpack_submodules(ud, url, module, modpath, workdir, d):
url += ";bareclone=1;nobranch=1"
# Figure out where we clone over the bare submodules...
--
2.20.1
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH v2] fetch2/gitsm: Unpack shallow mirror tarballs
2020-03-09 13:42 [PATCH v2] fetch2/gitsm: Unpack shallow mirror tarballs Paul Barker
@ 2020-03-09 13:45 ` Paul Barker
2020-07-21 9:03 ` (EXT) [bitbake-devel] " Matthias Schiffer
1 sibling, 0 replies; 3+ messages in thread
From: Paul Barker @ 2020-03-09 13:45 UTC (permalink / raw)
To: bitbake-devel
[-- Attachment #1: Type: text/plain, Size: 3351 bytes --]
Sorry, I missed the new test when copying this from a poky checkout to a
bitbake checkout to generate the patch. I'll send the test as a follow-up.
On Mon, 9 Mar 2020 at 13:43, Paul Barker <pbarker@konsulko.com> wrote:
> When a shallow mirror tarball is used to satisfy a gitsm URI it needs to
> be unpacked temporarily so that the .gitmodules file can be examined.
>
> Signed-off-by: Paul Barker <pbarker@konsulko.com>
> ---
> lib/bb/fetch2/gitsm.py | 21 ++++++++++++++++-----
> 1 file changed, 16 insertions(+), 5 deletions(-)
>
> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> index aa121cbe..e7083001 100644
> --- a/lib/bb/fetch2/gitsm.py
> +++ b/lib/bb/fetch2/gitsm.py
> @@ -20,6 +20,8 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://"
> requires a clean of your r
> import os
> import bb
> import copy
> +import shutil
> +import tempfile
> from bb.fetch2.git import Git
> from bb.fetch2 import runfetchcmd
> from bb.fetch2 import logger
> @@ -130,7 +132,7 @@ class GitSM(Git):
> ld.setVar('SRCPV', d.getVar('SRCPV'))
> ld.setVar('SRCREV_FORMAT', module)
>
> - function(ud, url, module, paths[module], ld)
> + function(ud, url, module, paths[module], workdir, ld)
>
> return submodules != []
>
> @@ -152,7 +154,7 @@ class GitSM(Git):
> return False
>
> def download(self, ud, d):
> - def download_submodule(ud, url, module, modpath, d):
> + def download_submodule(ud, url, module, modpath, workdir, d):
> url += ";bareclone=1;nobranch=1"
>
> # Is the following still needed?
> @@ -163,16 +165,25 @@ class GitSM(Git):
> newfetch.download()
> # Drop a nugget to add each of the srcrevs we've fetched
> (used by need_update)
> runfetchcmd("%s config --add bitbake.srcrev %s" % \
> - (ud.basecmd, ud.revisions[ud.names[0]]), d,
> workdir=ud.clonedir)
> + (ud.basecmd, ud.revisions[ud.names[0]]), d,
> workdir=workdir)
> except Exception as e:
> logger.error('gitsm: submodule download failed: %s %s' %
> (type(e).__name__, str(e)))
> raise
>
> Git.download(self, ud, d)
> - self.process_submodules(ud, ud.clonedir, download_submodule, d)
> +
> + # If we're using a shallow mirror tarball it needs to be unpacked
> + # temporarily so that we can examine the .gitmodules file
> + if ud.shallow and os.path.exists(ud.fullshallow) and
> self.need_update(ud, d):
> + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
> + runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
> + self.process_submodules(ud, tmpdir, download_submodule, d)
> + shutil.rmtree(tmpdir)
> + else:
> + self.process_submodules(ud, ud.clonedir, download_submodule,
> d)
>
> def unpack(self, ud, destdir, d):
> - def unpack_submodules(ud, url, module, modpath, d):
> + def unpack_submodules(ud, url, module, modpath, workdir, d):
> url += ";bareclone=1;nobranch=1"
>
> # Figure out where we clone over the bare submodules...
> --
> 2.20.1
>
>
[-- Attachment #2: Type: text/html, Size: 4169 bytes --]
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: (EXT) [bitbake-devel] [PATCH v2] fetch2/gitsm: Unpack shallow mirror tarballs
2020-03-09 13:42 [PATCH v2] fetch2/gitsm: Unpack shallow mirror tarballs Paul Barker
2020-03-09 13:45 ` Paul Barker
@ 2020-07-21 9:03 ` Matthias Schiffer
1 sibling, 0 replies; 3+ messages in thread
From: Matthias Schiffer @ 2020-07-21 9:03 UTC (permalink / raw)
To: Paul Barker; +Cc: bitbake-devel
On Mon, 2020-03-09 at 13:42 +0000, Paul Barker wrote:
> When a shallow mirror tarball is used to satisfy a gitsm URI it needs
> to
> be unpacked temporarily so that the .gitmodules file can be examined.
>
> Signed-off-by: Paul Barker <pbarker@konsulko.com>
Hi,
can we get this backported to the 1.44 branch?
Kind regards,
Matthias
> ---
> lib/bb/fetch2/gitsm.py | 21 ++++++++++++++++-----
> 1 file changed, 16 insertions(+), 5 deletions(-)
>
> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> index aa121cbe..e7083001 100644
> --- a/lib/bb/fetch2/gitsm.py
> +++ b/lib/bb/fetch2/gitsm.py
> @@ -20,6 +20,8 @@ NOTE: Switching a SRC_URI from "git://" to
> "gitsm://" requires a clean of your r
> import os
> import bb
> import copy
> +import shutil
> +import tempfile
> from bb.fetch2.git import Git
> from bb.fetch2 import runfetchcmd
> from bb.fetch2 import logger
> @@ -130,7 +132,7 @@ class GitSM(Git):
> ld.setVar('SRCPV', d.getVar('SRCPV'))
> ld.setVar('SRCREV_FORMAT', module)
>
> - function(ud, url, module, paths[module], ld)
> + function(ud, url, module, paths[module], workdir, ld)
>
> return submodules != []
>
> @@ -152,7 +154,7 @@ class GitSM(Git):
> return False
>
> def download(self, ud, d):
> - def download_submodule(ud, url, module, modpath, d):
> + def download_submodule(ud, url, module, modpath, workdir,
> d):
> url += ";bareclone=1;nobranch=1"
>
> # Is the following still needed?
> @@ -163,16 +165,25 @@ class GitSM(Git):
> newfetch.download()
> # Drop a nugget to add each of the srcrevs we've
> fetched (used by need_update)
> runfetchcmd("%s config --add bitbake.srcrev %s" % \
> - (ud.basecmd, ud.revisions[ud.names[0]]),
> d, workdir=ud.clonedir)
> + (ud.basecmd, ud.revisions[ud.names[0]]),
> d, workdir=workdir)
> except Exception as e:
> logger.error('gitsm: submodule download failed: %s
> %s' % (type(e).__name__, str(e)))
> raise
>
> Git.download(self, ud, d)
> - self.process_submodules(ud, ud.clonedir, download_submodule,
> d)
> +
> + # If we're using a shallow mirror tarball it needs to be
> unpacked
> + # temporarily so that we can examine the .gitmodules file
> + if ud.shallow and os.path.exists(ud.fullshallow) and
> self.need_update(ud, d):
> + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
> + runfetchcmd("tar -xzf %s" % ud.fullshallow, d,
> workdir=tmpdir)
> + self.process_submodules(ud, tmpdir, download_submodule,
> d)
> + shutil.rmtree(tmpdir)
> + else:
> + self.process_submodules(ud, ud.clonedir,
> download_submodule, d)
>
> def unpack(self, ud, destdir, d):
> - def unpack_submodules(ud, url, module, modpath, d):
> + def unpack_submodules(ud, url, module, modpath, workdir, d):
> url += ";bareclone=1;nobranch=1"
>
> # Figure out where we clone over the bare submodules...
> --
> 2.20.1
>
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2020-07-21 9:03 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-03-09 13:42 [PATCH v2] fetch2/gitsm: Unpack shallow mirror tarballs Paul Barker
2020-03-09 13:45 ` Paul Barker
2020-07-21 9:03 ` (EXT) [bitbake-devel] " Matthias Schiffer
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.