* [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-15 21:31 [PATCH 0/7 v2] gitsm.py: submodule init and ssh url processing Mark Hatle
@ 2019-01-15 21:31 ` Mark Hatle
2019-01-18 15:07 ` Khem Raj
2019-01-15 21:31 ` [PATCH 2/7] gitsm.py: Add support for alternative URL formats from submodule files Mark Hatle
` (5 subsequent siblings)
6 siblings, 1 reply; 17+ messages in thread
From: Mark Hatle @ 2019-01-15 21:31 UTC (permalink / raw)
To: bitbake-devel
It is possible for a submodule to be defined in the .gitmodules file, but
never initialized in the repository itself. This shows itself when searching
for the defined module hash you will get back a empty value.
Similarly we need to identify and skip defined but not initialized submodules
during the unpack stages as well.
Thanks to raphael.lisicki@siemens.com for their help is figuring out how
to resolve this issue.
Additionally a problem was found where, while unlikely, it may be possible
for the wrong revision to have been searched using ls-tree. This has been
resolved in the update_submodules function by keeping the correct revision
along with the submodule path.
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
1 file changed, 28 insertions(+), 4 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index 35729db..b7959ff 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -64,6 +64,7 @@ class GitSM(Git):
def update_submodules(self, ud, d):
submodules = []
paths = {}
+ revision = {}
uris = {}
local_paths = {}
@@ -77,6 +78,7 @@ class GitSM(Git):
for m, md in self.parse_gitmodules(gitmodules).items():
submodules.append(m)
paths[m] = md['path']
+ revision[m] = ud.revisions[name]
uris[m] = md['url']
if uris[m].startswith('..'):
newud = copy.copy(ud)
@@ -84,7 +86,17 @@ class GitSM(Git):
uris[m] = Git._get_repo_url(self, newud)
for module in submodules:
- module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
+ try:
+ module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
+ except:
+ # If the command fails, we don't have a valid file to check. If it doesn't
+ # fail -- it still might be a failure, see next check...
+ module_hash = ""
+
+ if not module_hash:
+ logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
+ continue
+
module_hash = module_hash.split()[2]
# Build new SRC_URI
@@ -143,7 +155,7 @@ class GitSM(Git):
if not ud.shallow or ud.localpath != ud.fullshallow:
self.update_submodules(ud, d)
- def copy_submodules(self, submodules, ud, destdir, d):
+ def copy_submodules(self, submodules, ud, name, destdir, d):
if ud.bareclone:
repo_conf = destdir
else:
@@ -156,6 +168,18 @@ class GitSM(Git):
srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
modpath = os.path.join(repo_conf, 'modules', md['path'])
+ # Check if the module is initialized
+ try:
+ module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
+ except:
+ # If the command fails, we don't have a valid file to check. If it doesn't
+ # fail -- it still might be a failure, see next check...
+ module_hash = ""
+
+ if not module_hash:
+ logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
+ continue
+
if os.path.exists(srcpath):
if os.path.exists(os.path.join(srcpath, '.git')):
srcpath = os.path.join(srcpath, '.git')
@@ -188,7 +212,7 @@ class GitSM(Git):
continue
submodules = self.parse_gitmodules(gitmodules)
- self.copy_submodules(submodules, ud, dest, d)
+ self.copy_submodules(submodules, ud, name, dest, d)
def unpack(self, ud, destdir, d):
Git.unpack(self, ud, destdir, d)
@@ -211,7 +235,7 @@ class GitSM(Git):
continue
submodules = self.parse_gitmodules(gitmodules)
- self.copy_submodules(submodules, ud, ud.destdir, d)
+ self.copy_submodules(submodules, ud, name, ud.destdir, d)
submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
while len(submodules_queue) != 0:
--
1.8.3.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-15 21:31 ` [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized Mark Hatle
@ 2019-01-18 15:07 ` Khem Raj
2019-01-18 18:07 ` Mark Hatle
0 siblings, 1 reply; 17+ messages in thread
From: Khem Raj @ 2019-01-18 15:07 UTC (permalink / raw)
To: Mark Hatle; +Cc: bitbake-devel
I am seeing
http://errors.yoctoproject.org/Errors/Details/217408/
http://errors.yoctoproject.org/Errors/Details/217412/
On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>
> It is possible for a submodule to be defined in the .gitmodules file, but
> never initialized in the repository itself. This shows itself when searching
> for the defined module hash you will get back a empty value.
>
> Similarly we need to identify and skip defined but not initialized submodules
> during the unpack stages as well.
>
> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
> to resolve this issue.
>
> Additionally a problem was found where, while unlikely, it may be possible
> for the wrong revision to have been searched using ls-tree. This has been
> resolved in the update_submodules function by keeping the correct revision
> along with the submodule path.
>
> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
> ---
> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
> 1 file changed, 28 insertions(+), 4 deletions(-)
>
> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> index 35729db..b7959ff 100644
> --- a/lib/bb/fetch2/gitsm.py
> +++ b/lib/bb/fetch2/gitsm.py
> @@ -64,6 +64,7 @@ class GitSM(Git):
> def update_submodules(self, ud, d):
> submodules = []
> paths = {}
> + revision = {}
> uris = {}
> local_paths = {}
>
> @@ -77,6 +78,7 @@ class GitSM(Git):
> for m, md in self.parse_gitmodules(gitmodules).items():
> submodules.append(m)
> paths[m] = md['path']
> + revision[m] = ud.revisions[name]
> uris[m] = md['url']
> if uris[m].startswith('..'):
> newud = copy.copy(ud)
> @@ -84,7 +86,17 @@ class GitSM(Git):
> uris[m] = Git._get_repo_url(self, newud)
>
> for module in submodules:
> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
> + try:
> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
> + except:
> + # If the command fails, we don't have a valid file to check. If it doesn't
> + # fail -- it still might be a failure, see next check...
> + module_hash = ""
> +
> + if not module_hash:
> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> + continue
> +
> module_hash = module_hash.split()[2]
>
> # Build new SRC_URI
> @@ -143,7 +155,7 @@ class GitSM(Git):
> if not ud.shallow or ud.localpath != ud.fullshallow:
> self.update_submodules(ud, d)
>
> - def copy_submodules(self, submodules, ud, destdir, d):
> + def copy_submodules(self, submodules, ud, name, destdir, d):
> if ud.bareclone:
> repo_conf = destdir
> else:
> @@ -156,6 +168,18 @@ class GitSM(Git):
> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
> modpath = os.path.join(repo_conf, 'modules', md['path'])
>
> + # Check if the module is initialized
> + try:
> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
> + except:
> + # If the command fails, we don't have a valid file to check. If it doesn't
> + # fail -- it still might be a failure, see next check...
> + module_hash = ""
> +
> + if not module_hash:
> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> + continue
> +
> if os.path.exists(srcpath):
> if os.path.exists(os.path.join(srcpath, '.git')):
> srcpath = os.path.join(srcpath, '.git')
> @@ -188,7 +212,7 @@ class GitSM(Git):
> continue
>
> submodules = self.parse_gitmodules(gitmodules)
> - self.copy_submodules(submodules, ud, dest, d)
> + self.copy_submodules(submodules, ud, name, dest, d)
>
> def unpack(self, ud, destdir, d):
> Git.unpack(self, ud, destdir, d)
> @@ -211,7 +235,7 @@ class GitSM(Git):
> continue
>
> submodules = self.parse_gitmodules(gitmodules)
> - self.copy_submodules(submodules, ud, ud.destdir, d)
> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
>
> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
> while len(submodules_queue) != 0:
> --
> 1.8.3.1
>
> --
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-18 15:07 ` Khem Raj
@ 2019-01-18 18:07 ` Mark Hatle
2019-01-18 18:31 ` Khem Raj
0 siblings, 1 reply; 17+ messages in thread
From: Mark Hatle @ 2019-01-18 18:07 UTC (permalink / raw)
To: Khem Raj; +Cc: bitbake-devel
On 1/18/19 9:07 AM, Khem Raj wrote:
> I am seeing
>
> http://errors.yoctoproject.org/Errors/Details/217408/
> http://errors.yoctoproject.org/Errors/Details/217412/
Do you know where these failing recipes are so that I can reproduce it locally?
rwmem looks like it might be meta-ti?
cli11 looks like something from meta-oe.
--Mark
> On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>>
>> It is possible for a submodule to be defined in the .gitmodules file, but
>> never initialized in the repository itself. This shows itself when searching
>> for the defined module hash you will get back a empty value.
>>
>> Similarly we need to identify and skip defined but not initialized submodules
>> during the unpack stages as well.
>>
>> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
>> to resolve this issue.
>>
>> Additionally a problem was found where, while unlikely, it may be possible
>> for the wrong revision to have been searched using ls-tree. This has been
>> resolved in the update_submodules function by keeping the correct revision
>> along with the submodule path.
>>
>> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
>> ---
>> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
>> 1 file changed, 28 insertions(+), 4 deletions(-)
>>
>> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
>> index 35729db..b7959ff 100644
>> --- a/lib/bb/fetch2/gitsm.py
>> +++ b/lib/bb/fetch2/gitsm.py
>> @@ -64,6 +64,7 @@ class GitSM(Git):
>> def update_submodules(self, ud, d):
>> submodules = []
>> paths = {}
>> + revision = {}
>> uris = {}
>> local_paths = {}
>>
>> @@ -77,6 +78,7 @@ class GitSM(Git):
>> for m, md in self.parse_gitmodules(gitmodules).items():
>> submodules.append(m)
>> paths[m] = md['path']
>> + revision[m] = ud.revisions[name]
>> uris[m] = md['url']
>> if uris[m].startswith('..'):
>> newud = copy.copy(ud)
>> @@ -84,7 +86,17 @@ class GitSM(Git):
>> uris[m] = Git._get_repo_url(self, newud)
>>
>> for module in submodules:
>> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
>> + try:
>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
>> + except:
>> + # If the command fails, we don't have a valid file to check. If it doesn't
>> + # fail -- it still might be a failure, see next check...
>> + module_hash = ""
>> +
>> + if not module_hash:
>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>> + continue
>> +
>> module_hash = module_hash.split()[2]
>>
>> # Build new SRC_URI
>> @@ -143,7 +155,7 @@ class GitSM(Git):
>> if not ud.shallow or ud.localpath != ud.fullshallow:
>> self.update_submodules(ud, d)
>>
>> - def copy_submodules(self, submodules, ud, destdir, d):
>> + def copy_submodules(self, submodules, ud, name, destdir, d):
>> if ud.bareclone:
>> repo_conf = destdir
>> else:
>> @@ -156,6 +168,18 @@ class GitSM(Git):
>> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
>> modpath = os.path.join(repo_conf, 'modules', md['path'])
>>
>> + # Check if the module is initialized
>> + try:
>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
>> + except:
>> + # If the command fails, we don't have a valid file to check. If it doesn't
>> + # fail -- it still might be a failure, see next check...
>> + module_hash = ""
>> +
>> + if not module_hash:
>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>> + continue
>> +
>> if os.path.exists(srcpath):
>> if os.path.exists(os.path.join(srcpath, '.git')):
>> srcpath = os.path.join(srcpath, '.git')
>> @@ -188,7 +212,7 @@ class GitSM(Git):
>> continue
>>
>> submodules = self.parse_gitmodules(gitmodules)
>> - self.copy_submodules(submodules, ud, dest, d)
>> + self.copy_submodules(submodules, ud, name, dest, d)
>>
>> def unpack(self, ud, destdir, d):
>> Git.unpack(self, ud, destdir, d)
>> @@ -211,7 +235,7 @@ class GitSM(Git):
>> continue
>>
>> submodules = self.parse_gitmodules(gitmodules)
>> - self.copy_submodules(submodules, ud, ud.destdir, d)
>> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
>>
>> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
>> while len(submodules_queue) != 0:
>> --
>> 1.8.3.1
>>
>> --
>> _______________________________________________
>> bitbake-devel mailing list
>> bitbake-devel@lists.openembedded.org
>> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-18 18:07 ` Mark Hatle
@ 2019-01-18 18:31 ` Khem Raj
2019-01-18 21:20 ` Mark Hatle
0 siblings, 1 reply; 17+ messages in thread
From: Khem Raj @ 2019-01-18 18:31 UTC (permalink / raw)
To: Mark Hatle; +Cc: bitbake-devel
On Fri, Jan 18, 2019 at 10:08 AM Mark Hatle <mark.hatle@windriver.com> wrote:
>
> On 1/18/19 9:07 AM, Khem Raj wrote:
> > I am seeing
> >
> > http://errors.yoctoproject.org/Errors/Details/217408/
> > http://errors.yoctoproject.org/Errors/Details/217412/
>
> Do you know where these failing recipes are so that I can reproduce it locally?
>
> rwmem looks like it might be meta-ti?
> cli11 looks like something from meta-oe.
>
yes thats right.
http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-support/cli11/cli11_1.6.2.bb?h=master
gitsm version of rwmem is not yes applied in meta-ti so you might have
to get it from my staging branch
https://github.com/YoeDistro/meta-ti/commit/3418beda1869882d5f94dc405d2727f09ffe20c4
> --Mark
>
> > On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
> >>
> >> It is possible for a submodule to be defined in the .gitmodules file, but
> >> never initialized in the repository itself. This shows itself when searching
> >> for the defined module hash you will get back a empty value.
> >>
> >> Similarly we need to identify and skip defined but not initialized submodules
> >> during the unpack stages as well.
> >>
> >> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
> >> to resolve this issue.
> >>
> >> Additionally a problem was found where, while unlikely, it may be possible
> >> for the wrong revision to have been searched using ls-tree. This has been
> >> resolved in the update_submodules function by keeping the correct revision
> >> along with the submodule path.
> >>
> >> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
> >> ---
> >> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
> >> 1 file changed, 28 insertions(+), 4 deletions(-)
> >>
> >> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> >> index 35729db..b7959ff 100644
> >> --- a/lib/bb/fetch2/gitsm.py
> >> +++ b/lib/bb/fetch2/gitsm.py
> >> @@ -64,6 +64,7 @@ class GitSM(Git):
> >> def update_submodules(self, ud, d):
> >> submodules = []
> >> paths = {}
> >> + revision = {}
> >> uris = {}
> >> local_paths = {}
> >>
> >> @@ -77,6 +78,7 @@ class GitSM(Git):
> >> for m, md in self.parse_gitmodules(gitmodules).items():
> >> submodules.append(m)
> >> paths[m] = md['path']
> >> + revision[m] = ud.revisions[name]
> >> uris[m] = md['url']
> >> if uris[m].startswith('..'):
> >> newud = copy.copy(ud)
> >> @@ -84,7 +86,17 @@ class GitSM(Git):
> >> uris[m] = Git._get_repo_url(self, newud)
> >>
> >> for module in submodules:
> >> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
> >> + try:
> >> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
> >> + except:
> >> + # If the command fails, we don't have a valid file to check. If it doesn't
> >> + # fail -- it still might be a failure, see next check...
> >> + module_hash = ""
> >> +
> >> + if not module_hash:
> >> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> >> + continue
> >> +
> >> module_hash = module_hash.split()[2]
> >>
> >> # Build new SRC_URI
> >> @@ -143,7 +155,7 @@ class GitSM(Git):
> >> if not ud.shallow or ud.localpath != ud.fullshallow:
> >> self.update_submodules(ud, d)
> >>
> >> - def copy_submodules(self, submodules, ud, destdir, d):
> >> + def copy_submodules(self, submodules, ud, name, destdir, d):
> >> if ud.bareclone:
> >> repo_conf = destdir
> >> else:
> >> @@ -156,6 +168,18 @@ class GitSM(Git):
> >> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
> >> modpath = os.path.join(repo_conf, 'modules', md['path'])
> >>
> >> + # Check if the module is initialized
> >> + try:
> >> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
> >> + except:
> >> + # If the command fails, we don't have a valid file to check. If it doesn't
> >> + # fail -- it still might be a failure, see next check...
> >> + module_hash = ""
> >> +
> >> + if not module_hash:
> >> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> >> + continue
> >> +
> >> if os.path.exists(srcpath):
> >> if os.path.exists(os.path.join(srcpath, '.git')):
> >> srcpath = os.path.join(srcpath, '.git')
> >> @@ -188,7 +212,7 @@ class GitSM(Git):
> >> continue
> >>
> >> submodules = self.parse_gitmodules(gitmodules)
> >> - self.copy_submodules(submodules, ud, dest, d)
> >> + self.copy_submodules(submodules, ud, name, dest, d)
> >>
> >> def unpack(self, ud, destdir, d):
> >> Git.unpack(self, ud, destdir, d)
> >> @@ -211,7 +235,7 @@ class GitSM(Git):
> >> continue
> >>
> >> submodules = self.parse_gitmodules(gitmodules)
> >> - self.copy_submodules(submodules, ud, ud.destdir, d)
> >> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
> >>
> >> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
> >> while len(submodules_queue) != 0:
> >> --
> >> 1.8.3.1
> >>
> >> --
> >> _______________________________________________
> >> bitbake-devel mailing list
> >> bitbake-devel@lists.openembedded.org
> >> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-18 18:31 ` Khem Raj
@ 2019-01-18 21:20 ` Mark Hatle
2019-01-18 23:06 ` Mark Hatle
0 siblings, 1 reply; 17+ messages in thread
From: Mark Hatle @ 2019-01-18 21:20 UTC (permalink / raw)
To: Khem Raj; +Cc: bitbake-devel
On 1/18/19 12:31 PM, Khem Raj wrote:
> On Fri, Jan 18, 2019 at 10:08 AM Mark Hatle <mark.hatle@windriver.com> wrote:
>>
>> On 1/18/19 9:07 AM, Khem Raj wrote:
>>> I am seeing
>>>
>>> http://errors.yoctoproject.org/Errors/Details/217408/
>>> http://errors.yoctoproject.org/Errors/Details/217412/
>>
>> Do you know where these failing recipes are so that I can reproduce it locally?
>>
>> rwmem looks like it might be meta-ti?
>> cli11 looks like something from meta-oe.
>>
>
> yes thats right.
> http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-support/cli11/cli11_1.6.2.bb?h=master
>
> gitsm version of rwmem is not yes applied in meta-ti so you might have
> to get it from my staging branch
>
> https://github.com/YoeDistro/meta-ti/commit/3418beda1869882d5f94dc405d2727f09ffe20c4
The reproducer appears to be that these repositories are 'git' based, but use
relative submodule locations from the main repository.
So now the gitsm code needs to resolve relative URLs, which is something not
previously implemented.
--Mark
>> --Mark
>>
>>> On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>>>>
>>>> It is possible for a submodule to be defined in the .gitmodules file, but
>>>> never initialized in the repository itself. This shows itself when searching
>>>> for the defined module hash you will get back a empty value.
>>>>
>>>> Similarly we need to identify and skip defined but not initialized submodules
>>>> during the unpack stages as well.
>>>>
>>>> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
>>>> to resolve this issue.
>>>>
>>>> Additionally a problem was found where, while unlikely, it may be possible
>>>> for the wrong revision to have been searched using ls-tree. This has been
>>>> resolved in the update_submodules function by keeping the correct revision
>>>> along with the submodule path.
>>>>
>>>> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
>>>> ---
>>>> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
>>>> 1 file changed, 28 insertions(+), 4 deletions(-)
>>>>
>>>> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
>>>> index 35729db..b7959ff 100644
>>>> --- a/lib/bb/fetch2/gitsm.py
>>>> +++ b/lib/bb/fetch2/gitsm.py
>>>> @@ -64,6 +64,7 @@ class GitSM(Git):
>>>> def update_submodules(self, ud, d):
>>>> submodules = []
>>>> paths = {}
>>>> + revision = {}
>>>> uris = {}
>>>> local_paths = {}
>>>>
>>>> @@ -77,6 +78,7 @@ class GitSM(Git):
>>>> for m, md in self.parse_gitmodules(gitmodules).items():
>>>> submodules.append(m)
>>>> paths[m] = md['path']
>>>> + revision[m] = ud.revisions[name]
>>>> uris[m] = md['url']
>>>> if uris[m].startswith('..'):
>>>> newud = copy.copy(ud)
>>>> @@ -84,7 +86,17 @@ class GitSM(Git):
>>>> uris[m] = Git._get_repo_url(self, newud)
>>>>
>>>> for module in submodules:
>>>> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
>>>> + try:
>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
>>>> + except:
>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
>>>> + # fail -- it still might be a failure, see next check...
>>>> + module_hash = ""
>>>> +
>>>> + if not module_hash:
>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>>>> + continue
>>>> +
>>>> module_hash = module_hash.split()[2]
>>>>
>>>> # Build new SRC_URI
>>>> @@ -143,7 +155,7 @@ class GitSM(Git):
>>>> if not ud.shallow or ud.localpath != ud.fullshallow:
>>>> self.update_submodules(ud, d)
>>>>
>>>> - def copy_submodules(self, submodules, ud, destdir, d):
>>>> + def copy_submodules(self, submodules, ud, name, destdir, d):
>>>> if ud.bareclone:
>>>> repo_conf = destdir
>>>> else:
>>>> @@ -156,6 +168,18 @@ class GitSM(Git):
>>>> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
>>>> modpath = os.path.join(repo_conf, 'modules', md['path'])
>>>>
>>>> + # Check if the module is initialized
>>>> + try:
>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
>>>> + except:
>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
>>>> + # fail -- it still might be a failure, see next check...
>>>> + module_hash = ""
>>>> +
>>>> + if not module_hash:
>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>>>> + continue
>>>> +
>>>> if os.path.exists(srcpath):
>>>> if os.path.exists(os.path.join(srcpath, '.git')):
>>>> srcpath = os.path.join(srcpath, '.git')
>>>> @@ -188,7 +212,7 @@ class GitSM(Git):
>>>> continue
>>>>
>>>> submodules = self.parse_gitmodules(gitmodules)
>>>> - self.copy_submodules(submodules, ud, dest, d)
>>>> + self.copy_submodules(submodules, ud, name, dest, d)
>>>>
>>>> def unpack(self, ud, destdir, d):
>>>> Git.unpack(self, ud, destdir, d)
>>>> @@ -211,7 +235,7 @@ class GitSM(Git):
>>>> continue
>>>>
>>>> submodules = self.parse_gitmodules(gitmodules)
>>>> - self.copy_submodules(submodules, ud, ud.destdir, d)
>>>> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
>>>>
>>>> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
>>>> while len(submodules_queue) != 0:
>>>> --
>>>> 1.8.3.1
>>>>
>>>> --
>>>> _______________________________________________
>>>> bitbake-devel mailing list
>>>> bitbake-devel@lists.openembedded.org
>>>> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
>>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-18 21:20 ` Mark Hatle
@ 2019-01-18 23:06 ` Mark Hatle
2019-01-19 1:27 ` Khem Raj
0 siblings, 1 reply; 17+ messages in thread
From: Mark Hatle @ 2019-01-18 23:06 UTC (permalink / raw)
To: Khem Raj; +Cc: bitbake-devel
I think I have a fix for this. Can you try the bitbake available from:
git://git.openembedded.org/bitbake-contrib
branch mgh/gitsm
The commit that resolves this issue is:
0a7d8bd99a5ba394b00ed35593101b4cd324a58d
--Mark
On 1/18/19 3:20 PM, Mark Hatle wrote:
> On 1/18/19 12:31 PM, Khem Raj wrote:
>> On Fri, Jan 18, 2019 at 10:08 AM Mark Hatle <mark.hatle@windriver.com> wrote:
>>>
>>> On 1/18/19 9:07 AM, Khem Raj wrote:
>>>> I am seeing
>>>>
>>>> http://errors.yoctoproject.org/Errors/Details/217408/
>>>> http://errors.yoctoproject.org/Errors/Details/217412/
>>>
>>> Do you know where these failing recipes are so that I can reproduce it locally?
>>>
>>> rwmem looks like it might be meta-ti?
>>> cli11 looks like something from meta-oe.
>>>
>>
>> yes thats right.
>> http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-support/cli11/cli11_1.6.2.bb?h=master
>>
>> gitsm version of rwmem is not yes applied in meta-ti so you might have
>> to get it from my staging branch
>>
>> https://github.com/YoeDistro/meta-ti/commit/3418beda1869882d5f94dc405d2727f09ffe20c4
>
> The reproducer appears to be that these repositories are 'git' based, but use
> relative submodule locations from the main repository.
>
> So now the gitsm code needs to resolve relative URLs, which is something not
> previously implemented.
>
> --Mark
>
>>> --Mark
>>>
>>>> On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>>>>>
>>>>> It is possible for a submodule to be defined in the .gitmodules file, but
>>>>> never initialized in the repository itself. This shows itself when searching
>>>>> for the defined module hash you will get back a empty value.
>>>>>
>>>>> Similarly we need to identify and skip defined but not initialized submodules
>>>>> during the unpack stages as well.
>>>>>
>>>>> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
>>>>> to resolve this issue.
>>>>>
>>>>> Additionally a problem was found where, while unlikely, it may be possible
>>>>> for the wrong revision to have been searched using ls-tree. This has been
>>>>> resolved in the update_submodules function by keeping the correct revision
>>>>> along with the submodule path.
>>>>>
>>>>> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
>>>>> ---
>>>>> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
>>>>> 1 file changed, 28 insertions(+), 4 deletions(-)
>>>>>
>>>>> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
>>>>> index 35729db..b7959ff 100644
>>>>> --- a/lib/bb/fetch2/gitsm.py
>>>>> +++ b/lib/bb/fetch2/gitsm.py
>>>>> @@ -64,6 +64,7 @@ class GitSM(Git):
>>>>> def update_submodules(self, ud, d):
>>>>> submodules = []
>>>>> paths = {}
>>>>> + revision = {}
>>>>> uris = {}
>>>>> local_paths = {}
>>>>>
>>>>> @@ -77,6 +78,7 @@ class GitSM(Git):
>>>>> for m, md in self.parse_gitmodules(gitmodules).items():
>>>>> submodules.append(m)
>>>>> paths[m] = md['path']
>>>>> + revision[m] = ud.revisions[name]
>>>>> uris[m] = md['url']
>>>>> if uris[m].startswith('..'):
>>>>> newud = copy.copy(ud)
>>>>> @@ -84,7 +86,17 @@ class GitSM(Git):
>>>>> uris[m] = Git._get_repo_url(self, newud)
>>>>>
>>>>> for module in submodules:
>>>>> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
>>>>> + try:
>>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
>>>>> + except:
>>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
>>>>> + # fail -- it still might be a failure, see next check...
>>>>> + module_hash = ""
>>>>> +
>>>>> + if not module_hash:
>>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>>>>> + continue
>>>>> +
>>>>> module_hash = module_hash.split()[2]
>>>>>
>>>>> # Build new SRC_URI
>>>>> @@ -143,7 +155,7 @@ class GitSM(Git):
>>>>> if not ud.shallow or ud.localpath != ud.fullshallow:
>>>>> self.update_submodules(ud, d)
>>>>>
>>>>> - def copy_submodules(self, submodules, ud, destdir, d):
>>>>> + def copy_submodules(self, submodules, ud, name, destdir, d):
>>>>> if ud.bareclone:
>>>>> repo_conf = destdir
>>>>> else:
>>>>> @@ -156,6 +168,18 @@ class GitSM(Git):
>>>>> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
>>>>> modpath = os.path.join(repo_conf, 'modules', md['path'])
>>>>>
>>>>> + # Check if the module is initialized
>>>>> + try:
>>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
>>>>> + except:
>>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
>>>>> + # fail -- it still might be a failure, see next check...
>>>>> + module_hash = ""
>>>>> +
>>>>> + if not module_hash:
>>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>>>>> + continue
>>>>> +
>>>>> if os.path.exists(srcpath):
>>>>> if os.path.exists(os.path.join(srcpath, '.git')):
>>>>> srcpath = os.path.join(srcpath, '.git')
>>>>> @@ -188,7 +212,7 @@ class GitSM(Git):
>>>>> continue
>>>>>
>>>>> submodules = self.parse_gitmodules(gitmodules)
>>>>> - self.copy_submodules(submodules, ud, dest, d)
>>>>> + self.copy_submodules(submodules, ud, name, dest, d)
>>>>>
>>>>> def unpack(self, ud, destdir, d):
>>>>> Git.unpack(self, ud, destdir, d)
>>>>> @@ -211,7 +235,7 @@ class GitSM(Git):
>>>>> continue
>>>>>
>>>>> submodules = self.parse_gitmodules(gitmodules)
>>>>> - self.copy_submodules(submodules, ud, ud.destdir, d)
>>>>> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
>>>>>
>>>>> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
>>>>> while len(submodules_queue) != 0:
>>>>> --
>>>>> 1.8.3.1
>>>>>
>>>>> --
>>>>> _______________________________________________
>>>>> bitbake-devel mailing list
>>>>> bitbake-devel@lists.openembedded.org
>>>>> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
>>>
>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-18 23:06 ` Mark Hatle
@ 2019-01-19 1:27 ` Khem Raj
2019-01-19 6:44 ` Khem Raj
0 siblings, 1 reply; 17+ messages in thread
From: Khem Raj @ 2019-01-19 1:27 UTC (permalink / raw)
To: Mark Hatle; +Cc: bitbake-devel
I cherry picked this commit on top of master and I still see failure to unpack
ERROR: cli11-1.6.2+gitAUTOINC+bd4dc91184-r0 do_unpack: Fetcher
failure: Fetch command export PSEUDO_DISABLED=1; export
DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/1000/bus"; export
SSH_AGENT_PID="1235"; export SSH_AUTH_SOCK="/tmp/ssh-agent.sock.1000";
export PATH="/mnt/a/yoe/build/tmp/sysroots-uninative/x86_64-linux/usr/bin:/mnt/a/yoe/sources/openembedded-core/scripts:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/bin/arm-yoe-linux-musleabi:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot/usr/bin/crossscripts:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/sbin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/bin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/sbin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/bin:/mnt/a/yoe/sources/bitbake/bin:/mnt/a/yoe/build/tmp/hosttools";
export HOME="/home/kraj"; git -c core.fsyncobjectfiles=0 config
core.bare false
ERROR: cli11-1.6.2+gitAUTOINC+bd4dc91184-r0 do_unpack: Function
failed: base_do_unpack
ERROR: Logfile of failure stored in:
/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/temp/log.do_unpack.10171
ERROR: Task (/mnt/a/yoe/sources/meta-openembedded/meta-oe/recipes-support/cli11/cli11_1.6.2.bb:do_unpack)
failed with exit code '1'
My bitbake tree is here
https://github.com/YoeDistro/bitbake/commits/yoe/mut
On Fri, Jan 18, 2019 at 3:06 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>
> I think I have a fix for this. Can you try the bitbake available from:
>
> git://git.openembedded.org/bitbake-contrib
>
> branch mgh/gitsm
>
> The commit that resolves this issue is:
>
> 0a7d8bd99a5ba394b00ed35593101b4cd324a58d
>
> --Mark
>
> On 1/18/19 3:20 PM, Mark Hatle wrote:
> > On 1/18/19 12:31 PM, Khem Raj wrote:
> >> On Fri, Jan 18, 2019 at 10:08 AM Mark Hatle <mark.hatle@windriver.com> wrote:
> >>>
> >>> On 1/18/19 9:07 AM, Khem Raj wrote:
> >>>> I am seeing
> >>>>
> >>>> http://errors.yoctoproject.org/Errors/Details/217408/
> >>>> http://errors.yoctoproject.org/Errors/Details/217412/
> >>>
> >>> Do you know where these failing recipes are so that I can reproduce it locally?
> >>>
> >>> rwmem looks like it might be meta-ti?
> >>> cli11 looks like something from meta-oe.
> >>>
> >>
> >> yes thats right.
> >> http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-support/cli11/cli11_1.6.2.bb?h=master
> >>
> >> gitsm version of rwmem is not yes applied in meta-ti so you might have
> >> to get it from my staging branch
> >>
> >> https://github.com/YoeDistro/meta-ti/commit/3418beda1869882d5f94dc405d2727f09ffe20c4
> >
> > The reproducer appears to be that these repositories are 'git' based, but use
> > relative submodule locations from the main repository.
> >
> > So now the gitsm code needs to resolve relative URLs, which is something not
> > previously implemented.
> >
> > --Mark
> >
> >>> --Mark
> >>>
> >>>> On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
> >>>>>
> >>>>> It is possible for a submodule to be defined in the .gitmodules file, but
> >>>>> never initialized in the repository itself. This shows itself when searching
> >>>>> for the defined module hash you will get back a empty value.
> >>>>>
> >>>>> Similarly we need to identify and skip defined but not initialized submodules
> >>>>> during the unpack stages as well.
> >>>>>
> >>>>> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
> >>>>> to resolve this issue.
> >>>>>
> >>>>> Additionally a problem was found where, while unlikely, it may be possible
> >>>>> for the wrong revision to have been searched using ls-tree. This has been
> >>>>> resolved in the update_submodules function by keeping the correct revision
> >>>>> along with the submodule path.
> >>>>>
> >>>>> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
> >>>>> ---
> >>>>> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
> >>>>> 1 file changed, 28 insertions(+), 4 deletions(-)
> >>>>>
> >>>>> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> >>>>> index 35729db..b7959ff 100644
> >>>>> --- a/lib/bb/fetch2/gitsm.py
> >>>>> +++ b/lib/bb/fetch2/gitsm.py
> >>>>> @@ -64,6 +64,7 @@ class GitSM(Git):
> >>>>> def update_submodules(self, ud, d):
> >>>>> submodules = []
> >>>>> paths = {}
> >>>>> + revision = {}
> >>>>> uris = {}
> >>>>> local_paths = {}
> >>>>>
> >>>>> @@ -77,6 +78,7 @@ class GitSM(Git):
> >>>>> for m, md in self.parse_gitmodules(gitmodules).items():
> >>>>> submodules.append(m)
> >>>>> paths[m] = md['path']
> >>>>> + revision[m] = ud.revisions[name]
> >>>>> uris[m] = md['url']
> >>>>> if uris[m].startswith('..'):
> >>>>> newud = copy.copy(ud)
> >>>>> @@ -84,7 +86,17 @@ class GitSM(Git):
> >>>>> uris[m] = Git._get_repo_url(self, newud)
> >>>>>
> >>>>> for module in submodules:
> >>>>> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
> >>>>> + try:
> >>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
> >>>>> + except:
> >>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
> >>>>> + # fail -- it still might be a failure, see next check...
> >>>>> + module_hash = ""
> >>>>> +
> >>>>> + if not module_hash:
> >>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> >>>>> + continue
> >>>>> +
> >>>>> module_hash = module_hash.split()[2]
> >>>>>
> >>>>> # Build new SRC_URI
> >>>>> @@ -143,7 +155,7 @@ class GitSM(Git):
> >>>>> if not ud.shallow or ud.localpath != ud.fullshallow:
> >>>>> self.update_submodules(ud, d)
> >>>>>
> >>>>> - def copy_submodules(self, submodules, ud, destdir, d):
> >>>>> + def copy_submodules(self, submodules, ud, name, destdir, d):
> >>>>> if ud.bareclone:
> >>>>> repo_conf = destdir
> >>>>> else:
> >>>>> @@ -156,6 +168,18 @@ class GitSM(Git):
> >>>>> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
> >>>>> modpath = os.path.join(repo_conf, 'modules', md['path'])
> >>>>>
> >>>>> + # Check if the module is initialized
> >>>>> + try:
> >>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
> >>>>> + except:
> >>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
> >>>>> + # fail -- it still might be a failure, see next check...
> >>>>> + module_hash = ""
> >>>>> +
> >>>>> + if not module_hash:
> >>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> >>>>> + continue
> >>>>> +
> >>>>> if os.path.exists(srcpath):
> >>>>> if os.path.exists(os.path.join(srcpath, '.git')):
> >>>>> srcpath = os.path.join(srcpath, '.git')
> >>>>> @@ -188,7 +212,7 @@ class GitSM(Git):
> >>>>> continue
> >>>>>
> >>>>> submodules = self.parse_gitmodules(gitmodules)
> >>>>> - self.copy_submodules(submodules, ud, dest, d)
> >>>>> + self.copy_submodules(submodules, ud, name, dest, d)
> >>>>>
> >>>>> def unpack(self, ud, destdir, d):
> >>>>> Git.unpack(self, ud, destdir, d)
> >>>>> @@ -211,7 +235,7 @@ class GitSM(Git):
> >>>>> continue
> >>>>>
> >>>>> submodules = self.parse_gitmodules(gitmodules)
> >>>>> - self.copy_submodules(submodules, ud, ud.destdir, d)
> >>>>> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
> >>>>>
> >>>>> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
> >>>>> while len(submodules_queue) != 0:
> >>>>> --
> >>>>> 1.8.3.1
> >>>>>
> >>>>> --
> >>>>> _______________________________________________
> >>>>> bitbake-devel mailing list
> >>>>> bitbake-devel@lists.openembedded.org
> >>>>> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
> >>>
> >
>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-19 1:27 ` Khem Raj
@ 2019-01-19 6:44 ` Khem Raj
2019-01-22 20:43 ` Mark Hatle
0 siblings, 1 reply; 17+ messages in thread
From: Khem Raj @ 2019-01-19 6:44 UTC (permalink / raw)
To: Mark Hatle; +Cc: bitbake-devel
Mark
another regression
http://errors.yoctoproject.org/Errors/Details/217455/
http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-core/dbus/dbus-broker_git.bb?h=master
On Fri, Jan 18, 2019 at 5:27 PM Khem Raj <raj.khem@gmail.com> wrote:
>
> I cherry picked this commit on top of master and I still see failure to unpack
>
> ERROR: cli11-1.6.2+gitAUTOINC+bd4dc91184-r0 do_unpack: Fetcher
> failure: Fetch command export PSEUDO_DISABLED=1; export
> DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/1000/bus"; export
> SSH_AGENT_PID="1235"; export SSH_AUTH_SOCK="/tmp/ssh-agent.sock.1000";
> export PATH="/mnt/a/yoe/build/tmp/sysroots-uninative/x86_64-linux/usr/bin:/mnt/a/yoe/sources/openembedded-core/scripts:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/bin/arm-yoe-linux-musleabi:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot/usr/bin/crossscripts:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/sbin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/bin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/sbin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/bin:/mnt/a/yoe/sources/bitbake/bin:/mnt/a/yoe/build/tmp/hosttools";
> export HOME="/home/kraj"; git -c core.fsyncobjectfiles=0 config
> core.bare false
> ERROR: cli11-1.6.2+gitAUTOINC+bd4dc91184-r0 do_unpack: Function
> failed: base_do_unpack
> ERROR: Logfile of failure stored in:
> /mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/temp/log.do_unpack.10171
> ERROR: Task (/mnt/a/yoe/sources/meta-openembedded/meta-oe/recipes-support/cli11/cli11_1.6.2.bb:do_unpack)
> failed with exit code '1'
>
> My bitbake tree is here
>
> https://github.com/YoeDistro/bitbake/commits/yoe/mut
>
> On Fri, Jan 18, 2019 at 3:06 PM Mark Hatle <mark.hatle@windriver.com> wrote:
> >
> > I think I have a fix for this. Can you try the bitbake available from:
> >
> > git://git.openembedded.org/bitbake-contrib
> >
> > branch mgh/gitsm
> >
> > The commit that resolves this issue is:
> >
> > 0a7d8bd99a5ba394b00ed35593101b4cd324a58d
> >
> > --Mark
> >
> > On 1/18/19 3:20 PM, Mark Hatle wrote:
> > > On 1/18/19 12:31 PM, Khem Raj wrote:
> > >> On Fri, Jan 18, 2019 at 10:08 AM Mark Hatle <mark.hatle@windriver.com> wrote:
> > >>>
> > >>> On 1/18/19 9:07 AM, Khem Raj wrote:
> > >>>> I am seeing
> > >>>>
> > >>>> http://errors.yoctoproject.org/Errors/Details/217408/
> > >>>> http://errors.yoctoproject.org/Errors/Details/217412/
> > >>>
> > >>> Do you know where these failing recipes are so that I can reproduce it locally?
> > >>>
> > >>> rwmem looks like it might be meta-ti?
> > >>> cli11 looks like something from meta-oe.
> > >>>
> > >>
> > >> yes thats right.
> > >> http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-support/cli11/cli11_1.6.2.bb?h=master
> > >>
> > >> gitsm version of rwmem is not yes applied in meta-ti so you might have
> > >> to get it from my staging branch
> > >>
> > >> https://github.com/YoeDistro/meta-ti/commit/3418beda1869882d5f94dc405d2727f09ffe20c4
> > >
> > > The reproducer appears to be that these repositories are 'git' based, but use
> > > relative submodule locations from the main repository.
> > >
> > > So now the gitsm code needs to resolve relative URLs, which is something not
> > > previously implemented.
> > >
> > > --Mark
> > >
> > >>> --Mark
> > >>>
> > >>>> On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
> > >>>>>
> > >>>>> It is possible for a submodule to be defined in the .gitmodules file, but
> > >>>>> never initialized in the repository itself. This shows itself when searching
> > >>>>> for the defined module hash you will get back a empty value.
> > >>>>>
> > >>>>> Similarly we need to identify and skip defined but not initialized submodules
> > >>>>> during the unpack stages as well.
> > >>>>>
> > >>>>> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
> > >>>>> to resolve this issue.
> > >>>>>
> > >>>>> Additionally a problem was found where, while unlikely, it may be possible
> > >>>>> for the wrong revision to have been searched using ls-tree. This has been
> > >>>>> resolved in the update_submodules function by keeping the correct revision
> > >>>>> along with the submodule path.
> > >>>>>
> > >>>>> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
> > >>>>> ---
> > >>>>> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
> > >>>>> 1 file changed, 28 insertions(+), 4 deletions(-)
> > >>>>>
> > >>>>> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> > >>>>> index 35729db..b7959ff 100644
> > >>>>> --- a/lib/bb/fetch2/gitsm.py
> > >>>>> +++ b/lib/bb/fetch2/gitsm.py
> > >>>>> @@ -64,6 +64,7 @@ class GitSM(Git):
> > >>>>> def update_submodules(self, ud, d):
> > >>>>> submodules = []
> > >>>>> paths = {}
> > >>>>> + revision = {}
> > >>>>> uris = {}
> > >>>>> local_paths = {}
> > >>>>>
> > >>>>> @@ -77,6 +78,7 @@ class GitSM(Git):
> > >>>>> for m, md in self.parse_gitmodules(gitmodules).items():
> > >>>>> submodules.append(m)
> > >>>>> paths[m] = md['path']
> > >>>>> + revision[m] = ud.revisions[name]
> > >>>>> uris[m] = md['url']
> > >>>>> if uris[m].startswith('..'):
> > >>>>> newud = copy.copy(ud)
> > >>>>> @@ -84,7 +86,17 @@ class GitSM(Git):
> > >>>>> uris[m] = Git._get_repo_url(self, newud)
> > >>>>>
> > >>>>> for module in submodules:
> > >>>>> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
> > >>>>> + try:
> > >>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
> > >>>>> + except:
> > >>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
> > >>>>> + # fail -- it still might be a failure, see next check...
> > >>>>> + module_hash = ""
> > >>>>> +
> > >>>>> + if not module_hash:
> > >>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> > >>>>> + continue
> > >>>>> +
> > >>>>> module_hash = module_hash.split()[2]
> > >>>>>
> > >>>>> # Build new SRC_URI
> > >>>>> @@ -143,7 +155,7 @@ class GitSM(Git):
> > >>>>> if not ud.shallow or ud.localpath != ud.fullshallow:
> > >>>>> self.update_submodules(ud, d)
> > >>>>>
> > >>>>> - def copy_submodules(self, submodules, ud, destdir, d):
> > >>>>> + def copy_submodules(self, submodules, ud, name, destdir, d):
> > >>>>> if ud.bareclone:
> > >>>>> repo_conf = destdir
> > >>>>> else:
> > >>>>> @@ -156,6 +168,18 @@ class GitSM(Git):
> > >>>>> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
> > >>>>> modpath = os.path.join(repo_conf, 'modules', md['path'])
> > >>>>>
> > >>>>> + # Check if the module is initialized
> > >>>>> + try:
> > >>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
> > >>>>> + except:
> > >>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
> > >>>>> + # fail -- it still might be a failure, see next check...
> > >>>>> + module_hash = ""
> > >>>>> +
> > >>>>> + if not module_hash:
> > >>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> > >>>>> + continue
> > >>>>> +
> > >>>>> if os.path.exists(srcpath):
> > >>>>> if os.path.exists(os.path.join(srcpath, '.git')):
> > >>>>> srcpath = os.path.join(srcpath, '.git')
> > >>>>> @@ -188,7 +212,7 @@ class GitSM(Git):
> > >>>>> continue
> > >>>>>
> > >>>>> submodules = self.parse_gitmodules(gitmodules)
> > >>>>> - self.copy_submodules(submodules, ud, dest, d)
> > >>>>> + self.copy_submodules(submodules, ud, name, dest, d)
> > >>>>>
> > >>>>> def unpack(self, ud, destdir, d):
> > >>>>> Git.unpack(self, ud, destdir, d)
> > >>>>> @@ -211,7 +235,7 @@ class GitSM(Git):
> > >>>>> continue
> > >>>>>
> > >>>>> submodules = self.parse_gitmodules(gitmodules)
> > >>>>> - self.copy_submodules(submodules, ud, ud.destdir, d)
> > >>>>> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
> > >>>>>
> > >>>>> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
> > >>>>> while len(submodules_queue) != 0:
> > >>>>> --
> > >>>>> 1.8.3.1
> > >>>>>
> > >>>>> --
> > >>>>> _______________________________________________
> > >>>>> bitbake-devel mailing list
> > >>>>> bitbake-devel@lists.openembedded.org
> > >>>>> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
> > >>>
> > >
> >
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-19 6:44 ` Khem Raj
@ 2019-01-22 20:43 ` Mark Hatle
2019-01-23 3:36 ` Khem Raj
0 siblings, 1 reply; 17+ messages in thread
From: Mark Hatle @ 2019-01-22 20:43 UTC (permalink / raw)
To: Khem Raj; +Cc: bitbake-devel
Sorry for the delay in getting a fix back to you..
See:
http://git.openembedded.org/bitbake-contrib/log/?h=mgh/gitsm
top two commits should resolve both this and the CLI11 issue. (I am assuming
the third one is the same problem.)
Can you please give this a try.. if it's working now, I'll send a formal pull
request.
--Mark
On 1/19/19 12:44 AM, Khem Raj wrote:
> Mark
>
> another regression
>
> http://errors.yoctoproject.org/Errors/Details/217455/
>
> http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-core/dbus/dbus-broker_git.bb?h=master
>
>
>
> On Fri, Jan 18, 2019 at 5:27 PM Khem Raj <raj.khem@gmail.com> wrote:
>>
>> I cherry picked this commit on top of master and I still see failure to unpack
>>
>> ERROR: cli11-1.6.2+gitAUTOINC+bd4dc91184-r0 do_unpack: Fetcher
>> failure: Fetch command export PSEUDO_DISABLED=1; export
>> DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/1000/bus"; export
>> SSH_AGENT_PID="1235"; export SSH_AUTH_SOCK="/tmp/ssh-agent.sock.1000";
>> export PATH="/mnt/a/yoe/build/tmp/sysroots-uninative/x86_64-linux/usr/bin:/mnt/a/yoe/sources/openembedded-core/scripts:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/bin/arm-yoe-linux-musleabi:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot/usr/bin/crossscripts:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/sbin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/bin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/sbin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/bin:/mnt/a/yoe/sources/bitbake/bin:/mnt/a/yoe/build/tmp/hosttools";
>> export HOME="/home/kraj"; git -c core.fsyncobjectfiles=0 config
>> core.bare false
>> ERROR: cli11-1.6.2+gitAUTOINC+bd4dc91184-r0 do_unpack: Function
>> failed: base_do_unpack
>> ERROR: Logfile of failure stored in:
>> /mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/temp/log.do_unpack.10171
>> ERROR: Task (/mnt/a/yoe/sources/meta-openembedded/meta-oe/recipes-support/cli11/cli11_1.6.2.bb:do_unpack)
>> failed with exit code '1'
>>
>> My bitbake tree is here
>>
>> https://github.com/YoeDistro/bitbake/commits/yoe/mut
>>
>> On Fri, Jan 18, 2019 at 3:06 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>>>
>>> I think I have a fix for this. Can you try the bitbake available from:
>>>
>>> git://git.openembedded.org/bitbake-contrib
>>>
>>> branch mgh/gitsm
>>>
>>> The commit that resolves this issue is:
>>>
>>> 0a7d8bd99a5ba394b00ed35593101b4cd324a58d
>>>
>>> --Mark
>>>
>>> On 1/18/19 3:20 PM, Mark Hatle wrote:
>>>> On 1/18/19 12:31 PM, Khem Raj wrote:
>>>>> On Fri, Jan 18, 2019 at 10:08 AM Mark Hatle <mark.hatle@windriver.com> wrote:
>>>>>>
>>>>>> On 1/18/19 9:07 AM, Khem Raj wrote:
>>>>>>> I am seeing
>>>>>>>
>>>>>>> http://errors.yoctoproject.org/Errors/Details/217408/
>>>>>>> http://errors.yoctoproject.org/Errors/Details/217412/
>>>>>>
>>>>>> Do you know where these failing recipes are so that I can reproduce it locally?
>>>>>>
>>>>>> rwmem looks like it might be meta-ti?
>>>>>> cli11 looks like something from meta-oe.
>>>>>>
>>>>>
>>>>> yes thats right.
>>>>> http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-support/cli11/cli11_1.6.2.bb?h=master
>>>>>
>>>>> gitsm version of rwmem is not yes applied in meta-ti so you might have
>>>>> to get it from my staging branch
>>>>>
>>>>> https://github.com/YoeDistro/meta-ti/commit/3418beda1869882d5f94dc405d2727f09ffe20c4
>>>>
>>>> The reproducer appears to be that these repositories are 'git' based, but use
>>>> relative submodule locations from the main repository.
>>>>
>>>> So now the gitsm code needs to resolve relative URLs, which is something not
>>>> previously implemented.
>>>>
>>>> --Mark
>>>>
>>>>>> --Mark
>>>>>>
>>>>>>> On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>>>>>>>>
>>>>>>>> It is possible for a submodule to be defined in the .gitmodules file, but
>>>>>>>> never initialized in the repository itself. This shows itself when searching
>>>>>>>> for the defined module hash you will get back a empty value.
>>>>>>>>
>>>>>>>> Similarly we need to identify and skip defined but not initialized submodules
>>>>>>>> during the unpack stages as well.
>>>>>>>>
>>>>>>>> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
>>>>>>>> to resolve this issue.
>>>>>>>>
>>>>>>>> Additionally a problem was found where, while unlikely, it may be possible
>>>>>>>> for the wrong revision to have been searched using ls-tree. This has been
>>>>>>>> resolved in the update_submodules function by keeping the correct revision
>>>>>>>> along with the submodule path.
>>>>>>>>
>>>>>>>> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
>>>>>>>> ---
>>>>>>>> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
>>>>>>>> 1 file changed, 28 insertions(+), 4 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
>>>>>>>> index 35729db..b7959ff 100644
>>>>>>>> --- a/lib/bb/fetch2/gitsm.py
>>>>>>>> +++ b/lib/bb/fetch2/gitsm.py
>>>>>>>> @@ -64,6 +64,7 @@ class GitSM(Git):
>>>>>>>> def update_submodules(self, ud, d):
>>>>>>>> submodules = []
>>>>>>>> paths = {}
>>>>>>>> + revision = {}
>>>>>>>> uris = {}
>>>>>>>> local_paths = {}
>>>>>>>>
>>>>>>>> @@ -77,6 +78,7 @@ class GitSM(Git):
>>>>>>>> for m, md in self.parse_gitmodules(gitmodules).items():
>>>>>>>> submodules.append(m)
>>>>>>>> paths[m] = md['path']
>>>>>>>> + revision[m] = ud.revisions[name]
>>>>>>>> uris[m] = md['url']
>>>>>>>> if uris[m].startswith('..'):
>>>>>>>> newud = copy.copy(ud)
>>>>>>>> @@ -84,7 +86,17 @@ class GitSM(Git):
>>>>>>>> uris[m] = Git._get_repo_url(self, newud)
>>>>>>>>
>>>>>>>> for module in submodules:
>>>>>>>> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
>>>>>>>> + try:
>>>>>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
>>>>>>>> + except:
>>>>>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
>>>>>>>> + # fail -- it still might be a failure, see next check...
>>>>>>>> + module_hash = ""
>>>>>>>> +
>>>>>>>> + if not module_hash:
>>>>>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>>>>>>>> + continue
>>>>>>>> +
>>>>>>>> module_hash = module_hash.split()[2]
>>>>>>>>
>>>>>>>> # Build new SRC_URI
>>>>>>>> @@ -143,7 +155,7 @@ class GitSM(Git):
>>>>>>>> if not ud.shallow or ud.localpath != ud.fullshallow:
>>>>>>>> self.update_submodules(ud, d)
>>>>>>>>
>>>>>>>> - def copy_submodules(self, submodules, ud, destdir, d):
>>>>>>>> + def copy_submodules(self, submodules, ud, name, destdir, d):
>>>>>>>> if ud.bareclone:
>>>>>>>> repo_conf = destdir
>>>>>>>> else:
>>>>>>>> @@ -156,6 +168,18 @@ class GitSM(Git):
>>>>>>>> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
>>>>>>>> modpath = os.path.join(repo_conf, 'modules', md['path'])
>>>>>>>>
>>>>>>>> + # Check if the module is initialized
>>>>>>>> + try:
>>>>>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
>>>>>>>> + except:
>>>>>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
>>>>>>>> + # fail -- it still might be a failure, see next check...
>>>>>>>> + module_hash = ""
>>>>>>>> +
>>>>>>>> + if not module_hash:
>>>>>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>>>>>>>> + continue
>>>>>>>> +
>>>>>>>> if os.path.exists(srcpath):
>>>>>>>> if os.path.exists(os.path.join(srcpath, '.git')):
>>>>>>>> srcpath = os.path.join(srcpath, '.git')
>>>>>>>> @@ -188,7 +212,7 @@ class GitSM(Git):
>>>>>>>> continue
>>>>>>>>
>>>>>>>> submodules = self.parse_gitmodules(gitmodules)
>>>>>>>> - self.copy_submodules(submodules, ud, dest, d)
>>>>>>>> + self.copy_submodules(submodules, ud, name, dest, d)
>>>>>>>>
>>>>>>>> def unpack(self, ud, destdir, d):
>>>>>>>> Git.unpack(self, ud, destdir, d)
>>>>>>>> @@ -211,7 +235,7 @@ class GitSM(Git):
>>>>>>>> continue
>>>>>>>>
>>>>>>>> submodules = self.parse_gitmodules(gitmodules)
>>>>>>>> - self.copy_submodules(submodules, ud, ud.destdir, d)
>>>>>>>> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
>>>>>>>>
>>>>>>>> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
>>>>>>>> while len(submodules_queue) != 0:
>>>>>>>> --
>>>>>>>> 1.8.3.1
>>>>>>>>
>>>>>>>> --
>>>>>>>> _______________________________________________
>>>>>>>> bitbake-devel mailing list
>>>>>>>> bitbake-devel@lists.openembedded.org
>>>>>>>> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
>>>>>>
>>>>
>>>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized
2019-01-22 20:43 ` Mark Hatle
@ 2019-01-23 3:36 ` Khem Raj
0 siblings, 0 replies; 17+ messages in thread
From: Khem Raj @ 2019-01-23 3:36 UTC (permalink / raw)
To: Mark Hatle; +Cc: bitbake-devel
It worked.
On Tue, Jan 22, 2019 at 3:43 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>
> Sorry for the delay in getting a fix back to you..
>
> See:
>
> http://git.openembedded.org/bitbake-contrib/log/?h=mgh/gitsm
>
> top two commits should resolve both this and the CLI11 issue. (I am assuming
> the third one is the same problem.)
>
> Can you please give this a try.. if it's working now, I'll send a formal pull
> request.
>
> --Mark
>
> On 1/19/19 12:44 AM, Khem Raj wrote:
> > Mark
> >
> > another regression
> >
> > http://errors.yoctoproject.org/Errors/Details/217455/
> >
> > http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-core/dbus/dbus-broker_git.bb?h=master
> >
> >
> >
> > On Fri, Jan 18, 2019 at 5:27 PM Khem Raj <raj.khem@gmail.com> wrote:
> >>
> >> I cherry picked this commit on top of master and I still see failure to unpack
> >>
> >> ERROR: cli11-1.6.2+gitAUTOINC+bd4dc91184-r0 do_unpack: Fetcher
> >> failure: Fetch command export PSEUDO_DISABLED=1; export
> >> DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/1000/bus"; export
> >> SSH_AGENT_PID="1235"; export SSH_AUTH_SOCK="/tmp/ssh-agent.sock.1000";
> >> export PATH="/mnt/a/yoe/build/tmp/sysroots-uninative/x86_64-linux/usr/bin:/mnt/a/yoe/sources/openembedded-core/scripts:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/bin/arm-yoe-linux-musleabi:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot/usr/bin/crossscripts:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/sbin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/usr/bin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/sbin:/mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/recipe-sysroot-native/bin:/mnt/a/yoe/sources/bitbake/bin:/mnt/a/yoe/build/tmp/hosttools";
> >> export HOME="/home/kraj"; git -c core.fsyncobjectfiles=0 config
> >> core.bare false
> >> ERROR: cli11-1.6.2+gitAUTOINC+bd4dc91184-r0 do_unpack: Function
> >> failed: base_do_unpack
> >> ERROR: Logfile of failure stored in:
> >> /mnt/a/yoe/build/tmp/work/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi/cli11/1.6.2+gitAUTOINC+bd4dc91184-r0/temp/log.do_unpack.10171
> >> ERROR: Task (/mnt/a/yoe/sources/meta-openembedded/meta-oe/recipes-support/cli11/cli11_1.6.2.bb:do_unpack)
> >> failed with exit code '1'
> >>
> >> My bitbake tree is here
> >>
> >> https://github.com/YoeDistro/bitbake/commits/yoe/mut
> >>
> >> On Fri, Jan 18, 2019 at 3:06 PM Mark Hatle <mark.hatle@windriver.com> wrote:
> >>>
> >>> I think I have a fix for this. Can you try the bitbake available from:
> >>>
> >>> git://git.openembedded.org/bitbake-contrib
> >>>
> >>> branch mgh/gitsm
> >>>
> >>> The commit that resolves this issue is:
> >>>
> >>> 0a7d8bd99a5ba394b00ed35593101b4cd324a58d
> >>>
> >>> --Mark
> >>>
> >>> On 1/18/19 3:20 PM, Mark Hatle wrote:
> >>>> On 1/18/19 12:31 PM, Khem Raj wrote:
> >>>>> On Fri, Jan 18, 2019 at 10:08 AM Mark Hatle <mark.hatle@windriver.com> wrote:
> >>>>>>
> >>>>>> On 1/18/19 9:07 AM, Khem Raj wrote:
> >>>>>>> I am seeing
> >>>>>>>
> >>>>>>> http://errors.yoctoproject.org/Errors/Details/217408/
> >>>>>>> http://errors.yoctoproject.org/Errors/Details/217412/
> >>>>>>
> >>>>>> Do you know where these failing recipes are so that I can reproduce it locally?
> >>>>>>
> >>>>>> rwmem looks like it might be meta-ti?
> >>>>>> cli11 looks like something from meta-oe.
> >>>>>>
> >>>>>
> >>>>> yes thats right.
> >>>>> http://cgit.openembedded.org/meta-openembedded/tree/meta-oe/recipes-support/cli11/cli11_1.6.2.bb?h=master
> >>>>>
> >>>>> gitsm version of rwmem is not yes applied in meta-ti so you might have
> >>>>> to get it from my staging branch
> >>>>>
> >>>>> https://github.com/YoeDistro/meta-ti/commit/3418beda1869882d5f94dc405d2727f09ffe20c4
> >>>>
> >>>> The reproducer appears to be that these repositories are 'git' based, but use
> >>>> relative submodule locations from the main repository.
> >>>>
> >>>> So now the gitsm code needs to resolve relative URLs, which is something not
> >>>> previously implemented.
> >>>>
> >>>> --Mark
> >>>>
> >>>>>> --Mark
> >>>>>>
> >>>>>>> On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
> >>>>>>>>
> >>>>>>>> It is possible for a submodule to be defined in the .gitmodules file, but
> >>>>>>>> never initialized in the repository itself. This shows itself when searching
> >>>>>>>> for the defined module hash you will get back a empty value.
> >>>>>>>>
> >>>>>>>> Similarly we need to identify and skip defined but not initialized submodules
> >>>>>>>> during the unpack stages as well.
> >>>>>>>>
> >>>>>>>> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
> >>>>>>>> to resolve this issue.
> >>>>>>>>
> >>>>>>>> Additionally a problem was found where, while unlikely, it may be possible
> >>>>>>>> for the wrong revision to have been searched using ls-tree. This has been
> >>>>>>>> resolved in the update_submodules function by keeping the correct revision
> >>>>>>>> along with the submodule path.
> >>>>>>>>
> >>>>>>>> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
> >>>>>>>> ---
> >>>>>>>> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
> >>>>>>>> 1 file changed, 28 insertions(+), 4 deletions(-)
> >>>>>>>>
> >>>>>>>> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> >>>>>>>> index 35729db..b7959ff 100644
> >>>>>>>> --- a/lib/bb/fetch2/gitsm.py
> >>>>>>>> +++ b/lib/bb/fetch2/gitsm.py
> >>>>>>>> @@ -64,6 +64,7 @@ class GitSM(Git):
> >>>>>>>> def update_submodules(self, ud, d):
> >>>>>>>> submodules = []
> >>>>>>>> paths = {}
> >>>>>>>> + revision = {}
> >>>>>>>> uris = {}
> >>>>>>>> local_paths = {}
> >>>>>>>>
> >>>>>>>> @@ -77,6 +78,7 @@ class GitSM(Git):
> >>>>>>>> for m, md in self.parse_gitmodules(gitmodules).items():
> >>>>>>>> submodules.append(m)
> >>>>>>>> paths[m] = md['path']
> >>>>>>>> + revision[m] = ud.revisions[name]
> >>>>>>>> uris[m] = md['url']
> >>>>>>>> if uris[m].startswith('..'):
> >>>>>>>> newud = copy.copy(ud)
> >>>>>>>> @@ -84,7 +86,17 @@ class GitSM(Git):
> >>>>>>>> uris[m] = Git._get_repo_url(self, newud)
> >>>>>>>>
> >>>>>>>> for module in submodules:
> >>>>>>>> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
> >>>>>>>> + try:
> >>>>>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
> >>>>>>>> + except:
> >>>>>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
> >>>>>>>> + # fail -- it still might be a failure, see next check...
> >>>>>>>> + module_hash = ""
> >>>>>>>> +
> >>>>>>>> + if not module_hash:
> >>>>>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> >>>>>>>> + continue
> >>>>>>>> +
> >>>>>>>> module_hash = module_hash.split()[2]
> >>>>>>>>
> >>>>>>>> # Build new SRC_URI
> >>>>>>>> @@ -143,7 +155,7 @@ class GitSM(Git):
> >>>>>>>> if not ud.shallow or ud.localpath != ud.fullshallow:
> >>>>>>>> self.update_submodules(ud, d)
> >>>>>>>>
> >>>>>>>> - def copy_submodules(self, submodules, ud, destdir, d):
> >>>>>>>> + def copy_submodules(self, submodules, ud, name, destdir, d):
> >>>>>>>> if ud.bareclone:
> >>>>>>>> repo_conf = destdir
> >>>>>>>> else:
> >>>>>>>> @@ -156,6 +168,18 @@ class GitSM(Git):
> >>>>>>>> srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
> >>>>>>>> modpath = os.path.join(repo_conf, 'modules', md['path'])
> >>>>>>>>
> >>>>>>>> + # Check if the module is initialized
> >>>>>>>> + try:
> >>>>>>>> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
> >>>>>>>> + except:
> >>>>>>>> + # If the command fails, we don't have a valid file to check. If it doesn't
> >>>>>>>> + # fail -- it still might be a failure, see next check...
> >>>>>>>> + module_hash = ""
> >>>>>>>> +
> >>>>>>>> + if not module_hash:
> >>>>>>>> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
> >>>>>>>> + continue
> >>>>>>>> +
> >>>>>>>> if os.path.exists(srcpath):
> >>>>>>>> if os.path.exists(os.path.join(srcpath, '.git')):
> >>>>>>>> srcpath = os.path.join(srcpath, '.git')
> >>>>>>>> @@ -188,7 +212,7 @@ class GitSM(Git):
> >>>>>>>> continue
> >>>>>>>>
> >>>>>>>> submodules = self.parse_gitmodules(gitmodules)
> >>>>>>>> - self.copy_submodules(submodules, ud, dest, d)
> >>>>>>>> + self.copy_submodules(submodules, ud, name, dest, d)
> >>>>>>>>
> >>>>>>>> def unpack(self, ud, destdir, d):
> >>>>>>>> Git.unpack(self, ud, destdir, d)
> >>>>>>>> @@ -211,7 +235,7 @@ class GitSM(Git):
> >>>>>>>> continue
> >>>>>>>>
> >>>>>>>> submodules = self.parse_gitmodules(gitmodules)
> >>>>>>>> - self.copy_submodules(submodules, ud, ud.destdir, d)
> >>>>>>>> + self.copy_submodules(submodules, ud, name, ud.destdir, d)
> >>>>>>>>
> >>>>>>>> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
> >>>>>>>> while len(submodules_queue) != 0:
> >>>>>>>> --
> >>>>>>>> 1.8.3.1
> >>>>>>>>
> >>>>>>>> --
> >>>>>>>> _______________________________________________
> >>>>>>>> bitbake-devel mailing list
> >>>>>>>> bitbake-devel@lists.openembedded.org
> >>>>>>>> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
> >>>>>>
> >>>>
> >>>
>
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 2/7] gitsm.py: Add support for alternative URL formats from submodule files
2019-01-15 21:31 [PATCH 0/7 v2] gitsm.py: submodule init and ssh url processing Mark Hatle
2019-01-15 21:31 ` [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized Mark Hatle
@ 2019-01-15 21:31 ` Mark Hatle
2019-01-15 21:31 ` [PATCH 3/7] tests/fetch.py: Add alternative gitsm test case Mark Hatle
` (4 subsequent siblings)
6 siblings, 0 replies; 17+ messages in thread
From: Mark Hatle @ 2019-01-15 21:31 UTC (permalink / raw)
To: bitbake-devel
The following appear to be the git supported formats:
proto://user:pass@host/path (URI format)
user@host:path (SSH format)
/path or ./path or ../path (local file format)
We adjust the parsing to find out if we have a URI format or not.
When we are NOT in URI format, we do our best to determine SSH or
file format by looking for a ':' in the overall string. If we find
a ':' we assume SSH format and adjust accordingly.
Note, in SSH format we simply replace the ':' with a '/' when constructing
the URL. However, if the original path was ":/...", we don't want '//' so
we deal with this corner case as well.
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
lib/bb/fetch2/gitsm.py | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index b7959ff..dd94186 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -100,8 +100,21 @@ class GitSM(Git):
module_hash = module_hash.split()[2]
# Build new SRC_URI
- proto = uris[module].split(':', 1)[0]
- url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+ if "://" not in uris[module]:
+ # It's ssh if the format does NOT have "://", but has a ':'
+ if ":" in uris[module]:
+ proto = "ssh"
+ if ":/" in uris[module]:
+ url = "gitsm://" + uris[module].replace(':/', '/', 1)
+ else:
+ url = "gitsm://" + uris[module].replace(':', '/', 1)
+ else: # Fall back to 'file' if there is no ':'
+ proto = "file"
+ url = "gitsm://" + uris[module]
+ else:
+ proto = uris[module].split(':', 1)[0]
+ url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+
url += ';protocol=%s' % proto
url += ";name=%s" % module
url += ";bareclone=1;nocheckout=1;nobranch=1"
--
1.8.3.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 3/7] tests/fetch.py: Add alternative gitsm test case
2019-01-15 21:31 [PATCH 0/7 v2] gitsm.py: submodule init and ssh url processing Mark Hatle
2019-01-15 21:31 ` [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized Mark Hatle
2019-01-15 21:31 ` [PATCH 2/7] gitsm.py: Add support for alternative URL formats from submodule files Mark Hatle
@ 2019-01-15 21:31 ` Mark Hatle
2019-01-15 21:31 ` [PATCH 4/7] gitsm.py: Optimize code and attempt to resolve locking issue Mark Hatle
` (3 subsequent siblings)
6 siblings, 0 replies; 17+ messages in thread
From: Mark Hatle @ 2019-01-15 21:31 UTC (permalink / raw)
To: bitbake-devel
In order to test the ssh processing in gitsm, we add an alternative
testcase that can be downloaded from git.yoctoproject.org. However,
this test case requries (read) access, via ssh, to git.yoctoproject.org.
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
lib/bb/tests/fetch.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 6848095..311c701 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -893,7 +893,11 @@ class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_git_submodule(self):
- fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
+ # URL with ssh submodules
+ url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=0d3ffc14bce95e8b3a21a0a67bfe4c4a96ba6350"
+ # Original URL (comment this if you have ssh access to git.yoctoproject.org)
+ url = "gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"
+ fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
os.chdir(os.path.dirname(self.unpackdir))
--
1.8.3.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 4/7] gitsm.py: Optimize code and attempt to resolve locking issue
2019-01-15 21:31 [PATCH 0/7 v2] gitsm.py: submodule init and ssh url processing Mark Hatle
` (2 preceding siblings ...)
2019-01-15 21:31 ` [PATCH 3/7] tests/fetch.py: Add alternative gitsm test case Mark Hatle
@ 2019-01-15 21:31 ` Mark Hatle
2019-01-15 21:31 ` [PATCH 5/7] gitsm.py: revise unpack Mark Hatle
` (2 subsequent siblings)
6 siblings, 0 replies; 17+ messages in thread
From: Mark Hatle @ 2019-01-15 21:31 UTC (permalink / raw)
To: bitbake-devel
It was reported that a race condition on a shared download directory could
occur with the gitsm fetcher, the result happened with a call to
git config
that occured within the update_submodules. Since the fetch is locked by the
upper level, it was probably the prior need_update(...) function causing this
because of some old code.
The gitsm class inherits the git class. The need_update was overridding the
version in gitsm, so that it forceably checked the submodules.
It's clear we can optimize the code by only updating if the primary repository
needs updating. Since we don't care if the submodule repository has changed
because if the primary hasn't, references to the submodule won't change.
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
lib/bb/fetch2/gitsm.py | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index dd94186..11bfa66 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -148,20 +148,6 @@ class GitSM(Git):
return True
- def need_update(self, ud, d):
- main_repo_needs_update = Git.need_update(self, ud, d)
-
- # First check that the main repository has enough history fetched. If it doesn't, then we don't
- # even have the .gitmodules and gitlinks for the submodules to attempt asking whether the
- # submodules' histories are recent enough.
- if main_repo_needs_update:
- return True
-
- # Now check that the submodule histories are new enough. The git-submodule command doesn't have
- # any clean interface for doing this aside from just attempting the checkout (with network
- # fetched disabled).
- return not self.update_submodules(ud, d)
-
def download(self, ud, d):
Git.download(self, ud, d)
--
1.8.3.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 5/7] gitsm.py: revise unpack
2019-01-15 21:31 [PATCH 0/7 v2] gitsm.py: submodule init and ssh url processing Mark Hatle
` (3 preceding siblings ...)
2019-01-15 21:31 ` [PATCH 4/7] gitsm.py: Optimize code and attempt to resolve locking issue Mark Hatle
@ 2019-01-15 21:31 ` Mark Hatle
2019-01-15 21:31 ` [PATCH 6/7] gitsm.py: Rework the shallow fetcher and test case Mark Hatle
2019-01-15 21:31 ` [PATCH 7/7] gitsm.py: Refactor the functions and simplify the class Mark Hatle
6 siblings, 0 replies; 17+ messages in thread
From: Mark Hatle @ 2019-01-15 21:31 UTC (permalink / raw)
To: bitbake-devel
Greatly simply the unpack rule by copying the general functionality of
update_submodules as unpack_submodules. This will recursively construct
a set of urls and unpack them using the standard system behaviors.
The overall code may be slightly bigger, but this ensures that all of the
standard locks are inplace, ensuring the code doesn't change out from
under the unpack function. (This could have happened before due to using
'cp' instead of further unpacks on submodules. This may still happen in
shallow clones.)
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
lib/bb/fetch2/gitsm.py | 116 +++++++++++++++++++++++++++++++++----------------
1 file changed, 78 insertions(+), 38 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index 11bfa66..c172ab1 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -213,61 +213,101 @@ class GitSM(Git):
submodules = self.parse_gitmodules(gitmodules)
self.copy_submodules(submodules, ud, name, dest, d)
- def unpack(self, ud, destdir, d):
- Git.unpack(self, ud, destdir, d)
-
- # Copy over the submodules' fetched histories too.
- if ud.bareclone:
- repo_conf = ud.destdir
- else:
- repo_conf = os.path.join(ud.destdir, '.git')
-
- update_submodules = False
+ def unpack_submodules(self, repo_conf, ud, d):
+ submodules = []
paths = {}
+ revision = {}
uris = {}
local_paths = {}
+
for name in ud.names:
try:
- gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+ gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.destdir)
except:
# No submodules to update
continue
- submodules = self.parse_gitmodules(gitmodules)
- self.copy_submodules(submodules, ud, name, ud.destdir, d)
+ for m, md in self.parse_gitmodules(gitmodules).items():
+ submodules.append(m)
+ paths[m] = md['path']
+ revision[m] = ud.revisions[name]
+ uris[m] = md['url']
+ if uris[m].startswith('..'):
+ newud = copy.copy(ud)
+ newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
+ uris[m] = Git._get_repo_url(self, newud)
- submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
- while len(submodules_queue) != 0:
- module, modpath = submodules_queue.pop()
+ modules_updated = False
- # add submodule children recursively
- try:
- gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=modpath)
- for m, md in self.parse_gitmodules(gitmodules).items():
- submodules_queue.append([m, os.path.join(modpath, 'modules', md['path'])])
- except:
- # no children
- pass
+ for module in submodules:
+ try:
+ module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.destdir)
+ except:
+ # If the command fails, we don't have a valid file to check. If it doesn't
+ # fail -- it still might be a failure, see next check...
+ module_hash = ""
+ if not module_hash:
+ logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
+ continue
- # There are submodules to update
- update_submodules = True
+ modules_updated = True
- # Determine (from the submodule) the correct url to reference
- try:
- output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath)
- except bb.fetch2.FetchError as e:
- # No remote url defined in this submodule
- continue
+ module_hash = module_hash.split()[2]
- local_paths[module] = output
+ # Build new SRC_URI
+ if "://" not in uris[module]:
+ # It's ssh if the format does NOT have "://", but has a ':'
+ if ":" in uris[module]:
+ proto = "ssh"
+ if ":/" in uris[module]:
+ url = "gitsm://" + uris[module].replace(':/', '/', 1)
+ else:
+ url = "gitsm://" + uris[module].replace(':', '/', 1)
+ else: # Fall back to 'file' if there is no ':'
+ proto = "file"
+ url = "gitsm://" + uris[module]
+ else:
+ proto = uris[module].split(':', 1)[0]
+ url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
- # Setup the local URL properly (like git submodule init or sync would do...)
- runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
+ url += ';protocol=%s' % proto
+ url += ";name=%s" % module
+ url += ";bareclone=1;nobranch=1;subpath=%s" % paths[module]
+
+ ld = d.createCopy()
+ # Not necessary to set SRC_URI, since we're passing the URI to
+ # Fetch.
+ #ld.setVar('SRC_URI', url)
+ ld.setVar('SRCREV_%s' % module, module_hash)
- # Ensure the submodule repository is NOT set to bare, since we're checking it out...
- runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath)
+ # Workaround for issues with SRCPV/SRCREV_FORMAT errors
+ # error refer to 'multiple' repositories. Only the repository
+ # in the original SRC_URI actually matters...
+ ld.setVar('SRCPV', d.getVar('SRCPV'))
+ ld.setVar('SRCREV_FORMAT', module)
+
+ newfetch = Fetch([url], ld, cache=False)
+ newfetch.unpack(root=os.path.join(repo_conf, 'modules'))
+ local_paths[module] = newfetch.localpath(url)
+
+ # Correct the submodule references to the local download version...
+ runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
+
+ # Ensure the submodule repository is NOT set to bare, since we're checking it out...
+ runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', paths[module]))
+
+ return modules_updated
+
+ def unpack(self, ud, destdir, d):
+ Git.unpack(self, ud, destdir, d)
+
+ # Copy over the submodules' fetched histories too.
+ if ud.bareclone:
+ repo_conf = ud.destdir
+ else:
+ repo_conf = os.path.join(ud.destdir, '.git')
- if update_submodules:
+ if self.unpack_submodules(repo_conf, ud, d):
# Run submodule update, this sets up the directories -- without touching the config
runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
--
1.8.3.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 6/7] gitsm.py: Rework the shallow fetcher and test case
2019-01-15 21:31 [PATCH 0/7 v2] gitsm.py: submodule init and ssh url processing Mark Hatle
` (4 preceding siblings ...)
2019-01-15 21:31 ` [PATCH 5/7] gitsm.py: revise unpack Mark Hatle
@ 2019-01-15 21:31 ` Mark Hatle
2019-01-15 21:31 ` [PATCH 7/7] gitsm.py: Refactor the functions and simplify the class Mark Hatle
6 siblings, 0 replies; 17+ messages in thread
From: Mark Hatle @ 2019-01-15 21:31 UTC (permalink / raw)
To: bitbake-devel
A custom shallow submodule is no longer necessary, as the regular git
fetcher is used and shallow handling works with the same code.
The only general difference between the regular change is simply declaring a
clone as shallow, when appropriate.
This also removes a potential race condition in copying repositories
vs cloning them.
The gitsm shallow fetcher test was revised to verify that the submodule
is shallow cloned along with the primary repository.
The first step of this change was to be sure to clean the gitsubmodule download
directory, as was previously done with the may gitsource directory.
Additional test components were added to verify commit counts, and an
obsolete (and likely incorrect) test for the .git/modules directory to be
empty was also removed.
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
lib/bb/fetch2/gitsm.py | 66 +++-----------------------------------------------
lib/bb/tests/fetch.py | 10 +++++++-
2 files changed, 13 insertions(+), 63 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index c172ab1..83571f8 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -150,68 +150,7 @@ class GitSM(Git):
def download(self, ud, d):
Git.download(self, ud, d)
-
- if not ud.shallow or ud.localpath != ud.fullshallow:
- self.update_submodules(ud, d)
-
- def copy_submodules(self, submodules, ud, name, destdir, d):
- if ud.bareclone:
- repo_conf = destdir
- else:
- repo_conf = os.path.join(destdir, '.git')
-
- if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')):
- os.mkdir(os.path.join(repo_conf, 'modules'))
-
- for module, md in submodules.items():
- srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
- modpath = os.path.join(repo_conf, 'modules', md['path'])
-
- # Check if the module is initialized
- try:
- module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
- except:
- # If the command fails, we don't have a valid file to check. If it doesn't
- # fail -- it still might be a failure, see next check...
- module_hash = ""
-
- if not module_hash:
- logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
- continue
-
- if os.path.exists(srcpath):
- if os.path.exists(os.path.join(srcpath, '.git')):
- srcpath = os.path.join(srcpath, '.git')
-
- target = modpath
- if os.path.exists(modpath):
- target = os.path.dirname(modpath)
-
- os.makedirs(os.path.dirname(target), exist_ok=True)
- runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d)
- elif os.path.exists(modpath):
- # Module already exists, likely unpacked from a shallow mirror clone
- pass
- else:
- # This is fatal, as we do NOT want git-submodule to hit the network
- raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath))
-
- def clone_shallow_local(self, ud, dest, d):
- super(GitSM, self).clone_shallow_local(ud, dest, d)
-
- # Copy over the submodules' fetched histories too.
- repo_conf = os.path.join(dest, '.git')
-
- submodules = []
- for name in ud.names:
- try:
- gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest)
- except:
- # No submodules to update
- continue
-
- submodules = self.parse_gitmodules(gitmodules)
- self.copy_submodules(submodules, ud, name, dest, d)
+ self.update_submodules(ud, d)
def unpack_submodules(self, repo_conf, ud, d):
submodules = []
@@ -294,6 +233,9 @@ class GitSM(Git):
# Correct the submodule references to the local download version...
runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
+ if ud.shallow:
+ runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
+
# Ensure the submodule repository is NOT set to bare, since we're checking it out...
runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', paths[module]))
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 311c701..5fb5d04 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -1316,6 +1316,7 @@ class GitShallowTest(FetcherTest):
# fetch and unpack, from the shallow tarball
bb.utils.remove(self.gitdir, recurse=True)
bb.utils.remove(ud.clonedir, recurse=True)
+ bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
# confirm that the unpacked repo is used when no git clone or git
# mirror tarball is available
@@ -1470,6 +1471,7 @@ class GitShallowTest(FetcherTest):
self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
self.add_empty_file('asub', cwd=smdir)
+ self.add_empty_file('bsub', cwd=smdir)
self.git('submodule init', cwd=self.srcdir)
self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
@@ -1479,10 +1481,16 @@ class GitShallowTest(FetcherTest):
uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
fetcher, ud = self.fetch_shallow(uri)
+ # Verify the main repository is shallow
self.assertRevCount(1)
- assert './.git/modules/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
+
+ # Verify the gitsubmodule directory is present
assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
+ # Verify the submodule is also shallow
+ self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
+
+
if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
def test_shallow_annex(self):
self.add_empty_file('a')
--
1.8.3.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 7/7] gitsm.py: Refactor the functions and simplify the class
2019-01-15 21:31 [PATCH 0/7 v2] gitsm.py: submodule init and ssh url processing Mark Hatle
` (5 preceding siblings ...)
2019-01-15 21:31 ` [PATCH 6/7] gitsm.py: Rework the shallow fetcher and test case Mark Hatle
@ 2019-01-15 21:31 ` Mark Hatle
6 siblings, 0 replies; 17+ messages in thread
From: Mark Hatle @ 2019-01-15 21:31 UTC (permalink / raw)
To: bitbake-devel
The update_submodules and unpack_submodules functions were nearly indentical,
so we made a common function where the different behavior could be passed
in by the download and unpack users. The new function is process_submodules.
Moved the parse_gitmodules function under the new process_submodules, since
there are no external callers.
Refactor the file relative path processing to the URL translation code.
We also add a warning to the translation if a relative ssh URL has been
detected. Since this can cause a problem.
In the case of a relative URL that does not work after being translated,
it should be possible to use the MIRROR functions to manual translate the
generated relative URL into one that works properly.
Remove 'git config' processing on download contents. It turns out this is not
necessary since all of the later components work using the git fetcher.
Limit the 'git submodule update' call to only when unpacking a non-bare
repository. Submodules are always loaded as bare, so this prevents
intermediate unpacks from being attempted.
Finally, the test cases were updated and the new commit ids in the test
repository were updates as well.
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
lib/bb/fetch2/gitsm.py | 232 +++++++++++++++++++------------------------------
lib/bb/tests/fetch.py | 12 ++-
2 files changed, 98 insertions(+), 146 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index 83571f8..86198ee 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -45,85 +45,96 @@ class GitSM(Git):
"""
return ud.type in ['gitsm']
- @staticmethod
- def parse_gitmodules(gitmodules):
- modules = {}
- module = ""
- for line in gitmodules.splitlines():
- if line.startswith('[submodule'):
- module = line.split('"')[1]
- modules[module] = {}
- elif module and line.strip().startswith('path'):
- path = line.split('=')[1].strip()
- modules[module]['path'] = path
- elif module and line.strip().startswith('url'):
- url = line.split('=')[1].strip()
- modules[module]['url'] = url
- return modules
-
- def update_submodules(self, ud, d):
+ def process_submodules(self, ud, workdir, function, d):
+ """
+ Iterate over all of the submodules in this repository and execute
+ the 'function' for each of them.
+ """
+
submodules = []
paths = {}
revision = {}
uris = {}
- local_paths = {}
-
+ subrevision = {}
+
+ def parse_gitmodules(gitmodules):
+ modules = {}
+ module = ""
+ for line in gitmodules.splitlines():
+ if line.startswith('[submodule'):
+ module = line.split('"')[1]
+ modules[module] = {}
+ elif module and line.strip().startswith('path'):
+ path = line.split('=')[1].strip()
+ modules[module]['path'] = path
+ elif module and line.strip().startswith('url'):
+ url = line.split('=')[1].strip()
+ modules[module]['url'] = url
+ return modules
+
+ # Collect the defined submodules, and their attributes
for name in ud.names:
try:
- gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir)
+ gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
except:
# No submodules to update
continue
- for m, md in self.parse_gitmodules(gitmodules).items():
+ for m, md in parse_gitmodules(gitmodules).items():
+ try:
+ module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
+ except:
+ # If the command fails, we don't have a valid file to check. If it doesn't
+ # fail -- it still might be a failure, see next check...
+ module_hash = ""
+
+ if not module_hash:
+ logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
+ continue
+
submodules.append(m)
paths[m] = md['path']
revision[m] = ud.revisions[name]
uris[m] = md['url']
- if uris[m].startswith('..'):
- newud = copy.copy(ud)
- newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
- uris[m] = Git._get_repo_url(self, newud)
+ subrevision[m] = module_hash.split()[2]
for module in submodules:
- try:
- module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
- except:
- # If the command fails, we don't have a valid file to check. If it doesn't
- # fail -- it still might be a failure, see next check...
- module_hash = ""
+ # Translate the module url into a SRC_URI
- if not module_hash:
- logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
- continue
-
- module_hash = module_hash.split()[2]
-
- # Build new SRC_URI
- if "://" not in uris[module]:
- # It's ssh if the format does NOT have "://", but has a ':'
+ if "://" in uris[module]:
+ # Properly formated URL already
+ proto = uris[module].split(':', 1)[0]
+ url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+ else:
if ":" in uris[module]:
+ # Most likely an SSH style reference
proto = "ssh"
if ":/" in uris[module]:
+ # Absolute reference, easy to convert..
url = "gitsm://" + uris[module].replace(':/', '/', 1)
else:
+ # Relative reference, no way to know if this is right!
+ logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
url = "gitsm://" + uris[module].replace(':', '/', 1)
- else: # Fall back to 'file' if there is no ':'
+ else:
+ # This has to be a file reference
proto = "file"
url = "gitsm://" + uris[module]
- else:
- proto = uris[module].split(':', 1)[0]
- url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+ if uris[module].startswith('..'):
+ # Local on disk relative reference
+ newud = copy.copy(ud)
+ newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
+ url = "gitsm://" + Git._get_repo_url(self, newud)
url += ';protocol=%s' % proto
url += ";name=%s" % module
- url += ";bareclone=1;nocheckout=1;nobranch=1"
+ url += ";subpath=%s" % paths[module]
ld = d.createCopy()
# Not necessary to set SRC_URI, since we're passing the URI to
# Fetch.
#ld.setVar('SRC_URI', url)
- ld.setVar('SRCREV_%s' % module, module_hash)
+ ld.setVar('SRCREV_%s' % module, subrevision[module])
# Workaround for issues with SRCPV/SRCREV_FORMAT errors
# error refer to 'multiple' repositories. Only the repository
@@ -131,125 +142,58 @@ class GitSM(Git):
ld.setVar('SRCPV', d.getVar('SRCPV'))
ld.setVar('SRCREV_FORMAT', module)
- newfetch = Fetch([url], ld, cache=False)
- newfetch.download()
- local_paths[module] = newfetch.localpath(url)
+ function(ud, url, module, paths[module], ld)
- # Correct the submodule references to the local download version...
- runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir)
-
- symlink_path = os.path.join(ud.clonedir, 'modules', paths[module])
- if not os.path.exists(symlink_path):
- try:
- os.makedirs(os.path.dirname(symlink_path), exist_ok=True)
- except OSError:
- pass
- os.symlink(local_paths[module], symlink_path)
-
- return True
+ return submodules != []
def download(self, ud, d):
- Git.download(self, ud, d)
- self.update_submodules(ud, d)
-
- def unpack_submodules(self, repo_conf, ud, d):
- submodules = []
- paths = {}
- revision = {}
- uris = {}
- local_paths = {}
-
- for name in ud.names:
- try:
- gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.destdir)
- except:
- # No submodules to update
- continue
-
- for m, md in self.parse_gitmodules(gitmodules).items():
- submodules.append(m)
- paths[m] = md['path']
- revision[m] = ud.revisions[name]
- uris[m] = md['url']
- if uris[m].startswith('..'):
- newud = copy.copy(ud)
- newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
- uris[m] = Git._get_repo_url(self, newud)
+ def download_submodule(ud, url, module, modpath, d):
+ url += ";bareclone=1;nobranch=1"
- modules_updated = False
+ # Is the following still needed?
+ #url += ";nocheckout=1"
- for module in submodules:
try:
- module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.destdir)
- except:
- # If the command fails, we don't have a valid file to check. If it doesn't
- # fail -- it still might be a failure, see next check...
- module_hash = ""
+ newfetch = Fetch([url], d, cache=False)
+ newfetch.download()
+ except Exception as e:
+ logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
+ raise
- if not module_hash:
- logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
- continue
-
- modules_updated = True
+ Git.download(self, ud, d)
+ self.process_submodules(ud, ud.clonedir, download_submodule, d)
- module_hash = module_hash.split()[2]
+ def unpack(self, ud, destdir, d):
+ def unpack_submodules(ud, url, module, modpath, d):
+ url += ";bareclone=1;nobranch=1"
- # Build new SRC_URI
- if "://" not in uris[module]:
- # It's ssh if the format does NOT have "://", but has a ':'
- if ":" in uris[module]:
- proto = "ssh"
- if ":/" in uris[module]:
- url = "gitsm://" + uris[module].replace(':/', '/', 1)
- else:
- url = "gitsm://" + uris[module].replace(':', '/', 1)
- else: # Fall back to 'file' if there is no ':'
- proto = "file"
- url = "gitsm://" + uris[module]
+ # Figure out where we clone over the bare submodules...
+ if ud.bareclone:
+ repo_conf = ud.destdir
else:
- proto = uris[module].split(':', 1)[0]
- url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
-
- url += ';protocol=%s' % proto
- url += ";name=%s" % module
- url += ";bareclone=1;nobranch=1;subpath=%s" % paths[module]
-
- ld = d.createCopy()
- # Not necessary to set SRC_URI, since we're passing the URI to
- # Fetch.
- #ld.setVar('SRC_URI', url)
- ld.setVar('SRCREV_%s' % module, module_hash)
+ repo_conf = os.path.join(ud.destdir, '.git')
- # Workaround for issues with SRCPV/SRCREV_FORMAT errors
- # error refer to 'multiple' repositories. Only the repository
- # in the original SRC_URI actually matters...
- ld.setVar('SRCPV', d.getVar('SRCPV'))
- ld.setVar('SRCREV_FORMAT', module)
+ try:
+ newfetch = Fetch([url], d, cache=False)
+ newfetch.unpack(root=os.path.join(repo_conf, 'modules'))
+ except Exception as e:
+ logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
+ raise
- newfetch = Fetch([url], ld, cache=False)
- newfetch.unpack(root=os.path.join(repo_conf, 'modules'))
- local_paths[module] = newfetch.localpath(url)
+ newfetch = Fetch([url], d, cache=False)
+ local_path = newfetch.localpath(url)
# Correct the submodule references to the local download version...
- runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
+ runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
if ud.shallow:
runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
# Ensure the submodule repository is NOT set to bare, since we're checking it out...
- runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', paths[module]))
-
- return modules_updated
+ runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
- def unpack(self, ud, destdir, d):
Git.unpack(self, ud, destdir, d)
- # Copy over the submodules' fetched histories too.
- if ud.bareclone:
- repo_conf = ud.destdir
- else:
- repo_conf = os.path.join(ud.destdir, '.git')
-
- if self.unpack_submodules(repo_conf, ud, d):
+ if not ud.bareclone and self.process_submodules(ud, ud.destdir, unpack_submodules, d):
# Run submodule update, this sets up the directories -- without touching the config
runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 5fb5d04..1497a3c 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -894,15 +894,23 @@ class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_git_submodule(self):
# URL with ssh submodules
- url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=0d3ffc14bce95e8b3a21a0a67bfe4c4a96ba6350"
+ url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=f53765f515e0eeca569ed385bb1c89ce008bb058"
# Original URL (comment this if you have ssh access to git.yoctoproject.org)
- url = "gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"
+ url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=132fea6e4dee56b61bcf5721c94e8b2445c6a017"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
os.chdir(os.path.dirname(self.unpackdir))
fetcher.unpack(self.unpackdir)
+ repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
+ self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
+ self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
+
+ # Only when we're running the extended test with a submodule's submodule, can we check this.
+ if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
+ self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
class TrustedNetworksTest(FetcherTest):
def test_trusted_network(self):
--
1.8.3.1
^ permalink raw reply related [flat|nested] 17+ messages in thread