* [PATCH 1/4] fetch2/repo: Fix missing logger import in repo fetcher
2017-10-24 11:35 [PATCH 0/4] fetch2/repo: Repo fetcher fixes and improvements Oleksandr Andrushchenko
@ 2017-10-24 11:35 ` Oleksandr Andrushchenko
2017-10-24 11:35 ` [PATCH 2/4] fetch2/repo: Always check if branch is correct Oleksandr Andrushchenko
` (2 subsequent siblings)
3 siblings, 0 replies; 8+ messages in thread
From: Oleksandr Andrushchenko @ 2017-10-24 11:35 UTC (permalink / raw)
To: bitbake-devel; +Cc: andr2000, Oleksandr Andrushchenko
From: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
After cleaning deprecated API usage repo fetcher is missing
logger as it was indirectly imported via deprecated bb.data.
Fix this by importing logger directly.
Fixes: 9752fd1c10b8 ("fetch2: don't use deprecated bb.data APIs")
Signed-off-by: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
---
lib/bb/fetch2/repo.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/bb/fetch2/repo.py b/lib/bb/fetch2/repo.py
index 1be91cc698e3..c22d9b5578eb 100644
--- a/lib/bb/fetch2/repo.py
+++ b/lib/bb/fetch2/repo.py
@@ -27,6 +27,7 @@ import os
import bb
from bb.fetch2 import FetchMethod
from bb.fetch2 import runfetchcmd
+from bb.fetch2 import logger
class Repo(FetchMethod):
"""Class to fetch a module or modules from repo (git) repositories"""
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 2/4] fetch2/repo: Always check if branch is correct
2017-10-24 11:35 [PATCH 0/4] fetch2/repo: Repo fetcher fixes and improvements Oleksandr Andrushchenko
2017-10-24 11:35 ` [PATCH 1/4] fetch2/repo: Fix missing logger import in repo fetcher Oleksandr Andrushchenko
@ 2017-10-24 11:35 ` Oleksandr Andrushchenko
2017-10-24 11:35 ` [PATCH 3/4] fetch2/repo: Make fetcher always sync on unpack Oleksandr Andrushchenko
2017-10-24 11:35 ` [PATCH 4/4] fetch2/repo: Use multiple jobs to fetch and sync Oleksandr Andrushchenko
3 siblings, 0 replies; 8+ messages in thread
From: Oleksandr Andrushchenko @ 2017-10-24 11:35 UTC (permalink / raw)
To: bitbake-devel; +Cc: andr2000, Oleksandr Andrushchenko
From: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
Current implementation doesn't pay attention if
the same repository is used for manifest, but different
branch, e.g. existing cached repository is not checked
if the branch is correct. Fix this by always running
repo init/sync.
Signed-off-by: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
---
lib/bb/fetch2/repo.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/lib/bb/fetch2/repo.py b/lib/bb/fetch2/repo.py
index c22d9b5578eb..861437dd22b2 100644
--- a/lib/bb/fetch2/repo.py
+++ b/lib/bb/fetch2/repo.py
@@ -71,9 +71,8 @@ class Repo(FetchMethod):
repodir = os.path.join(codir, "repo")
bb.utils.mkdirhier(repodir)
- if not os.path.exists(os.path.join(repodir, ".repo")):
- bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
- runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
+ bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
+ runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
runfetchcmd("repo sync", d, workdir=repodir)
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 3/4] fetch2/repo: Make fetcher always sync on unpack
2017-10-24 11:35 [PATCH 0/4] fetch2/repo: Repo fetcher fixes and improvements Oleksandr Andrushchenko
2017-10-24 11:35 ` [PATCH 1/4] fetch2/repo: Fix missing logger import in repo fetcher Oleksandr Andrushchenko
2017-10-24 11:35 ` [PATCH 2/4] fetch2/repo: Always check if branch is correct Oleksandr Andrushchenko
@ 2017-10-24 11:35 ` Oleksandr Andrushchenko
2017-11-07 13:45 ` Richard Purdie
2017-10-24 11:35 ` [PATCH 4/4] fetch2/repo: Use multiple jobs to fetch and sync Oleksandr Andrushchenko
3 siblings, 1 reply; 8+ messages in thread
From: Oleksandr Andrushchenko @ 2017-10-24 11:35 UTC (permalink / raw)
To: bitbake-devel; +Cc: andr2000, Oleksandr Andrushchenko
From: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
Currently if repo has fetched the repository and
cached it to a tar file it never runs repo sync again,
thus making the build use potentially outdated repo.
Fix this by implementing unpack functionality which runs
repo sync.
Signed-off-by: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
---
lib/bb/fetch2/repo.py | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/lib/bb/fetch2/repo.py b/lib/bb/fetch2/repo.py
index 861437dd22b2..3c28fe04ab1e 100644
--- a/lib/bb/fetch2/repo.py
+++ b/lib/bb/fetch2/repo.py
@@ -52,6 +52,10 @@ class Repo(FetchMethod):
ud.manifest += '.xml'
ud.localfile = d.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch))
+ repodir = d.getVar("REPODIR", True) or os.path.join(d.getVar("DL_DIR", True), "repo")
+ gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
+ ud.codir = os.path.join(repodir, gitsrcname, ud.manifest)
+ ud.repodir = os.path.join(ud.codir, "repo")
def download(self, ud, d):
"""Fetch url"""
@@ -60,22 +64,17 @@ class Repo(FetchMethod):
logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
return
- gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
- repodir = d.getVar("REPODIR") or os.path.join(d.getVar("DL_DIR"), "repo")
- codir = os.path.join(repodir, gitsrcname, ud.manifest)
-
if ud.user:
username = ud.user + "@"
else:
username = ""
- repodir = os.path.join(codir, "repo")
- bb.utils.mkdirhier(repodir)
+ bb.utils.mkdirhier(ud.repodir)
bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
- runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
+ runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=ud.repodir)
bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
- runfetchcmd("repo sync", d, workdir=repodir)
+ runfetchcmd("repo sync", d, workdir=ud.repodir)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
@@ -84,7 +83,12 @@ class Repo(FetchMethod):
tar_flags = "--exclude='.repo' --exclude='.git'"
# Create a cache
- runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d, workdir=codir)
+ runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d, workdir=ud.codir)
+
+ def unpack(self, ud, destdir, d):
+ FetchMethod.unpack(self, ud, destdir, d)
+ bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
+ runfetchcmd("repo sync", d, workdir=ud.repodir)
def supports_srcrev(self):
return False
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [PATCH 3/4] fetch2/repo: Make fetcher always sync on unpack
2017-10-24 11:35 ` [PATCH 3/4] fetch2/repo: Make fetcher always sync on unpack Oleksandr Andrushchenko
@ 2017-11-07 13:45 ` Richard Purdie
2017-11-07 19:31 ` Oleksandr Andrushchenko
0 siblings, 1 reply; 8+ messages in thread
From: Richard Purdie @ 2017-11-07 13:45 UTC (permalink / raw)
To: Oleksandr Andrushchenko, bitbake-devel; +Cc: Oleksandr Andrushchenko
On Tue, 2017-10-24 at 14:35 +0300, Oleksandr Andrushchenko wrote:
> From: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
>
> Currently if repo has fetched the repository and
> cached it to a tar file it never runs repo sync again,
> thus making the build use potentially outdated repo.
> Fix this by implementing unpack functionality which runs
> repo sync.
This sounds incorrect to me. The unpack task is not meant to touch the
network at all, all network access should be done in the fetch
function.
Cheers,
Richard
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 3/4] fetch2/repo: Make fetcher always sync on unpack
2017-11-07 13:45 ` Richard Purdie
@ 2017-11-07 19:31 ` Oleksandr Andrushchenko
2017-11-08 18:09 ` Richard Purdie
0 siblings, 1 reply; 8+ messages in thread
From: Oleksandr Andrushchenko @ 2017-11-07 19:31 UTC (permalink / raw)
To: Richard Purdie, bitbake-devel; +Cc: Oleksandr Andrushchenko
On 11/07/2017 03:45 PM, Richard Purdie wrote:
> On Tue, 2017-10-24 at 14:35 +0300, Oleksandr Andrushchenko wrote:
>> From: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
>>
>> Currently if repo has fetched the repository and
>> cached it to a tar file it never runs repo sync again,
>> thus making the build use potentially outdated repo.
>> Fix this by implementing unpack functionality which runs
>> repo sync.
> This sounds incorrect to me. The unpack task is not meant to touch the
> network at all, all network access should be done in the fetch
> function.
agree, so, instead this should be done via supporting source
revisions, so fetcher can deal with it? Is this understanding correct?
> Cheers,
>
> Richard
Thank you,
Oleksandr
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 3/4] fetch2/repo: Make fetcher always sync on unpack
2017-11-07 19:31 ` Oleksandr Andrushchenko
@ 2017-11-08 18:09 ` Richard Purdie
0 siblings, 0 replies; 8+ messages in thread
From: Richard Purdie @ 2017-11-08 18:09 UTC (permalink / raw)
To: Oleksandr Andrushchenko, bitbake-devel; +Cc: Oleksandr Andrushchenko
On Tue, 2017-11-07 at 21:31 +0200, Oleksandr Andrushchenko wrote:
> On 11/07/2017 03:45 PM, Richard Purdie wrote:
> >
> > On Tue, 2017-10-24 at 14:35 +0300, Oleksandr Andrushchenko wrote:
> > >
> > > From: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
> > >
> > > Currently if repo has fetched the repository and
> > > cached it to a tar file it never runs repo sync again,
> > > thus making the build use potentially outdated repo.
> > > Fix this by implementing unpack functionality which runs
> > > repo sync.
> > This sounds incorrect to me. The unpack task is not meant to touch
> > the
> > network at all, all network access should be done in the fetch
> > function.
> agree, so, instead this should be done via supporting source
>
> revisions, so fetcher can deal with it? Is this understanding
> correct?
Yes, correct.
Cheers,
Richard
^ permalink raw reply [flat|nested] 8+ messages in thread
* [PATCH 4/4] fetch2/repo: Use multiple jobs to fetch and sync
2017-10-24 11:35 [PATCH 0/4] fetch2/repo: Repo fetcher fixes and improvements Oleksandr Andrushchenko
` (2 preceding siblings ...)
2017-10-24 11:35 ` [PATCH 3/4] fetch2/repo: Make fetcher always sync on unpack Oleksandr Andrushchenko
@ 2017-10-24 11:35 ` Oleksandr Andrushchenko
3 siblings, 0 replies; 8+ messages in thread
From: Oleksandr Andrushchenko @ 2017-10-24 11:35 UTC (permalink / raw)
To: bitbake-devel; +Cc: andr2000, Oleksandr Andrushchenko
From: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
Google repo supports "--jobs" parameter, so one can
speed up the process of fetching and syncing. According to
https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/ny3nkgE9AE8
it is even more gets to speed if used separately for
network jobs and "local" ones.
Use BB_NUMBER_THREADS to run parallel repo sync.
Signed-off-by: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
---
lib/bb/fetch2/repo.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/bb/fetch2/repo.py b/lib/bb/fetch2/repo.py
index 3c28fe04ab1e..08331acd6706 100644
--- a/lib/bb/fetch2/repo.py
+++ b/lib/bb/fetch2/repo.py
@@ -57,6 +57,13 @@ class Repo(FetchMethod):
ud.codir = os.path.join(repodir, gitsrcname, ud.manifest)
ud.repodir = os.path.join(ud.codir, "repo")
+ def sync(self, ud, d):
+ # repo can separate the "network" sync (the git fetch part, network bound)
+ # from the local sync (the git rebase part, compute & i/o bound).
+ num_jobs = d.getVar("BB_NUMBER_THREADS", True) or "1"
+ runfetchcmd("repo sync -c -n -j%s" % num_jobs, d, workdir=ud.repodir)
+ runfetchcmd("repo sync -c -l -j%s" % num_jobs, d, workdir=ud.repodir)
+
def download(self, ud, d):
"""Fetch url"""
@@ -74,7 +81,7 @@ class Repo(FetchMethod):
runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=ud.repodir)
bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
- runfetchcmd("repo sync", d, workdir=ud.repodir)
+ self.sync(ud, d)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
@@ -88,7 +95,7 @@ class Repo(FetchMethod):
def unpack(self, ud, destdir, d):
FetchMethod.unpack(self, ud, destdir, d)
bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
- runfetchcmd("repo sync", d, workdir=ud.repodir)
+ self.sync(ud, d)
def supports_srcrev(self):
return False
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread