* [PATCH 01/11] bitbake/fetch2/git: Add backwards compatibility code for branch name handling
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 02/11] bb.fetch2: add unpack method in fetcher Yu Ke
` (9 subsequent siblings)
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
this patch has been applied for bitbake/fetch/git, so also apply it for bitbake/fetch/git
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bitbake/lib/bb/fetch2/git.py | 82 ++++++++++++++++++++++++++++++++++++++++-
1 files changed, 80 insertions(+), 2 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index c621457..e85f1da 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -22,6 +22,7 @@ BitBake 'Fetch' git implementation
import os
import bb
+import bb.persist_data
from bb import data
from bb.fetch2 import Fetch
from bb.fetch2 import runfetchcmd
@@ -116,6 +117,7 @@ class Git(Fetch):
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
+
coname = '%s' % (ud.tag)
codir = os.path.join(ud.clonedir, coname)
@@ -205,11 +207,19 @@ class Git(Fetch):
output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True)
return output.split()[0] != "0"
- def _revision_key(self, url, ud, d):
+ def _revision_key(self, url, ud, d, branch=False):
"""
Return a unique key for the url
"""
- return "git:" + ud.host + ud.path.replace('/', '.') + ud.branch
+ key = 'git:' + ud.host + ud.path.replace('/', '.')
+ if branch:
+ return key + ud.branch
+ else:
+ return key
+
+ def generate_revision_key(self, url, ud, d, branch=False):
+ key = self._revision_key(url, ud, d, branch)
+ return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
def _latest_revision(self, url, ud, d):
"""
@@ -227,6 +237,74 @@ class Git(Fetch):
raise bb.fetch2.FetchError("Fetch command %s gave empty output\n" % (cmd))
return output.split()[0]
+ def latest_revision(self, url, ud, d):
+ """
+ Look in the cache for the latest revision, if not present ask the SCM.
+ """
+ persisted = bb.persist_data.persist(d)
+ revs = persisted['BB_URI_HEADREVS']
+
+ key = self.generate_revision_key(url, ud, d, branch=True)
+ rev = revs[key]
+ if rev is None:
+ # Compatibility with old key format, no branch included
+ oldkey = self.generate_revision_key(url, ud, d, branch=False)
+ rev = revs[oldkey]
+ if rev is not None:
+ del revs[oldkey]
+ else:
+ rev = self._latest_revision(url, ud, d)
+ revs[key] = rev
+
+ return str(rev)
+
+ def sortable_revision(self, url, ud, d):
+ """
+
+ """
+ pd = bb.persist_data.persist(d)
+ localcounts = pd['BB_URI_LOCALCOUNT']
+ key = self.generate_revision_key(url, ud, d, branch=True)
+ oldkey = self.generate_revision_key(url, ud, d, branch=False)
+
+ latest_rev = self._build_revision(url, ud, d)
+ last_rev = localcounts[key + '_rev']
+ if last_rev is None:
+ last_rev = localcounts[oldkey + '_rev']
+ if last_rev is not None:
+ del localcounts[oldkey + '_rev']
+ localcounts[key + '_rev'] = last_rev
+
+ uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False
+ count = None
+ if uselocalcount:
+ count = Fetch.localcount_internal_helper(ud, d)
+ if count is None:
+ count = localcounts[key + '_count']
+ if count is None:
+ count = localcounts[oldkey + '_count']
+ if count is not None:
+ del localcounts[oldkey + '_count']
+ localcounts[key + '_count'] = count
+
+ if last_rev == latest_rev:
+ return str(count + "+" + latest_rev)
+
+ buildindex_provided = hasattr(self, "_sortable_buildindex")
+ if buildindex_provided:
+ count = self._sortable_buildindex(url, ud, d, latest_rev)
+ if count is None:
+ count = "0"
+ elif uselocalcount or buildindex_provided:
+ count = str(count)
+ else:
+ count = str(int(count) + 1)
+
+ localcounts[key + '_rev'] = latest_rev
+ localcounts[key + '_count'] = count
+
+ return str(count + "+" + latest_rev)
+
def _build_revision(self, url, ud, d):
return ud.tag
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 02/11] bb.fetch2: add unpack method in fetcher
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
2011-01-18 17:48 ` [PATCH 01/11] bitbake/fetch2/git: Add backwards compatibility code for branch name handling Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 03/11] bb.fetch2: revise the Fetch.unpack API Yu Ke
` (8 subsequent siblings)
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
copy exactly the base.bbclass:oe_unpack_file() to bb.fetch2 as the code base
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 84 +++++++++++++++++++++++++++++++++++++
1 files changed, 84 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 751e514..4b4ac47 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -259,6 +259,13 @@ def verify_checksum(u, ud, d):
ud.sha256_expected, sha256data)
raise FetchError("%s checksum mismatch." % u)
+def subprocess_setup():
+ import signal
+ # Python installs a SIGPIPE handler by default. This is usually not what
+ # non-Python subprocesses expect.
+ # SIGPIPE errors are known issues with gzip/bash
+ signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
def go(d, urls = None):
"""
Fetch all urls
@@ -629,6 +636,83 @@ class Fetch(object):
"""
raise NoMethodError("Missing implementation for url")
+ def unpack(file, data, url = None):
+ import subprocess
+ if not url:
+ url = "file://%s" % file
+ dots = file.split(".")
+ if dots[-1] in ['gz', 'bz2', 'Z']:
+ efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
+ else:
+ efile = file
+ cmd = None
+ if file.endswith('.tar'):
+ cmd = 'tar x --no-same-owner -f %s' % file
+ elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
+ cmd = 'tar xz --no-same-owner -f %s' % file
+ elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
+ cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
+ elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
+ cmd = 'gzip -dc %s > %s' % (file, efile)
+ elif file.endswith('.bz2'):
+ cmd = 'bzip2 -dc %s > %s' % (file, efile)
+ elif file.endswith('.tar.xz'):
+ cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
+ elif file.endswith('.xz'):
+ cmd = 'xz -dc %s > %s' % (file, efile)
+ elif file.endswith('.zip') or file.endswith('.jar'):
+ cmd = 'unzip -q -o'
+ (type, host, path, user, pswd, parm) = bb.decodeurl(url)
+ if 'dos' in parm:
+ cmd = '%s -a' % cmd
+ cmd = "%s '%s'" % (cmd, file)
+ elif os.path.isdir(file):
+ filesdir = os.path.realpath(bb.data.getVar("FILESDIR", data, 1))
+ destdir = "."
+ if file[0:len(filesdir)] == filesdir:
+ destdir = file[len(filesdir):file.rfind('/')]
+ destdir = destdir.strip('/')
+ if len(destdir) < 1:
+ destdir = "."
+ elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
+ os.makedirs("%s/%s" % (os.getcwd(), destdir))
+ cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
+ else:
+ (type, host, path, user, pswd, parm) = bb.decodeurl(url)
+ if not 'patch' in parm:
+ # The "destdir" handling was specifically done for FILESPATH
+ # items. So, only do so for file:// entries.
+ if type == "file" and path.find("/") != -1:
+ destdir = path.rsplit("/", 1)[0]
+ else:
+ destdir = "."
+ bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
+ cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
+
+ if not cmd:
+ return True
+
+ dest = os.path.join(os.getcwd(), os.path.basename(file))
+ if os.path.exists(dest):
+ if os.path.samefile(file, dest):
+ return True
+
+ # Change to subdir before executing command
+ save_cwd = os.getcwd();
+ parm = bb.decodeurl(url)[5]
+ if 'subdir' in parm:
+ newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
+ bb.mkdirhier(newdir)
+ os.chdir(newdir)
+
+ cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd)
+ bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
+ ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+
+ os.chdir(save_cwd)
+
+ return ret == 0
+
def try_premirror(self, url, urldata, d):
"""
Should premirrors be used?
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 03/11] bb.fetch2: revise the Fetch.unpack API
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
2011-01-18 17:48 ` [PATCH 01/11] bitbake/fetch2/git: Add backwards compatibility code for branch name handling Yu Ke
2011-01-18 17:48 ` [PATCH 02/11] bb.fetch2: add unpack method in fetcher Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 04/11] bb.fetch: add fetch version to distinguish bb.fetch and bb.fetch2 Yu Ke
` (7 subsequent siblings)
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
use the urldata and rootdir parameter for more natural info passing:
- urldata is the FetchData instance
- rootdir is the dir to put the extracted source. the original unpack use current dir (os.getcwd) as destination dir, which is not flexible and error-prone
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 29 +++++++++++++----------------
1 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 4b4ac47..7e0f420 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -636,10 +636,9 @@ class Fetch(object):
"""
raise NoMethodError("Missing implementation for url")
- def unpack(file, data, url = None):
+ def unpack(self, urldata, rootdir, data):
import subprocess
- if not url:
- url = "file://%s" % file
+ file = urldata.localpath
dots = file.split(".")
if dots[-1] in ['gz', 'bz2', 'Z']:
efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
@@ -662,8 +661,7 @@ class Fetch(object):
cmd = 'xz -dc %s > %s' % (file, efile)
elif file.endswith('.zip') or file.endswith('.jar'):
cmd = 'unzip -q -o'
- (type, host, path, user, pswd, parm) = bb.decodeurl(url)
- if 'dos' in parm:
+ if 'dos' in urldata.parm:
cmd = '%s -a' % cmd
cmd = "%s '%s'" % (cmd, file)
elif os.path.isdir(file):
@@ -674,34 +672,33 @@ class Fetch(object):
destdir = destdir.strip('/')
if len(destdir) < 1:
destdir = "."
- elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
- os.makedirs("%s/%s" % (os.getcwd(), destdir))
- cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
+ elif not os.access("%s/%s" % (rootdir, destdir), os.F_OK):
+ os.makedirs("%s/%s" % (rootdir, destdir))
+ cmd = 'cp -pPR %s %s/%s/' % (file, rootdir, destdir)
else:
- (type, host, path, user, pswd, parm) = bb.decodeurl(url)
- if not 'patch' in parm:
+ if not 'patch' in urldata.parm:
# The "destdir" handling was specifically done for FILESPATH
# items. So, only do so for file:// entries.
if type == "file" and path.find("/") != -1:
destdir = path.rsplit("/", 1)[0]
else:
destdir = "."
- bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
- cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
+ bb.mkdirhier("%s/%s" % (rootdir, destdir))
+ cmd = 'cp %s %s/%s/' % (file, rootdir, destdir)
if not cmd:
return True
- dest = os.path.join(os.getcwd(), os.path.basename(file))
+ dest = os.path.join(rootdir, os.path.basename(file))
if os.path.exists(dest):
if os.path.samefile(file, dest):
return True
# Change to subdir before executing command
save_cwd = os.getcwd();
- parm = bb.decodeurl(url)[5]
- if 'subdir' in parm:
- newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
+ os.chdir(rootdir)
+ if 'subdir' in urldata.parm:
+ newdir = ("%s/%s" % (rootdir, urldata.parm['subdir']))
bb.mkdirhier(newdir)
os.chdir(newdir)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 04/11] bb.fetch: add fetch version to distinguish bb.fetch and bb.fetch2
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
` (2 preceding siblings ...)
2011-01-18 17:48 ` [PATCH 03/11] bb.fetch2: revise the Fetch.unpack API Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 05/11] base.bbclass: use bb.fetch2 unpack API Yu Ke
` (6 subsequent siblings)
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch/__init__.py | 2 ++
bitbake/lib/bb/fetch2/__init__.py | 2 ++
2 files changed, 4 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index 07eb77d..49fe502 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -32,6 +32,8 @@ import bb
from bb import data
from bb import persist_data
+__version__ = "1"
+
logger = logging.getLogger("BitBake.Fetch")
class MalformedUrl(Exception):
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 7e0f420..ef1a52a 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -32,6 +32,8 @@ import bb
from bb import data
from bb import persist_data
+__version__ = "2"
+
logger = logging.getLogger("BitBake.Fetch")
class MalformedUrl(Exception):
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 05/11] base.bbclass: use bb.fetch2 unpack API
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
` (3 preceding siblings ...)
2011-01-18 17:48 ` [PATCH 04/11] bb.fetch: add fetch version to distinguish bb.fetch and bb.fetch2 Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 06/11] bb.fetch2: rename "go" with "download" to better reflect its functionality Yu Ke
` (5 subsequent siblings)
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
meta/classes/base.bbclass | 8 +++++++-
1 files changed, 7 insertions(+), 1 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index f1ffb45..23b17f2 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -256,7 +256,13 @@ python base_do_unpack() {
continue
local = os.path.realpath(local)
lf = bb.utils.lockfile(urldata[url].lockfile)
- ret = oe_unpack_file(local, localdata, url)
+ if bb.fetch.__version__ == "1":
+ ret = oe_unpack_file(local, localdata, url)
+ else:
+ # use bb.fetch2 unpack API
+ ud = urldata[url]
+ rootdir = bb.data.getVar('WORKDIR', localdata, True)
+ ret = ud.method.unpack(ud, rootdir, localdata)
bb.utils.unlockfile(lf)
if not ret:
raise bb.build.FuncFailed("oe_unpack_file failed with return value %s" % ret)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 06/11] bb.fetch2: rename "go" with "download" to better reflect its functionality
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
` (4 preceding siblings ...)
2011-01-18 17:48 ` [PATCH 05/11] base.bbclass: use bb.fetch2 unpack API Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 07/11] bbclasee: rename "go" with "download" Yu Ke
` (4 subsequent siblings)
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
no functional change
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 8 ++++----
bitbake/lib/bb/fetch2/bzr.py | 2 +-
bitbake/lib/bb/fetch2/cvs.py | 2 +-
bitbake/lib/bb/fetch2/git.py | 6 +++---
bitbake/lib/bb/fetch2/hg.py | 2 +-
bitbake/lib/bb/fetch2/local.py | 2 +-
bitbake/lib/bb/fetch2/osc.py | 2 +-
bitbake/lib/bb/fetch2/perforce.py | 2 +-
bitbake/lib/bb/fetch2/repo.py | 2 +-
bitbake/lib/bb/fetch2/ssh.py | 2 +-
bitbake/lib/bb/fetch2/svk.py | 2 +-
bitbake/lib/bb/fetch2/svn.py | 2 +-
bitbake/lib/bb/fetch2/wget.py | 4 ++--
13 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index ef1a52a..8d0f907 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -268,7 +268,7 @@ def subprocess_setup():
# SIGPIPE errors are known issues with gzip/bash
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-def go(d, urls = None):
+def download(d, urls = None):
"""
Fetch all urls
init must have previously been called
@@ -298,7 +298,7 @@ def go(d, urls = None):
if m.forcefetch(u, ud, d) or not localpath:
# Next try fetching from the original uri, u
try:
- m.go(u, ud, d)
+ m.download(u, ud, d)
localpath = ud.localpath
except FetchError:
# Remove any incomplete file
@@ -504,7 +504,7 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
if found:
return found
else:
- ud.method.go(newuri, ud, ld)
+ ud.method.download(newuri, ud, ld)
return ud.localpath
except (bb.fetch2.MissingParameterError,
bb.fetch2.FetchError,
@@ -631,7 +631,7 @@ class Fetch(object):
"""
return False
- def go(self, url, urldata, d):
+ def download(self, url, urldata, d):
"""
Fetch urls
Assumes localpath was called first
diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py
index 97b042b..608ecc7 100644
--- a/bitbake/lib/bb/fetch2/bzr.py
+++ b/bitbake/lib/bb/fetch2/bzr.py
@@ -79,7 +79,7 @@ class Bzr(Fetch):
return bzrcmd
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
diff --git a/bitbake/lib/bb/fetch2/cvs.py b/bitbake/lib/bb/fetch2/cvs.py
index 1570cab..8e72090 100644
--- a/bitbake/lib/bb/fetch2/cvs.py
+++ b/bitbake/lib/bb/fetch2/cvs.py
@@ -72,7 +72,7 @@ class Cvs(Fetch):
return True
return False
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
method = ud.parm.get('method', 'pserver')
localdir = ud.parm.get('localdir', ud.module)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index e85f1da..c40a19c 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -107,7 +107,7 @@ class Git(Fetch):
return True
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
if ud.user:
@@ -320,7 +320,7 @@ class Git(Fetch):
if not os.path.exists(ud.clonedir):
print("no repo")
- self.go(None, ud, d)
+ self.download(None, ud, d)
if not os.path.exists(ud.clonedir):
logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir)
return None
@@ -328,7 +328,7 @@ class Git(Fetch):
os.chdir(ud.clonedir)
if not self._contains_ref(rev, d):
- self.go(None, ud, d)
+ self.download(None, ud, d)
output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True)
os.chdir(cwd)
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index 0ba8433..635ecbf 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -112,7 +112,7 @@ class Hg(Fetch):
return cmd
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py
index bcb30df..89fbdf6 100644
--- a/bitbake/lib/bb/fetch2/local.py
+++ b/bitbake/lib/bb/fetch2/local.py
@@ -56,7 +56,7 @@ class Local(Fetch):
# We don't set localfile as for this fetcher the file is already local!
return newpath
- def go(self, url, urldata, d):
+ def download(self, url, urldata, d):
"""Fetch urls (no-op for Local method)"""
# no need to fetch local files, we'll deal with them in place.
return 1
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
index 06ac5a9..619e2f1 100644
--- a/bitbake/lib/bb/fetch2/osc.py
+++ b/bitbake/lib/bb/fetch2/osc.py
@@ -79,7 +79,7 @@ class Osc(Fetch):
return osccmd
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""
Fetch url
"""
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py
index 18b2781..bda0bb8 100644
--- a/bitbake/lib/bb/fetch2/perforce.py
+++ b/bitbake/lib/bb/fetch2/perforce.py
@@ -121,7 +121,7 @@ class Perforce(Fetch):
return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""
Fetch urls
"""
diff --git a/bitbake/lib/bb/fetch2/repo.py b/bitbake/lib/bb/fetch2/repo.py
index 3330957..510ba46 100644
--- a/bitbake/lib/bb/fetch2/repo.py
+++ b/bitbake/lib/bb/fetch2/repo.py
@@ -55,7 +55,7 @@ class Repo(Fetch):
return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 8b28322..78f55a6 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -74,7 +74,7 @@ class SSH(Fetch):
lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path))
return lpath
- def go(self, url, urldata, d):
+ def download(self, url, urldata, d):
dldir = data.getVar('DL_DIR', d, 1)
m = __pattern__.match(url)
diff --git a/bitbake/lib/bb/fetch2/svk.py b/bitbake/lib/bb/fetch2/svk.py
index 7990ff2..3bb4c38 100644
--- a/bitbake/lib/bb/fetch2/svk.py
+++ b/bitbake/lib/bb/fetch2/svk.py
@@ -57,7 +57,7 @@ class Svk(Fetch):
def forcefetch(self, url, ud, d):
return ud.date == "now"
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch urls"""
svkroot = ud.host + ud.path
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index 1116795..547c04f 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -128,7 +128,7 @@ class Svn(Fetch):
return svncmd
- def go(self, loc, ud, d):
+ def download(self, loc, ud, d):
"""Fetch url"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index cf36cca..91cfafb 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -48,7 +48,7 @@ class Wget(Fetch):
return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
- def go(self, uri, ud, d, checkonly = False):
+ def download(self, uri, ud, d, checkonly = False):
"""Fetch urls"""
def fetch_uri(uri, ud, d):
@@ -90,4 +90,4 @@ class Wget(Fetch):
def checkstatus(self, uri, ud, d):
- return self.go(uri, ud, d, True)
+ return self.download(uri, ud, d, True)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 07/11] bbclasee: rename "go" with "download"
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
` (5 preceding siblings ...)
2011-01-18 17:48 ` [PATCH 06/11] bb.fetch2: rename "go" with "download" to better reflect its functionality Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 08/11] bb.fetch2.git: split download to download() + build_mirror_data() Yu Ke
` (3 subsequent siblings)
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
no functional change
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
meta/classes/base.bbclass | 5 ++++-
meta/classes/sstate.bbclass | 5 ++++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 23b17f2..d1d60f7 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -135,7 +135,10 @@ python base_do_fetch() {
raise bb.build.FuncFailed("Malformed URL: %s" % value)
try:
- bb.fetch.go(localdata)
+ if bb.fetch.__version__ == "1":
+ bb.fetch.go(localdata)
+ else:
+ bb.fetch.download(localdata)
except bb.fetch.MissingParameterError:
(type, value, traceback) = sys.exc_info()
raise bb.build.FuncFailed("Missing parameters: %s" % value)
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index b6e6c92..fa184bc 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -319,7 +319,10 @@ def pstaging_fetch(sstatepkg, d):
# we will build the package
try:
bb.fetch.init([srcuri], localdata)
- bb.fetch.go(localdata, [srcuri])
+ if bb.fetch.__version__ == "1":
+ bb.fetch.go(localdata, [srcuri])
+ else:
+ bb.fetch.download(localdata, [srcuri])
# Need to optimise this, if using file:// urls, the fetcher just changes the local path
# For now work around by symlinking
localpath = bb.data.expand(bb.fetch.localpath(srcuri, localdata), localdata)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 08/11] bb.fetch2.git: split download to download() + build_mirror_data()
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
` (6 preceding siblings ...)
2011-01-18 17:48 ` [PATCH 07/11] bbclasee: rename "go" with "download" Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 09/11] bb.fetch2: remove the obsolate Fetch.try_mirrors referrence Yu Ke
` (2 subsequent siblings)
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
As the name implies, the download is to fetch the source from URL, build_mirror_data is to create the mirror tar ball. the original go() method mix them together, it is more clean to split them.
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/__init__.py | 4 ++++
bitbake/lib/bb/fetch2/git.py | 8 +++++---
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 8d0f907..3ff5b7b 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -299,6 +299,8 @@ def download(d, urls = None):
# Next try fetching from the original uri, u
try:
m.download(u, ud, d)
+ if hasattr(m, "build_mirror_data"):
+ m.build_mirror_data(u, ud, d)
localpath = ud.localpath
except FetchError:
# Remove any incomplete file
@@ -505,6 +507,8 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
return found
else:
ud.method.download(newuri, ud, ld)
+ if hasattr(ud.method,"build_mirror_data"):
+ ud.method.build_mirror_data(newuri, ud, ld)
return ud.localpath
except (bb.fetch2.MissingParameterError,
bb.fetch2.FetchError,
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index c40a19c..690bb08 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -118,9 +118,6 @@ class Git(Fetch):
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
- coname = '%s' % (ud.tag)
- codir = os.path.join(ud.clonedir, coname)
-
# If we have no existing clone and no mirror tarball, try and obtain one
if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
try:
@@ -151,7 +148,12 @@ class Git(Fetch):
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+ def build_mirror_data(self, url, ud, d):
# Generate a mirror tarball if needed
+ coname = '%s' % (ud.tag)
+ codir = os.path.join(ud.clonedir, coname)
+ repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
+
os.chdir(ud.clonedir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0" or 'fullclone' in ud.parm:
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 09/11] bb.fetch2: remove the obsolate Fetch.try_mirrors referrence
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
` (7 preceding siblings ...)
2011-01-18 17:48 ` [PATCH 08/11] bb.fetch2.git: split download to download() + build_mirror_data() Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-18 17:48 ` [PATCH 10/11] bb.fetch2: add git unpack Yu Ke
2011-01-18 17:48 ` [PATCH 11/11] git.py: remove the source tree tar ball Yu Ke
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
Fetch.try_mirrors is no longer exists, so the code is obsolate
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/git.py | 8 --------
1 files changed, 0 insertions(+), 8 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 690bb08..438756a 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -117,14 +117,6 @@ class Git(Fetch):
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
-
- # If we have no existing clone and no mirror tarball, try and obtain one
- if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
- try:
- Fetch.try_mirrors(ud.mirrortarball)
- except:
- pass
-
# If the checkout doesn't exist and the mirror tarball does, extract it
if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
bb.mkdirhier(ud.clonedir)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* [PATCH 10/11] bb.fetch2: add git unpack
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
` (8 preceding siblings ...)
2011-01-18 17:48 ` [PATCH 09/11] bb.fetch2: remove the obsolate Fetch.try_mirrors referrence Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
2011-01-25 18:07 ` Richard Purdie
2011-01-18 17:48 ` [PATCH 11/11] git.py: remove the source tree tar ball Yu Ke
10 siblings, 1 reply; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
git download will clone git repo to local, and git unpack just do a clone with referrence to the original repo
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/git.py | 16 ++++++++++++++++
1 files changed, 16 insertions(+), 0 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 438756a..bb62a87 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -193,6 +193,22 @@ class Git(Fetch):
os.chdir(ud.clonedir)
bb.utils.prunedir(codir)
+ def unpack(self, ud, destdir, d):
+ """ unpack the downloaded src to destdir"""
+ subdir = ud.parm.get("subpath", "")
+ if subdir != "":
+ readpathspec = ":%s" % (subdir)
+ else:
+ readpathspec = ""
+
+ # extract to ${WORKDIR}/git
+ destdir = destdir + "/git/"
+
+ os.chdir(ud.clonedir)
+ runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+ runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
+ return 1
+
def supports_srcrev(self):
return True
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [PATCH 10/11] bb.fetch2: add git unpack
2011-01-18 17:48 ` [PATCH 10/11] bb.fetch2: add git unpack Yu Ke
@ 2011-01-25 18:07 ` Richard Purdie
2011-01-25 19:39 ` Bruce Ashfield
0 siblings, 1 reply; 18+ messages in thread
From: Richard Purdie @ 2011-01-25 18:07 UTC (permalink / raw)
To: Yu Ke; +Cc: poky
Hi Ke,
On Wed, 2011-01-19 at 01:48 +0800, Yu Ke wrote:
> git download will clone git repo to local, and git unpack just do a clone with referrence to the original repo
>
> Signed-off-by: Yu Ke <ke.yu@intel.com>
> ---
> bitbake/lib/bb/fetch2/git.py | 16 ++++++++++++++++
> 1 files changed, 16 insertions(+), 0 deletions(-)
>
> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
> index 438756a..bb62a87 100644
> --- a/bitbake/lib/bb/fetch2/git.py
> +++ b/bitbake/lib/bb/fetch2/git.py
> @@ -193,6 +193,22 @@ class Git(Fetch):
> os.chdir(ud.clonedir)
> bb.utils.prunedir(codir)
>
> + def unpack(self, ud, destdir, d):
> + """ unpack the downloaded src to destdir"""
> + subdir = ud.parm.get("subpath", "")
> + if subdir != "":
> + readpathspec = ":%s" % (subdir)
> + else:
> + readpathspec = ""
> +
> + # extract to ${WORKDIR}/git
> + destdir = destdir + "/git/"
> +
> + os.chdir(ud.clonedir)
> + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
> + runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
> + return 1
> +
> def supports_srcrev(self):
> return True
I know we talked about this via email and you revised the description
to:
http://git.pokylinux.org/cgit/cgit.cgi/poky-contrib/commit/?h=kyu3/fetcher-api-v3&id=6df3037ea3679b804c2679afba8377b7e3796aca
I'm wondering if we could optimise this cp command to be a "git clone
-l" or a "git clone -s" (see man git-clone). I'm cc'ing Bruce for his
thoughts on this and whether the linux-yocto tools would have a problem
with either of these.
The idea would be one central checkout with the references in and then
the subcomponents would all just reference the one local clone saving
diskspace and time.
Cheers,
Richard
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH 10/11] bb.fetch2: add git unpack
2011-01-25 18:07 ` Richard Purdie
@ 2011-01-25 19:39 ` Bruce Ashfield
2011-01-26 4:41 ` Yu Ke
0 siblings, 1 reply; 18+ messages in thread
From: Bruce Ashfield @ 2011-01-25 19:39 UTC (permalink / raw)
To: Richard Purdie; +Cc: poky
On Tue, Jan 25, 2011 at 1:07 PM, Richard Purdie
<richard.purdie@linuxfoundation.org> wrote:
> Hi Ke,
>
> On Wed, 2011-01-19 at 01:48 +0800, Yu Ke wrote:
>> git download will clone git repo to local, and git unpack just do a clone with referrence to the original repo
>>
>> Signed-off-by: Yu Ke <ke.yu@intel.com>
>> ---
>> bitbake/lib/bb/fetch2/git.py | 16 ++++++++++++++++
>> 1 files changed, 16 insertions(+), 0 deletions(-)
>>
>> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
>> index 438756a..bb62a87 100644
>> --- a/bitbake/lib/bb/fetch2/git.py
>> +++ b/bitbake/lib/bb/fetch2/git.py
>> @@ -193,6 +193,22 @@ class Git(Fetch):
>> os.chdir(ud.clonedir)
>> bb.utils.prunedir(codir)
>>
>> + def unpack(self, ud, destdir, d):
>> + """ unpack the downloaded src to destdir"""
>> + subdir = ud.parm.get("subpath", "")
>> + if subdir != "":
>> + readpathspec = ":%s" % (subdir)
>> + else:
>> + readpathspec = ""
>> +
>> + # extract to ${WORKDIR}/git
>> + destdir = destdir + "/git/"
>> +
>> + os.chdir(ud.clonedir)
>> + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
>> + runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
>> + return 1
>> +
>> def supports_srcrev(self):
>> return True
>
> I know we talked about this via email and you revised the description
> to:
>
> http://git.pokylinux.org/cgit/cgit.cgi/poky-contrib/commit/?h=kyu3/fetcher-api-v3&id=6df3037ea3679b804c2679afba8377b7e3796aca
>
> I'm wondering if we could optimise this cp command to be a "git clone
> -l" or a "git clone -s" (see man git-clone). I'm cc'ing Bruce for his
> thoughts on this and whether the linux-yocto tools would have a problem
> with either of these.
Neither will pose a problem, and for what it is worth, I think using git
for the copy from src/dest is a good idea. In other build systems, I'm
doing the same thing with --shared (so I know for sure that works), and
--local should also be fine.
>
> The idea would be one central checkout with the references in and then
> the subcomponents would all just reference the one local clone saving
> diskspace and time.
Absolutely. And if someone does something to the central checkout,
minor errors ensue, but that can be recovered as well.
Cheers,
Bruce
>
> Cheers,
>
> Richard
>
>
>
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
>
--
"Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end"
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH 10/11] bb.fetch2: add git unpack
2011-01-25 19:39 ` Bruce Ashfield
@ 2011-01-26 4:41 ` Yu Ke
2011-01-26 4:43 ` Bruce Ashfield
0 siblings, 1 reply; 18+ messages in thread
From: Yu Ke @ 2011-01-26 4:41 UTC (permalink / raw)
To: Bruce Ashfield; +Cc: poky
On Jan 25, 14:39, Bruce Ashfield wrote:
> On Tue, Jan 25, 2011 at 1:07 PM, Richard Purdie
> <richard.purdie@linuxfoundation.org> wrote:
> > Hi Ke,
> >
> > On Wed, 2011-01-19 at 01:48 +0800, Yu Ke wrote:
> >> git download will clone git repo to local, and git unpack just do a clone with referrence to the original repo
> >>
> >> Signed-off-by: Yu Ke <ke.yu@intel.com>
> >> ---
> >> bitbake/lib/bb/fetch2/git.py | 16 ++++++++++++++++
> >> 1 files changed, 16 insertions(+), 0 deletions(-)
> >>
> >> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
> >> index 438756a..bb62a87 100644
> >> --- a/bitbake/lib/bb/fetch2/git.py
> >> +++ b/bitbake/lib/bb/fetch2/git.py
> >> @@ -193,6 +193,22 @@ class Git(Fetch):
> >> os.chdir(ud.clonedir)
> >> bb.utils.prunedir(codir)
> >>
> >> + def unpack(self, ud, destdir, d):
> >> + """ unpack the downloaded src to destdir"""
> >> + subdir = ud.parm.get("subpath", "")
> >> + if subdir != "":
> >> + readpathspec = ":%s" % (subdir)
> >> + else:
> >> + readpathspec = ""
> >> +
> >> + # extract to ${WORKDIR}/git
> >> + destdir = destdir + "/git/"
> >> +
> >> + os.chdir(ud.clonedir)
> >> + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
> >> + runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
> >> + return 1
> >> +
> >> def supports_srcrev(self):
> >> return True
> >
> > I know we talked about this via email and you revised the description
> > to:
> >
> > http://git.pokylinux.org/cgit/cgit.cgi/poky-contrib/commit/?h=kyu3/fetcher-api-v3&id=6df3037ea3679b804c2679afba8377b7e3796aca
> >
> > I'm wondering if we could optimise this cp command to be a "git clone
> > -l" or a "git clone -s" (see man git-clone). I'm cc'ing Bruce for his
> > thoughts on this and whether the linux-yocto tools would have a problem
> > with either of these.
>
> Neither will pose a problem, and for what it is worth, I think using git
> for the copy from src/dest is a good idea. In other build systems, I'm
> doing the same thing with --shared (so I know for sure that works), and
> --local should also be fine.
"git clone" is my first try when cooking the patch, unfortunately I meet one issue here. the git clone target dir is ${WORKDIR}, while git requires the target dir to be empty, which is not true for ${WORKDIR}, so git clone will fail. so I fall back to cp approach.
But I agree that git clone is better. So I'd like use this apprach: git clone to {WORKDIR}/git/, and modify the kernel-yocto.bbclass:do_kernel_checkout() to use the new layout.
I will revise the patch accordingly for review.
Regards
Ke
>
> >
> > The idea would be one central checkout with the references in and then
> > the subcomponents would all just reference the one local clone saving
> > diskspace and time.
>
> Absolutely. And if someone does something to the central checkout,
> minor errors ensue, but that can be recovered as well.
>
> Cheers,
>
> Bruce
>
> >
> > Cheers,
> >
> > Richard
> >
> >
> >
> > _______________________________________________
> > poky mailing list
> > poky@yoctoproject.org
> > https://lists.yoctoproject.org/listinfo/poky
> >
>
>
>
> --
> "Thou shalt not follow the NULL pointer, for chaos and madness await
> thee at its end"
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH 10/11] bb.fetch2: add git unpack
2011-01-26 4:41 ` Yu Ke
@ 2011-01-26 4:43 ` Bruce Ashfield
2011-01-26 8:25 ` Yu Ke
0 siblings, 1 reply; 18+ messages in thread
From: Bruce Ashfield @ 2011-01-26 4:43 UTC (permalink / raw)
To: Yu Ke; +Cc: poky
On Tue, Jan 25, 2011 at 11:41 PM, Yu Ke <ke.yu@intel.com> wrote:
> On Jan 25, 14:39, Bruce Ashfield wrote:
>> On Tue, Jan 25, 2011 at 1:07 PM, Richard Purdie
>> <richard.purdie@linuxfoundation.org> wrote:
>> > Hi Ke,
>> >
>> > On Wed, 2011-01-19 at 01:48 +0800, Yu Ke wrote:
>> >> git download will clone git repo to local, and git unpack just do a clone with referrence to the original repo
>> >>
>> >> Signed-off-by: Yu Ke <ke.yu@intel.com>
>> >> ---
>> >> bitbake/lib/bb/fetch2/git.py | 16 ++++++++++++++++
>> >> 1 files changed, 16 insertions(+), 0 deletions(-)
>> >>
>> >> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
>> >> index 438756a..bb62a87 100644
>> >> --- a/bitbake/lib/bb/fetch2/git.py
>> >> +++ b/bitbake/lib/bb/fetch2/git.py
>> >> @@ -193,6 +193,22 @@ class Git(Fetch):
>> >> os.chdir(ud.clonedir)
>> >> bb.utils.prunedir(codir)
>> >>
>> >> + def unpack(self, ud, destdir, d):
>> >> + """ unpack the downloaded src to destdir"""
>> >> + subdir = ud.parm.get("subpath", "")
>> >> + if subdir != "":
>> >> + readpathspec = ":%s" % (subdir)
>> >> + else:
>> >> + readpathspec = ""
>> >> +
>> >> + # extract to ${WORKDIR}/git
>> >> + destdir = destdir + "/git/"
>> >> +
>> >> + os.chdir(ud.clonedir)
>> >> + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
>> >> + runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
>> >> + return 1
>> >> +
>> >> def supports_srcrev(self):
>> >> return True
>> >
>> > I know we talked about this via email and you revised the description
>> > to:
>> >
>> > http://git.pokylinux.org/cgit/cgit.cgi/poky-contrib/commit/?h=kyu3/fetcher-api-v3&id=6df3037ea3679b804c2679afba8377b7e3796aca
>> >
>> > I'm wondering if we could optimise this cp command to be a "git clone
>> > -l" or a "git clone -s" (see man git-clone). I'm cc'ing Bruce for his
>> > thoughts on this and whether the linux-yocto tools would have a problem
>> > with either of these.
>>
>> Neither will pose a problem, and for what it is worth, I think using git
>> for the copy from src/dest is a good idea. In other build systems, I'm
>> doing the same thing with --shared (so I know for sure that works), and
>> --local should also be fine.
>
> "git clone" is my first try when cooking the patch, unfortunately I meet one issue here. the git clone target dir is ${WORKDIR}, while git requires the target dir to be empty, which is not true for ${WORKDIR}, so git clone will fail. so I fall back to cp approach.
>
> But I agree that git clone is better. So I'd like use this apprach: git clone to {WORKDIR}/git/, and modify the kernel-yocto.bbclass:do_kernel_checkout() to use the new layout.
> I will revise the patch accordingly for review.
> Regards
> Ke
This may be problematic. The kernel checkout needs to be done in a very
specific way. After the clone, you must have all branches local (no origin/)
and in the 'linux' directory. There are other tools that depend on
this particular
layout.
The most efficient way to do this has proven to be to do a bare clone
and then forcibly convert it to a non-bare clone. That gets you the
instantiation
and local branches in a single clone + minor fix ups to the directory.
Just some things to consider, I'll wait to see the patches before commenting
further.
Bruce
>
>>
>> >
>> > The idea would be one central checkout with the references in and then
>> > the subcomponents would all just reference the one local clone saving
>> > diskspace and time.
>>
>> Absolutely. And if someone does something to the central checkout,
>> minor errors ensue, but that can be recovered as well.
>>
>> Cheers,
>>
>> Bruce
>>
>> >
>> > Cheers,
>> >
>> > Richard
>> >
>> >
>> >
>> > _______________________________________________
>> > poky mailing list
>> > poky@yoctoproject.org
>> > https://lists.yoctoproject.org/listinfo/poky
>> >
>>
>>
>>
>> --
>> "Thou shalt not follow the NULL pointer, for chaos and madness await
>> thee at its end"
>> _______________________________________________
>> poky mailing list
>> poky@yoctoproject.org
>> https://lists.yoctoproject.org/listinfo/poky
>
--
"Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end"
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH 10/11] bb.fetch2: add git unpack
2011-01-26 4:43 ` Bruce Ashfield
@ 2011-01-26 8:25 ` Yu Ke
2011-01-26 15:01 ` Bruce Ashfield
0 siblings, 1 reply; 18+ messages in thread
From: Yu Ke @ 2011-01-26 8:25 UTC (permalink / raw)
To: Bruce Ashfield; +Cc: poky
On Jan 25, 23:43, Bruce Ashfield wrote:
> On Tue, Jan 25, 2011 at 11:41 PM, Yu Ke <ke.yu@intel.com> wrote:
> > "git clone" is my first try when cooking the patch, unfortunately I meet one issue here. the git clone target dir is ${WORKDIR}, while git requires the target dir to be empty, which is not true for ${WORKDIR}, so git clone will fail. so I fall back to cp approach.
> >
> > But I agree that git clone is better. So I'd like use this apprach: git clone to {WORKDIR}/git/, and modify the kernel-yocto.bbclass:do_kernel_checkout() to use the new layout.
> > I will revise the patch accordingly for review.
> > Regards
> > Ke
>
> This may be problematic. The kernel checkout needs to be done in a very
> specific way. After the clone, you must have all branches local (no origin/)
> and in the 'linux' directory. There are other tools that depend on
> this particular
> layout.
>
> The most efficient way to do this has proven to be to do a bare clone
> and then forcibly convert it to a non-bare clone. That gets you the
> instantiation
> and local branches in a single clone + minor fix ups to the directory.
>
> Just some things to consider, I'll wait to see the patches before commenting
> further.
>
> Bruce
Bruce, thanks for the comments. You are right. I realize that kernel checkout need to output a specific layout, so I only change the input layout of kernel checkout. more specificly, change the input dir from ${WORKDIR}/.git to ${WORKDIR}/git/.git. and the output layout is unchanged.
please see if the revised patch works for you:
git unpack patch - http://git.pokylinux.org/cgit/cgit.cgi/poky-contrib/commit/?h=kyu3/git-unpack&id=e654e3d75550282c4d112f1050465cde850c93e9
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 08daa20..eadbf33 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -194,6 +194,31 @@ class Git(Fetch):
os.chdir(ud.clonedir)
bb.utils.prunedir(codir)
+ def unpack(self, ud, destdir, d):
+ """ unpack the downloaded src to destdir"""
+
+ subdir = ud.parm.get("subpath", "")
+ if subdir != "":
+ readpathspec = ":%s" % (subdir)
+ else:
+ readpathspec = ""
+
+ destdir = os.path.join(destdir, "git/")
+ if os.path.exists(destdir):
+ bb.utils.prunedir(destdir)
+
+ if 'fullclone' in ud.parm:
+ runfetchcmd("git clone -l -n %s %s" % (ud.clonedir, destdir), d)
+ if os.path.exists("%s/.git/refs/remotes/origin" % ud.clonedir):
+ runfetchcmd("cp -af %s/.git/refs/remotes/origin/* %s/.git/refs/remotes/origin/" %(ud.clonedir, destdir), d)
+ if os.path.exists("%s/.git/packed-refs" % ud.clonedir):
+ runfetchcmd("cp -af %s/.git/packed-refs %s/.git/" %(ud.clonedir, destdir), d)
+ else:
+ os.chdir(ud.clonedir)
+ runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+ runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
+ return True
+
def supports_srcrev(self):
return True
kernel checkout patch - http://git.pokylinux.org/cgit/cgit.cgi/poky-contrib/commit/?h=kyu3/git-unpack&id=b1916da1f1a8b7e3c4b50d540918aa72be704d9a
diff --git a/meta/classes/kernel-yocto.bbclass b/meta/classes/kernel-yocto.bbclass
index 8479b39..ca1fb15 100644
--- a/meta/classes/kernel-yocto.bbclass
+++ b/meta/classes/kernel-yocto.bbclass
@@ -89,11 +89,11 @@ do_patch() {
}
do_kernel_checkout() {
- if [ -d ${WORKDIR}/.git/refs/remotes/origin ]; then
+ if [ -d ${WORKDIR}/git/.git/refs/remotes/origin ]; then
echo "Fixing up git directory for ${LINUX_KERNEL_TYPE}/${KMACHINE}"
rm -rf ${S}
mkdir ${S}
- mv ${WORKDIR}/.git ${S}
+ mv ${WORKDIR}/git/.git ${S}
if [ -e ${S}/.git/packed-refs ]; then
cd ${S}
>
> >
> >>
> >> >
> >> > The idea would be one central checkout with the references in and then
> >> > the subcomponents would all just reference the one local clone saving
> >> > diskspace and time.
> >>
> >> Absolutely. And if someone does something to the central checkout,
> >> minor errors ensue, but that can be recovered as well.
> >>
> >> Cheers,
> >>
> >> Bruce
> >>
> >> >
> >> > Cheers,
> >> >
> >> > Richard
> >> >
> >> >
> >> >
> >> > _______________________________________________
> >> > poky mailing list
> >> > poky@yoctoproject.org
> >> > https://lists.yoctoproject.org/listinfo/poky
> >> >
> >>
> >>
> >>
> >> --
> >> "Thou shalt not follow the NULL pointer, for chaos and madness await
> >> thee at its end"
> >> _______________________________________________
> >> poky mailing list
> >> poky@yoctoproject.org
> >> https://lists.yoctoproject.org/listinfo/poky
> >
>
>
>
> --
> "Thou shalt not follow the NULL pointer, for chaos and madness await
> thee at its end"
> _______________________________________________
> poky mailing list
> poky@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/poky
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [PATCH 10/11] bb.fetch2: add git unpack
2011-01-26 8:25 ` Yu Ke
@ 2011-01-26 15:01 ` Bruce Ashfield
0 siblings, 0 replies; 18+ messages in thread
From: Bruce Ashfield @ 2011-01-26 15:01 UTC (permalink / raw)
To: Yu Ke; +Cc: poky
On Wed, Jan 26, 2011 at 3:25 AM, Yu Ke <ke.yu@intel.com> wrote:
> On Jan 25, 23:43, Bruce Ashfield wrote:
>> On Tue, Jan 25, 2011 at 11:41 PM, Yu Ke <ke.yu@intel.com> wrote:
>> > "git clone" is my first try when cooking the patch, unfortunately I meet one issue here. the git clone target dir is ${WORKDIR}, while git requires the target dir to be empty, which is not true for ${WORKDIR}, so git clone will fail. so I fall back to cp approach.
>> >
>> > But I agree that git clone is better. So I'd like use this apprach: git clone to {WORKDIR}/git/, and modify the kernel-yocto.bbclass:do_kernel_checkout() to use the new layout.
>> > I will revise the patch accordingly for review.
>> > Regards
>> > Ke
>>
>> This may be problematic. The kernel checkout needs to be done in a very
>> specific way. After the clone, you must have all branches local (no origin/)
>> and in the 'linux' directory. There are other tools that depend on
>> this particular
>> layout.
>>
>> The most efficient way to do this has proven to be to do a bare clone
>> and then forcibly convert it to a non-bare clone. That gets you the
>> instantiation
>> and local branches in a single clone + minor fix ups to the directory.
>>
>> Just some things to consider, I'll wait to see the patches before commenting
>> further.
>>
>> Bruce
>
> Bruce, thanks for the comments. You are right. I realize that kernel checkout need to output a specific layout, so I only change the input layout of kernel checkout. more specificly, change the input dir from ${WORKDIR}/.git to ${WORKDIR}/git/.git. and the output layout is unchanged.
>
> please see if the revised patch works for you:
I haven't actually tried the patch, but just by having a look at it, it looks
fine. No harm in the directory shuffling at all, and it makes more sense
this way.
Cheers,
Bruce
>
> git unpack patch - http://git.pokylinux.org/cgit/cgit.cgi/poky-contrib/commit/?h=kyu3/git-unpack&id=e654e3d75550282c4d112f1050465cde850c93e9
>
> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
> index 08daa20..eadbf33 100644
> --- a/bitbake/lib/bb/fetch2/git.py
> +++ b/bitbake/lib/bb/fetch2/git.py
> @@ -194,6 +194,31 @@ class Git(Fetch):
> os.chdir(ud.clonedir)
> bb.utils.prunedir(codir)
>
> + def unpack(self, ud, destdir, d):
> + """ unpack the downloaded src to destdir"""
> +
> + subdir = ud.parm.get("subpath", "")
> + if subdir != "":
> + readpathspec = ":%s" % (subdir)
> + else:
> + readpathspec = ""
> +
> + destdir = os.path.join(destdir, "git/")
> + if os.path.exists(destdir):
> + bb.utils.prunedir(destdir)
> +
> + if 'fullclone' in ud.parm:
> + runfetchcmd("git clone -l -n %s %s" % (ud.clonedir, destdir), d)
> + if os.path.exists("%s/.git/refs/remotes/origin" % ud.clonedir):
> + runfetchcmd("cp -af %s/.git/refs/remotes/origin/* %s/.git/refs/remotes/origin/" %(ud.clonedir, destdir), d)
> + if os.path.exists("%s/.git/packed-refs" % ud.clonedir):
> + runfetchcmd("cp -af %s/.git/packed-refs %s/.git/" %(ud.clonedir, destdir), d)
> + else:
> + os.chdir(ud.clonedir)
> + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
> + runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
> + return True
> +
> def supports_srcrev(self):
> return True
>
> kernel checkout patch - http://git.pokylinux.org/cgit/cgit.cgi/poky-contrib/commit/?h=kyu3/git-unpack&id=b1916da1f1a8b7e3c4b50d540918aa72be704d9a
>
> diff --git a/meta/classes/kernel-yocto.bbclass b/meta/classes/kernel-yocto.bbclass
> index 8479b39..ca1fb15 100644
> --- a/meta/classes/kernel-yocto.bbclass
> +++ b/meta/classes/kernel-yocto.bbclass
> @@ -89,11 +89,11 @@ do_patch() {
> }
>
> do_kernel_checkout() {
> - if [ -d ${WORKDIR}/.git/refs/remotes/origin ]; then
> + if [ -d ${WORKDIR}/git/.git/refs/remotes/origin ]; then
> echo "Fixing up git directory for ${LINUX_KERNEL_TYPE}/${KMACHINE}"
> rm -rf ${S}
> mkdir ${S}
> - mv ${WORKDIR}/.git ${S}
> + mv ${WORKDIR}/git/.git ${S}
>
> if [ -e ${S}/.git/packed-refs ]; then
> cd ${S}
>
>
>>
>> >
>> >>
>> >> >
>> >> > The idea would be one central checkout with the references in and then
>> >> > the subcomponents would all just reference the one local clone saving
>> >> > diskspace and time.
>> >>
>> >> Absolutely. And if someone does something to the central checkout,
>> >> minor errors ensue, but that can be recovered as well.
>> >>
>> >> Cheers,
>> >>
>> >> Bruce
>> >>
>> >> >
>> >> > Cheers,
>> >> >
>> >> > Richard
>> >> >
>> >> >
>> >> >
>> >> > _______________________________________________
>> >> > poky mailing list
>> >> > poky@yoctoproject.org
>> >> > https://lists.yoctoproject.org/listinfo/poky
>> >> >
>> >>
>> >>
>> >>
>> >> --
>> >> "Thou shalt not follow the NULL pointer, for chaos and madness await
>> >> thee at its end"
>> >> _______________________________________________
>> >> poky mailing list
>> >> poky@yoctoproject.org
>> >> https://lists.yoctoproject.org/listinfo/poky
>> >
>>
>>
>>
>> --
>> "Thou shalt not follow the NULL pointer, for chaos and madness await
>> thee at its end"
>> _______________________________________________
>> poky mailing list
>> poky@yoctoproject.org
>> https://lists.yoctoproject.org/listinfo/poky
>
--
"Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end"
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH 11/11] git.py: remove the source tree tar ball
2011-01-18 17:48 [PATCH 00/11] [RFC] fetch overhaul - API reorg and git optimization Yu Ke
` (9 preceding siblings ...)
2011-01-18 17:48 ` [PATCH 10/11] bb.fetch2: add git unpack Yu Ke
@ 2011-01-18 17:48 ` Yu Ke
10 siblings, 0 replies; 18+ messages in thread
From: Yu Ke @ 2011-01-18 17:48 UTC (permalink / raw)
To: poky
we already create repo tar ball, so no need to create the source tree tar ball
Signed-off-by: Yu Ke <ke.yu@intel.com>
---
bitbake/lib/bb/fetch2/git.py | 62 +++---------------------------------------
1 files changed, 4 insertions(+), 58 deletions(-)
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index bb62a87..7bfe178 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -61,24 +61,14 @@ class Git(Fetch):
ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
+ ud.repochanged = False
def localpath(self, url, ud, d):
ud.tag = ud.revision
if not ud.tag or ud.tag == "master":
ud.tag = self.latest_revision(url, ud, d)
- subdir = ud.parm.get("subpath", "")
- if subdir != "":
- if subdir.endswith("/"):
- subdir = subdir[:-1]
- subdirpath = os.path.join(ud.path, subdir);
- else:
- subdirpath = ud.path;
-
- if 'fullclone' in ud.parm:
- ud.localfile = ud.mirrortarball
- else:
- ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
+ ud.localfile = ud.mirrortarball
if 'noclone' in ud.parm:
ud.localfile = None
@@ -91,8 +81,6 @@ class Git(Fetch):
return True
if 'noclone' in ud.parm:
return False
- if os.path.exists(ud.localpath):
- return False
if not self._contains_ref(ud.tag, d):
return True
return False
@@ -139,60 +127,18 @@ class Git(Fetch):
runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+ ud.repochanged = True
def build_mirror_data(self, url, ud, d):
# Generate a mirror tarball if needed
- coname = '%s' % (ud.tag)
- codir = os.path.join(ud.clonedir, coname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
os.chdir(ud.clonedir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
- if mirror_tarballs != "0" or 'fullclone' in ud.parm:
+ if mirror_tarballs != "0" or 'fullclone' in ud.parm or ud.repochanged:
logger.info("Creating tarball of git repository")
runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
- if 'fullclone' in ud.parm:
- return
-
- if os.path.exists(codir):
- bb.utils.prunedir(codir)
-
- subdir = ud.parm.get("subpath", "")
- if subdir != "":
- if subdir.endswith("/"):
- subdirbase = os.path.basename(subdir[:-1])
- else:
- subdirbase = os.path.basename(subdir)
- else:
- subdirbase = ""
-
- if subdir != "":
- readpathspec = ":%s" % (subdir)
- codir = os.path.join(codir, "git")
- coprefix = os.path.join(codir, subdirbase, "")
- else:
- readpathspec = ""
- coprefix = os.path.join(codir, "git", "")
-
- scmdata = ud.parm.get("scmdata", "")
- if scmdata == "keep":
- runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
- os.chdir(coprefix)
- runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
- else:
- bb.mkdirhier(codir)
- os.chdir(ud.clonedir)
- runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
- runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
-
- os.chdir(codir)
- logger.info("Creating tarball of git checkout")
- runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
-
- os.chdir(ud.clonedir)
- bb.utils.prunedir(codir)
-
def unpack(self, ud, destdir, d):
""" unpack the downloaded src to destdir"""
subdir = ud.parm.get("subpath", "")
--
1.7.0.4
^ permalink raw reply related [flat|nested] 18+ messages in thread