[PATCH] Add support for remote layering.

[Jeremy Puhlman <jpuhlman@mvista.com>]

The bulk of this patch is based on Chris Larson's collection.inc,
but refactored for use inside bitbake, and for layers.

Adds two new configuration options:

LAYER_UNPACKDIR - directory to unpack downloaded layers for use in configuration.
LAYER_REMOTE_STASH - location to store downloaded layers for use in configuration.

Layers can be specified in the current manner:
BBLAYERS = " \
   /path/to/layer/one \
   /path/to/layer/two \
"

Or you can specify uris:

BBLAYERS = " \
	http://url.to.my.tarball/tarball.tar.gz \
	git://gitserver.com/git/repo;proto=http \
        /path/to/layer/three \
        file:///path/to/layer/four \
"

Currently there is a single layer option, that can be added to a uri, layerBase=<subpath>.
This option would be used to specify if a layer starts somewhere other then the base of the
tarball/scm repository. For example if you wanted to add oe-core you would do:

BBLAYERS  = "git://git.openembedded.org/openembedded-core;protocol=git;tag=master;layerBase=meta"

Currently for fetch the following uris should work  : 'git','http','https','ftp','file'
Currently for fetch2 the following uris should work : 'git','http','https','ftp','file','svn'

There are certainly some ugly bits in here, and there are certainly lots of places to improve,
but this gets a good chunk of the base support in.

Signed-off-by: Jeremy Puhlman <jpuhlman@mvista.com>
---
 lib/bb/cooker.py       |    3 +-
 lib/bb/fetch/layer.py  |   65 ++++++++++++++++
 lib/bb/fetch2/layer.py |   76 ++++++++++++++++++
 lib/bb/remotelayer.py  |  197 ++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 340 insertions(+), 1 deletions(-)
 create mode 100644 lib/bb/fetch/layer.py
 create mode 100644 lib/bb/fetch2/layer.py
 create mode 100644 lib/bb/remotelayer.py

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index d4415d3..0099a65 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -575,6 +575,7 @@ class BBCooker:
             path, _ = os.path.split(path)
 
     def parseConfigurationFiles(self, files):
+        from bb.remotelayer import RemoteLayers
         data = self.configuration.data
         bb.parse.init_parser(data)
         for f in files:
@@ -586,7 +587,7 @@ class BBCooker:
             data = _parse(layerconf, data)
 
             layers = (bb.data.getVar('BBLAYERS', data, True) or "").split()
-
+            layers = RemoteLayers(layers,data,parselog).getLayers()
             data = bb.data.createCopy(data)
             for layer in layers:
                 parselog.debug(2, "Adding layer %s", layer)
diff --git a/lib/bb/fetch/layer.py b/lib/bb/fetch/layer.py
new file mode 100644
index 0000000..3b0d319
--- /dev/null
+++ b/lib/bb/fetch/layer.py
@@ -0,0 +1,65 @@
+"""
+BitBake 'layer' fetch implementation.
+
+"""
+
+# Copyright (C) 2011 Jeremy Puhlman 
+#
+#   Classes for obtaining upstream sources for the
+#   BitBake build tools.
+#   Copyright (C) 2003, 2004  Chris Larson
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+class LayerFetch:
+
+    def __init__(self, url, pathurl, logger, ld):
+	self.parselog = logger
+	self.tarballpath = self._fetch(url,pathurl,ld)
+
+    def supported(self, ud):
+	return ud.type in ['git','http','https','ftp','file']
+
+    def getTarballPath(self):
+        return self.tarballpath
+
+    def _fetch(self, url, pathurl, ld):
+        urldata = {}
+        try:
+            urldata[url] = bb.fetch.FetchData(url,ld)
+        except bb.fetch.NoMethodError: 
+            self.parselog.debug(1, "No method for %s" % url)
+            return
+        ud = urldata[url]
+        filename=os.path.basename(ud.path)
+
+	if not self.supported(ud):
+           bb.fatal("Layer fetching does not support %s uris" % ud.type)
+        if not ud.setup:
+           ud.setup_localpath(ld)
+        m = ud.method
+        tarballpath = bb.fetch.localpath(url, ld)
+        if os.path.exists(tarballpath) :
+           return tarballpath
+        try: 
+           m.go(url,ud,ld)
+        except (bb.fetch.MissingParameterError,
+           bb.fetch.FetchError,
+           bb.fetch.MD5SumError):
+
+           import sys
+           (type, value, traceback) = sys.exc_info()
+           self.parselog.debug(1, "layer fetch failure: %s" % value)
+           return
+        return tarballpath
diff --git a/lib/bb/fetch2/layer.py b/lib/bb/fetch2/layer.py
new file mode 100644
index 0000000..09de8a0
--- /dev/null
+++ b/lib/bb/fetch2/layer.py
@@ -0,0 +1,76 @@
+"""
+BitBake 'layer' fetch2 implementation.
+
+"""
+
+# Copyright (C) 2011 Jeremy Puhlman 
+#
+#   Classes for obtaining upstream sources for the
+#   BitBake build tools.
+#   Copyright (C) 2003, 2004  Chris Larson
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+class LayerFetch:
+
+    def __init__(self, url, pathurl, logger, ld):
+	self.parselog = logger
+	self.tarballpath = self._fetch(url,pathurl,ld)
+
+    def supported(self, ud):
+	return ud.type in ['git','http','https','ftp', 'svn', 'file']
+
+    def getTarballPath(self):
+        return self.tarballpath
+
+    def _fetch(self, url, pathurl, ld):
+        urldata = {}
+        layer_unpackdir =  ld.getVar("LAYER_UNPACKDIR", True)		
+        try:
+            urldata[url] = bb.fetch.FetchData(url,ld)
+        except bb.fetch.NoMethodError: 
+            self.parselog.debug(1, "No method for %s" % url)
+            return
+        ud = urldata[url]
+        filename=os.path.basename(ud.path)
+
+	if not self.supported(ud):
+           bb.fatal("Layer fetching does not support %s uris" % ud.type)
+        if not ud.setup:
+           ud.setup_localpath(ld)
+        m = ud.method
+	if ud.type not in ['git']:
+              tarballpath = bb.fetch.localpath(url, ld)
+              if os.path.exists(bb.fetch.localpath(url, ld)) :
+                 return tarballpath		
+        try: 
+           m.download(url,ud,ld)
+        except (bb.fetch.MissingParameterError,
+           bb.fetch.FetchError,
+           bb.fetch.MD5SumError):
+
+           import sys
+           (type, value, traceback) = sys.exc_info()
+           self.parselog.debug(1, "layer fetch failure: %s" % value)
+           return
+	
+	if ud.type in ['git']:
+           unpackedscm = "%s/%s" % (layer_unpackdir, ud.mirrortarball)
+           if m.need_update(url,ud,ld):
+              m.build_mirror_data(url, ud,ld)
+              m.unpack(ud, unpackedscm ,ld)
+           tarballpath = unpackedscm + "/git" 
+
+        return tarballpath
+
diff --git a/lib/bb/remotelayer.py b/lib/bb/remotelayer.py
new file mode 100644
index 0000000..2a61e71
--- /dev/null
+++ b/lib/bb/remotelayer.py
@@ -0,0 +1,197 @@
+"""
+BitBake 'remote layer' handling.
+
+"""
+
+# Copyright (C) 2011 Jeremy Puhlman 
+#
+#   Classes for obtaining upstream sources for the
+#   BitBake build tools.
+#   Copyright (C) 2003, 2004  Chris Larson
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+class RemoteLayers:
+    """
+    Get remote layers.
+    """
+
+    def __init__(self, layers, data, logger):
+        self.parselog = logger
+        self.layers = self._getRemoteLayers(layers, data)
+
+    def getLayers(self):
+        return self.layers    
+
+    """
+    Ugly. We have not loaded bitbake.conf yet, and need the values set for fetching to work
+    Could load preliminary bitbake.conf, but this works.
+    """
+    def _setFetchValues(self,data):
+	"""Need these for any fetching to work"""
+        def setifunset(d,k,v): 
+           if d.getVar(k,0) is None:
+                d.setVar(k,v)
+
+        setifunset(data, "TMPDIR", "${TOPDIR}/tmp")
+        setifunset(data, "LAYER_REMOTE_STASH", "${TMPDIR}/layers-remote")
+        setifunset(data, "LAYER_UNPACKDIR", "${TMPDIR}/layers")
+
+        ld = data.createCopy()
+        """Provide settings for fetchers.""" 
+        setifunset(ld, "CACHE", "${TMPDIR}/cache")
+        setifunset(ld, "OVERRIDES", "")
+        setifunset(ld, "CVSDIR", "${DL_DIR}/cvs")
+        setifunset(ld, "GITDIR", "${DL_DIR}/git")
+        setifunset(ld, "SVNDIR", "${DL_DIR}/svn")
+        setifunset(ld, "FETCHCOMMAND", "")
+        setifunset(ld, "FETCHCOMMAND_cvs", "/usr/bin/env cvs -d${CVSROOT} co ${CVSCOOPTS} ${CVSMODULE}")
+        setifunset(ld, "FETCHCOMMAND_svn", "/usr/bin/env svn co ${SVNCOOPTS} ${SVNROOT} ${SVNMODULE}")
+        setifunset(ld, "FETCHCOMMAND_wget", "/usr/bin/env wget -t 5 -q --passive-ftp -P ${DL_DIR} ${URI}")
+        setifunset(ld, "FETCHCMD_cvs", "/usr/bin/env cvs ")
+        setifunset(ld, "FETCHCMD_svn", "/usr/bin/env svn ")
+        setifunset(ld, "FETCHCMD_bzr", "/usr/bin/env bzr ")
+        setifunset(ld, "FETCHCMD_hg", "/usr/bin/env hg ")
+        setifunset(ld, "FETCHCMD_wget", "/usr/bin/env wget -t 5 -q")
+        setifunset(ld, "CHECKCOMMAND_wget", "/usr/bin/env wget --spider -t 5 --passive-ftp -P ${DL_DIR} '${URI}'")
+        setifunset(ld, "UPDATECOMMAND", "")
+        setifunset(ld, "UPDATECOMMAND_cvs", "/usr/bin/env cvs -d${CVSROOT} update ${CVSCOOPTS}")
+        setifunset(ld, "UPDATECOMMAND_svn", "/usr/bin/env svn update ${SVNCOOPTS}")
+        setifunset(ld, "BB_GENERATE_MIRROR_TARBALLS", "1")
+
+        localdir=data.getVar("LAYER_REMOTE_STASH", 1)
+
+        if not os.path.isdir(localdir):
+            bb.utils.mkdirhier(localdir)
+        ld.setVar("DL_DIR", localdir)
+        ld.delVar("MIRRORS")
+        ld.delVar("PREMIRRORS")
+        ld.delVar("SRC_TARBALL_STASH")
+	return ld 
+    
+    """
+    Unpack remote layer. Code largely pulled from collections.inc
+    """
+    def _layerUnpack(self, layer, data):
+        """ Unpack a layer archive and return the path to it. """
+        from hashlib import md5
+
+        handlers = {
+            ("tar"): "tar x --no-same-owner -f %s",
+            ("tar.gz", "tgz", "tar.Z"): "tar xz --no-same-owner -f %s",
+            ("tar.bz2", "tbz", "tbz2"): "tar xj --no-same-owner -f %s",
+            ("zip", "jar"): "unzip -q -o %s",
+        }
+        basename = os.path.basename(layer)
+        try:
+            cmd, name = ((cmd, basename[:-len(e)-1]) for (exts, cmd) in handlers.iteritems()
+                         for e in exts
+                         if basename.endswith(e)).next()
+        except StopIteration:
+            bb.fatal("No method available to unpack %s (unsupported file type?)" % layer)
+        else:
+            outpath = os.path.join(data.getVar("LAYER_UNPACKDIR", 1), name)
+            cmd = "cd %s && PATH=\"%s\" %s" % (outpath, data.getVar("PATH", 1), cmd)
+
+        try:
+            layerdata = open(layer, "r").read()
+        except IOError:
+            bb.fatal("Unable to open %s to calculate md5 sum" % layer)
+
+        md5obj = md5()
+        md5obj.update(layerdata)
+        md5sum = md5obj.hexdigest()
+        md5file = os.path.join(outpath, "md5")
+        if os.path.exists(md5file):
+            try:
+                oldmd5sum = open(md5file).read()
+            except IOError:
+                pass
+            else:
+                if oldmd5sum == md5sum:
+                    self.parselog.debug(1, "Using existing %s for layer '%s'" % (outpath, name))
+                    return outpath, name
+
+            self.parselog.plain("Removing old unpacked layer at %s" % outpath)
+        
+        if not os.path.isdir(outpath):
+            os.makedirs(outpath)
+
+        self.parselog.debug(1,"Unpacking %s to %s/" % (layer, outpath))
+        ret = os.system(cmd % layer)
+        if ret != 0:
+            bb.fatal("Unable to unpack %s" % layer)
+        md5out = open(md5file, "w")
+        md5out.write(md5sum)
+        md5out.close()
+        return outpath, name
+
+    """
+    Add method for grab layer specific data from uri.
+    """
+    def _getLayerSettings(self, urlinfo):
+        layersettings={}
+        layersettingslist = ["layerBase"]
+        for setting in urlinfo.path.split(";") + urlinfo.params.split(";"):
+            for layersetting in layersettingslist:
+                if setting.startswith("%s=" % layersetting):
+                    layersettings[layersetting] = setting.split("=")[1]
+        return [layersettings]
+
+    """
+    Main hooks for grabing remote layers
+    """
+    def _getRemoteLayers(self, layers, data):
+        from itertools import izip, chain
+        from glob import glob
+        from urlparse import urlparse, urlunparse
+	from bb.fetch.layer import LayerFetch
+        if not layers:
+            return []
+    
+        globbed = []
+        layersettings = []
+        localpath = ""
+	ld = self._setFetchValues(data)
+        for path in layers:
+            pathurl = urlparse(path)
+            layersettings += self._getLayerSettings(pathurl)
+            if not pathurl[0]:
+                localpath = glob(os.path.normpath(path))
+            else:
+                tarballpath = LayerFetch(path,pathurl,self.parselog, ld).getTarballPath()
+                if tarballpath:
+                    localpath=[tarballpath]
+            if not localpath:
+                self.parselog.plain("No matches in filesystem for %s in BBLAYERS" % path)
+            globbed += localpath
+        layers = globbed
+        layers_unpacked = ""
+        for (layer, settings) in izip(layers, layersettings):
+            origpath = layer
+            basepath=""
+            if "layerBase" in settings:
+               basepath = settings["layerBase"]
+            if not os.path.isdir(layer):
+                unpacked, name = self._layerUnpack(layer, ld)
+                if unpacked:
+                    layer = unpacked
+                    for dir in glob("%s/*/" % layer):
+                        layerdir = dir + "/" +  basepath
+                        layers_unpacked += layerdir + " "
+            else:
+                layers_unpacked += layer + "/" + basepath + " "
+        return layers_unpacked.encode('ascii').split()
+
+
-- 
1.7.3.2