[review-request][PATCH] toaster: Import external layers outside poky from toasterconf.json

* [review-request][PATCH] toaster: Import external layers outside poky from toasterconf.json
@ 2015-08-25  9:53 Sujith H
  2015-08-31 11:17 ` sujith h
  0 siblings, 1 reply; 4+ messages in thread
From: Sujith H @ 2015-08-25  9:53 UTC (permalink / raw)
  To: toaster

This patch helps to import external layers from toasterconf.json
which are already checked out. The branch of the checkedout layers
is HEAD. Hence no checkout happens when toaster is using the configuration.
This essentially speeds up toaster, while using locally checked out layers.

Signed-off-by: Sujith H <sujith.h@gmail.com>
Signed-off-by: Sujith Haridasan <Sujith_Haridasan@mentor.com>
---
 .../toaster/bldcontrol/localhostbecontroller.py    | 112 +++++++++++++--------
 .../bldcontrol/management/commands/loadconf.py     |  14 ++-
 2 files changed, 79 insertions(+), 47 deletions(-)

diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index 231a7d3..837145b 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -32,6 +32,8 @@ from toastermain import settings
 
 from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException
 
+from orm.models import Layer_Version
+
 import logging
 logger = logging.getLogger("toaster")
 
@@ -196,6 +198,21 @@ class LocalhostBEController(BuildEnvironmentController):
         #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
         return local_checkout_path
 
+    def getHeadLayers(self, gitrepos, layers):
+        """ HEAD layers are special cases. They don't need to be cloned. """
+
+        layerlist = []
+        headlayers = [commit for giturl, commit in gitrepos.keys() if commit == "HEAD"]
+        if set(headlayers) == set(["HEAD"]):
+           # Populate layerlist if all of them are HEAD
+            for layer_version in Layer_Version.objects.all().filter(local_path__startswith = "/"):
+                if layer_version.commit == "HEAD":
+                    if layer_version.local_path not in layerlist:
+                        layerlist.append(layer_version.local_path)
+                        if "poky" in layer_version.local_path and os.path.exists(os.path.join(os.path.dirname(layer_version.local_path), "oe-init-build-env")):
+                            self.pokydirname = os.path.dirname(layer_version.local_path)
+        return layerlist
+
 
     def setLayers(self, bitbakes, layers):
         """ a word of attention: by convention, the first layer for any build will be poky! """
@@ -242,51 +259,58 @@ class LocalhostBEController(BuildEnvironmentController):
 
         layerlist = []
 
+        # Verify HEAD layers
+
+        layerlist = self.getHeadLayers(gitrepos,layers)
+
+        # If the layerlist doesn't have already cloned layers then checkout repositories
+        if len(layerlist) == 0:
+
+           # 3. checkout the repositories
+           for giturl, commit in gitrepos.keys():
+               localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
+               logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
+
+               # make sure our directory is a git repository
+               if os.path.exists(localdirname):
+                   localremotes = self._shellcmd("git remote -v", localdirname)
+                   if not giturl in localremotes:
+                       raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
+               else:
+                   if giturl in cached_layers:
+                       logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
+                       self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
+                       self._shellcmd("git remote remove origin", localdirname)
+                       self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
+                   else:
+                       logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname))
+                       self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
+
+               # branch magic name "HEAD" will inhibit checkout
+               if commit != "HEAD":
+                   logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
+                   self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname)
+
+               # take the localdirname as poky dir if we can find the oe-init-build-env
+               if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
+                   logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
+                   self.pokydirname = localdirname
+
+                   # make sure we have a working bitbake
+                   if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
+                       logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
+                       self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
+
+               # verify our repositories
+               for name, dirpath in gitrepos[(giturl, commit)]:
+                   localdirpath = os.path.join(localdirname, dirpath)
+                   logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
+                   if not os.path.exists(localdirpath):
+                       raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
+
+                   if name != "bitbake":
+                       layerlist.append(localdirpath.rstrip("/"))
 
-        # 3. checkout the repositories
-        for giturl, commit in gitrepos.keys():
-            localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
-            logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
-
-            # make sure our directory is a git repository
-            if os.path.exists(localdirname):
-                localremotes = self._shellcmd("git remote -v", localdirname)
-                if not giturl in localremotes:
-                    raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
-            else:
-                if giturl in cached_layers:
-                    logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
-                    self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
-                    self._shellcmd("git remote remove origin", localdirname)
-                    self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
-                else:
-                    logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname))
-                    self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
-
-            # branch magic name "HEAD" will inhibit checkout
-            if commit != "HEAD":
-                logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
-                self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname)
-
-            # take the localdirname as poky dir if we can find the oe-init-build-env
-            if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
-                logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
-                self.pokydirname = localdirname
-
-                # make sure we have a working bitbake
-                if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
-                    logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
-                    self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
-
-            # verify our repositories
-            for name, dirpath in gitrepos[(giturl, commit)]:
-                localdirpath = os.path.join(localdirname, dirpath)
-                logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
-                if not os.path.exists(localdirpath):
-                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
-
-                if name != "bitbake":
-                    layerlist.append(localdirpath.rstrip("/"))
 
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
index 5022b59..21bc380 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
@@ -45,12 +45,14 @@ class Command(BaseCommand):
         for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']:
             assert i in data
 
-        def _read_git_url_from_local_repository(address):
+        def _read_git_url_from_local_repository(address, path = None):
             url = None
+            if not path:
+                path = os.path.dirname(filepath)
             # we detect the remote name at runtime
             import subprocess
             (remote, remote_name) = address.split(":", 1)
-            cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE)
+            cmd = subprocess.Popen("git remote -v", shell=True, cwd = path, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
             (out,err) = cmd.communicate()
             if cmd.returncode != 0:
                 logging.warning("Error while importing layer vcs_url: git error: %s" % err)
@@ -121,7 +123,11 @@ class Command(BaseCommand):
                         logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path)
 
                     if layerinfo['vcs_url'].startswith("remote:"):
-                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'])
+                        if not layerinfo['local_path'].startswith("/"):
+                           path = None
+                        else:
+                           path = layerinfo['local_path']
+                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'], path)
                         if lo.vcs_url is None:
                             logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name'])
 
@@ -138,6 +144,8 @@ class Command(BaseCommand):
                                 commit = branch.name,
                                 layer = lo)
                         lvo.dirpath = layerinfo['dirpath']
+                        if len(layerinfo['local_path']) > 1 and layerinfo['local_path'].startswith("/") and branch.name == "HEAD":
+                            lvo.local_path = layerinfo['local_path']
                         lvo.save()
         # set releases
         for ri in data['releases']:
-- 
1.9.1



^ permalink raw reply related	[flat|nested] 4+ messages in thread