[PATCH 1/2] fetch2/repo: Implement AUTOREV for repo fetcher

* [PATCH 1/2] fetch2/repo: Implement AUTOREV for repo fetcher
@ 2021-11-05 13:30 Jasper Orschulko
  2021-11-05 13:30 ` [PATCH 2/2] fetch2: Fix race condition in latest_revision Jasper Orschulko
  0 siblings, 1 reply; 6+ messages in thread
From: Jasper Orschulko @ 2021-11-05 13:30 UTC (permalink / raw)
  To: bitbake-devel
  Cc: martin, Daniel.Baumgart, openembedded-core, Jasper Orschulko

From: Martin Koppehel <martin@mko.dev>

- Implement AUTOINC and submodule support for REPO provider
- Implement full srcrev support
- Add comments and fixup empty DL_DIR initialization
- Distinguish between artificial and plain rev
- Comments/documentation

The previous implementation of the repo fetcher could not handle updates
to the repo manifest file, nor deal with dynamic refspecs within this
manifest.

This patch fixes these shortcomings as follows:
During the recipe parsing phase, the repository containing the repo
manifest is cloned. This is done, as we need to parse the XML file
contained within, in order to discover all involved git repositories. A
combined hash is then calculated from the manifest repo, as well as any
git repo specified in the manifest. This hash is used for determining the
necessity of an update.

Additionally, the recipe will throw an error if the repo source is set to
a fixed revision but one or more repositories within the manifest
reference a dynamic refspec. This is done to ensure the reproducibility
of a version-pinned recipe.

Signed-off-by: Jasper Orschulko <Jasper.Orschulko@iris-sensing.com>
---
 lib/bb/fetch2/repo.py | 226 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 198 insertions(+), 28 deletions(-)

diff --git a/lib/bb/fetch2/repo.py b/lib/bb/fetch2/repo.py
index fa4cb814..22ee5b80 100644
--- a/lib/bb/fetch2/repo.py
+++ b/lib/bb/fetch2/repo.py
@@ -3,6 +3,7 @@ BitBake "Fetch" repo (git) implementation
 
 """
 
+# Copyright (C) 2021 Martin Koppehel <martin@mko.dev>, iris-GmbH infrared & intelligent sensors
 # Copyright (C) 2009 Tom Rini <trini@embeddedalley.com>
 #
 # Based on git.py which is:
@@ -13,10 +14,13 @@ BitBake "Fetch" repo (git) implementation
 
 import os
 import bb
+import hashlib
+import xml.etree.ElementTree as ET
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 
+
 class Repo(FetchMethod):
     """Class to fetch a module or modules from repo (git) repositories"""
     def supports(self, ud, d):
@@ -27,46 +31,74 @@ class Repo(FetchMethod):
 
     def urldata_init(self, ud, d):
         """
-        We don"t care about the git rev of the manifests repository, but
-        we do care about the manifest to use.  The default is "default".
-        We also care about the branch or tag to be used.  The default is
-        "master".
+        We do care about the rev of the manifests repository, as well as the
+        manifest file. However, when SRCREV=AUTOINC, then we use the specified
+        branch in SRC_URI, with a fallback to master.
+        use sm=fetch to fetch possibly referenced submodules in repositories.
         """
 
         ud.basecmd = d.getVar("FETCHCMD_repo") or "/usr/bin/env repo"
+        ud.gitcmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
 
         ud.proto = ud.parm.get('protocol', 'git')
         ud.branch = ud.parm.get('branch', 'master')
+
+        ud.submodules = ud.parm.get('sm', 'fetch')
         ud.manifest = ud.parm.get('manifest', 'default.xml')
         if not ud.manifest.endswith('.xml'):
             ud.manifest += '.xml'
 
-        ud.localfile = d.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch))
+        repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo")
+        gitsrcname = "%s%s.%s" % (ud.host, ud.path.replace("/", "."), ud.manifest)
+        ud.codir = os.path.join(repodir, d.getVar("BPN"), gitsrcname)
+
+        if ud.user:
+            ud.username = ud.user + "@"
+        else:
+            ud.username = ""
+        ud.remoteRepo = "%s://%s%s%s" % (ud.proto, ud.username, ud.host, ud.path)
+
+        ud.repodir = os.path.join(ud.codir, "repo")
+        # a temporary directory to compute _latest_revision
+        ud.tempdir = os.path.join(ud.codir, "temp")
+        ud.stampfile = os.path.join(ud.codir, "__hash.txt")
+        ud.setup_revisions(d)
+
+        # ud.localfile is used to fill localpath, where the downloaded tarball is stored.
+        # in our case, we want something like repo_$GIT_URL_$MANIFEST_$SRCREV
+        # todo: do we want the packagename?
+        ud.localfile = "repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, d.getVar("SRCREV"))
+
+    def need_update(self, ud, d):
+        if d.getVar("SRCREV") == "AUTOINC":
+            return True
+        return os.path.exists(ud.localfile)
 
     def download(self, ud, d):
         """Fetch url"""
 
-        if os.access(os.path.join(d.getVar("DL_DIR"), ud.localfile), os.R_OK):
-            logger.debug("%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
-            return
-
-        repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo")
-        gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
-        codir = os.path.join(repodir, gitsrcname, ud.manifest)
+        bb.utils.mkdirhier(ud.repodir)
 
-        if ud.user:
-            username = ud.user + "@"
+        # we want to run a repo init *always* in case the branch or manifest name changes.
+        # if not os.path.exists(os.path.join(repodir, ".repo")):
+        if ud.submodules == "fetch":
+            submodules = "--fetch-submodules"
         else:
-            username = ""
+            submodules = ""
+
+        # fixup the revision -> when it starts with underscore, it's an artificial one
+        # therefore we then use the specified remote branch used to generate
+        # the artificial revision in _latest_revision
+        realRevision = ud.revision
+        if ud.revision.startswith("_"):
+            realRevision = ud.branch
 
-        repodir = os.path.join(codir, "repo")
-        bb.utils.mkdirhier(repodir)
-        if not os.path.exists(os.path.join(repodir, ".repo")):
-            bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
-            runfetchcmd("%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
+        # always run repo init, because we might want to switch branch or manifests.
+        bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s" % (ud.basecmd, ud.manifest, realRevision, ud.remoteRepo), ud.url)
+        runfetchcmd("%s init -m %s -b %s -u %s" % (ud.basecmd, ud.manifest, realRevision, ud.remoteRepo), d, workdir=ud.repodir)
 
-        bb.fetch2.check_network_access(d, "%s sync %s" % (ud.basecmd, ud.url), ud.url)
-        runfetchcmd("%s sync" % ud.basecmd, d, workdir=repodir)
+        bb.fetch2.check_network_access(d, "%s sync %s %s" % (ud.basecmd, submodules, ud.url), ud.url)
+        runfetchcmd("%s sync %s" % (ud.basecmd, submodules), d, workdir=ud.repodir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":
@@ -75,13 +107,151 @@ class Repo(FetchMethod):
             tar_flags = "--exclude='.repo' --exclude='.git'"
 
         # Create a cache
-        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d, workdir=codir)
+        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*")), d, workdir=ud.codir)
 
     def supports_srcrev(self):
-        return False
+        return True
+
+    def clean(self, ud, d):
+        """ clean the repo directory """
+
+        to_remove = [ud.localpath, ud.repodir, ud.tempdir, ud.stampfile]
+        # The localpath is a symlink to clonedir when it is cloned from a
+        # mirror, so remove both of them.
+        if os.path.islink(ud.localpath):
+            clonedir = os.path.realpath(ud.localpath)
+            to_remove.append(clonedir)
+
+        for r in to_remove:
+            if os.path.exists(r):
+                bb.utils.remove(r, True)
+
+    # this is taken from the git fetcher
+    def _lsremote(self, ud, d, search, repo):
+        """
+        Run git ls-remote with the specified search string
+        """
+        # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR,
+        # and WORKDIR is in PATH (as a result of RSS), our call to
+        # runfetchcmd() exports PATH so this function will get called again (!)
+        # In this scenario the return call of the function isn't actually
+        # important - WORKDIR isn't needed in PATH to call git ls-remote
+        # anyway.
+        if d.getVar('_BB_REPO_IN_LSREMOTE', False):
+            return ''
+        d.setVar('_BB_REPO_IN_LSREMOTE', '1')
+        try:
+            cmd = "%s ls-remote \"%s\" %s" % \
+                (ud.gitcmd, repo, search)
+            if ud.proto.lower() != 'file':
+                bb.fetch2.check_network_access(d, cmd, ud.remoteRepo)
+            output = runfetchcmd(cmd, d, True)
+            if not output:
+                raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
+        finally:
+            d.delVar('_BB_REPO_IN_LSREMOTE')
+        return output
+
+    def _checkBranch(self, ud, d, name, repo):
+        output = self._lsremote(ud, d, name, repo)
+        searchstring = "refs/heads/%s" % name
+        found = False
+        for line in output.strip().split('\n'):
+            sha1, ref = line.split()
+            if searchstring == ref:
+                logger.debug(1, "resolved %s@%s to %s", repo, name, sha1)
+                return sha1
+                break
+
+        if not found:
+            raise bb.fetch2.FetchError("Could not determine remote ref!")
+
+    def _build_revision(self, ud, d, name):
+        return ud.revisions[name]
+
+    def _revision_key(self, ud, d, name):
+        return "%s-%s" % (d.getVar("BPN"), name)
+
+    def _latest_revision(self, ud, d, name):
+        """
+        Computes an artificial revision from the manifest repository and all
+        referenced repositories and their remote revisions.
+        name is ignored because we can only have a single branch/name
+        """
+        if d.getVar('_BB_REPO_IN_LATEST_REV', False):
+            return ''
+        d.setVar('_BB_REPO_IN_LATEST_REV', '1')
+
+        # we use a sha256 to mixup all the hashes we have
+        hashCalc = hashlib.sha256()
+
+        # first, add the hash of the repo itself
+        sha1 = self._checkBranch(ud, d, ud.branch, ud.remoteRepo)
+        hashUpdate = bytes(sha1, 'utf-8')
+        hashCalc.update(hashUpdate)
+
+        # Parse the repo XML files, remove things
+        try:
+            # create/cleanup temporary dir where to clone the repo-manifest URL
+            if os.path.isdir(ud.tempdir):
+                bb.utils.prunedir(ud.tempdir)
+            bb.utils.mkdirhier(ud.tempdir)
+
+            # clone the manifest repo to the temporary dir we just set up
+            bb.fetch2.check_network_access(d, "%s clone -b %s --depth 1 --single-branch %s ." % (ud.gitcmd, ud.branch, ud.remoteRepo), ud.url)
+            runfetchcmd("%s clone -b %s --depth 1 --single-branch %s %s" % (ud.gitcmd, ud.branch, ud.remoteRepo, ud.tempdir), d, workdir=ud.tempdir)
+
+            # parse the specified XML manifest
+            xml = ET.parse("%s/%s" % (ud.tempdir, ud.manifest))
+
+            # repo manifest *may* specify a <default> element, specifying fallback remotes and revisions
+            defaultObject = xml.find('default')
+
+            # parse all remotes and their corresponding default revisions
+            remotes = {}
+            remoteRevisions = {}
+            for remote in xml.findall('remote'):
+                remotes[remote.get('name')] = remote.get('fetch')
+                remoteRevisions[remote.get('name')] = remote.get('revision')
+
+            # iterate through the <project> elements, resolving the correct remote
+            # and revision
+            for project in xml.findall('project'):
+
+                # resolve the remote of the project
+                # when no remote is specified in the project take the one from <default>
+                # when both aren't specified, throw
+                remoteName = project.get('remote')
+                if remoteName is None and defaultObject is not None:
+                    remoteName = defaultObject.get('remote')
+                if remoteName is None:
+                    raise bb.fetch2.FetchError("repo manifest specifies no remote for %s" % project.get('name'))
+
+                # resolve the remoteName to a git remote URL and optionally
+                # the revision if it was specified in <remote>
+                if remotes[remoteName] is not None:
+                    remoteRev = remoteRevisions[remoteName]
+                    remote = remotes[remoteName]
+
+                # use revision in the project, when not specified use the one from <remote>
+                # when that is not specified use <default> and when we not have anything specified
+                # throw an exception
+                revision = project.get('revision') or remoteRev
+                if revision is None and defaultObject is not None:
+                    revision = defaultObject.get('revision')
+                if revision is None:
+                    raise bb.fetch2.FetchError("repo manifest specifies no revision for %s" % project.get('name'))
+
+                # perform an ls-remote on the branch, update the checksum with the commit hash
+                gitRemotePath = "%s/%s" % (remote, project.get('name'))
+
+                sha1 = self._checkBranch(ud, d, revision, gitRemotePath)
+                hashUpdate = bytes(sha1, 'utf-8')
+                hashCalc.update(hashUpdate)
+
+        finally:
+            d.delVar('_BB_REPO_IN_LATEST_REV')
+        digest = "_" + hashCalc.hexdigest()
+        return digest
 
-    def _build_revision(self, ud, d):
-        return ud.manifest
 
-    def _want_sortable_revision(self, ud, d):
-        return False
-- 
2.33.1



^ permalink raw reply related	[flat|nested] 6+ messages in thread