From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-it1-f196.google.com (mail-it1-f196.google.com [209.85.166.196]) by mail.openembedded.org (Postfix) with ESMTP id 560C77C686 for ; Tue, 12 Mar 2019 22:42:44 +0000 (UTC) Received: by mail-it1-f196.google.com with SMTP id k193so7220346ita.3 for ; Tue, 12 Mar 2019 15:42:45 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=konsulko.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=lVbneraWETzCQWHzRYLNnG0yRqSWi91seYb3nPkMqzw=; b=sy1eAybFOWDf8Bh+Z5ytq6SUwQmTgjkifv2g7HzGdZnWE+aAIPeX5jEYVpMsTzdwsr FK+meJQk5gKyWBXogHZRKlvvHnERR3t9hy+dbg2T898SpoovT8lxbVIM5cqIprzE1lfn 7szl10Bvn1CzIcDGxh+WDkrgLXfMV+79i54hU= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=lVbneraWETzCQWHzRYLNnG0yRqSWi91seYb3nPkMqzw=; b=drWizqMu1/t71Sd5kb1T9NEIKpe3NID4U2+xBCyXY/bbs81jDrFoqKm5pETQwTr6S5 r1IDj5dxNHRkANDTnIaaHZPLRvO1yW9R37qxJTggcN3M6jpdyRFu5A46uLF7nZIIf1v9 nuH3lVHDZmsJTQucKI0T8ph3w2SOxBtdy6Pok+ikEINZdgDUgFv+p3mQg5kRx3wZNgC6 pxtBfx4FmZeIEiJixt7u/QWq7MJpraI1bRkTuycwm2nSPiP+c7vq37yE35POXM7QRZbd igfyxerve9Tk9k3/MP+5o8ubOby1QZ5SQHyCkqpLWlZgUPtiPPC2vSz/Ugl8hEaRh5CG GUPg== X-Gm-Message-State: APjAAAXFQtO8JtMv/mXLmuAiUQi/q9ee8plU+pSx2CiXJ1iF5Pxx5cgs JjLq73W2Y05JLnGAruTHrcr3b05gsQ8= X-Google-Smtp-Source: APXvYqxotmlfuwAsw7WVJAmtESCSQ78m4Ol9nxdEm1ea3t25Xajs0EKa1ut3DbCikEdMC5QCiUoB2Q== X-Received: by 2002:a24:7546:: with SMTP id y67mr115517itc.16.1552430565157; Tue, 12 Mar 2019 15:42:45 -0700 (PDT) Received: from ghidorah.spiteful.org (69-165-244-87.cable.teksavvy.com. [69.165.244.87]) by smtp.gmail.com with ESMTPSA id 25sm2843292iog.83.2019.03.12.15.42.44 (version=TLS1_2 cipher=ECDHE-RSA-CHACHA20-POLY1305 bits=256/256); Tue, 12 Mar 2019 15:42:44 -0700 (PDT) From: Scott Murray To: bitbake-devel@lists.openembedded.org Date: Tue, 12 Mar 2019 18:46:23 -0400 Message-Id: <20190312224627.12667-7-scott.murray@konsulko.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190312224627.12667-1-scott.murray@konsulko.com> References: <20190312224627.12667-1-scott.murray@konsulko.com> Subject: [1.40][PATCH 06/10] gitsm.py: Rework the shallow fetcher and test case X-BeenThere: bitbake-devel@lists.openembedded.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: Patches and discussion that advance bitbake development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 12 Mar 2019 22:42:44 -0000 From: Mark Hatle A custom shallow submodule is no longer necessary, as the regular git fetcher is used and shallow handling works with the same code. The only general difference between the regular change is simply declaring a clone as shallow, when appropriate. This also removes a potential race condition in copying repositories vs cloning them. The gitsm shallow fetcher test was revised to verify that the submodule is shallow cloned along with the primary repository. The first step of this change was to be sure to clean the gitsubmodule download directory, as was previously done with the may gitsource directory. Additional test components were added to verify commit counts, and an obsolete (and likely incorrect) test for the .git/modules directory to be empty was also removed. Signed-off-by: Mark Hatle Signed-off-by: Richard Purdie (cherry picked from commit f9cc4684dcf4281acc557cda8cb35602354ac3d6) Signed-off-by: Scott Murray --- lib/bb/fetch2/gitsm.py | 66 +++--------------------------------------- lib/bb/tests/fetch.py | 10 ++++++- 2 files changed, 13 insertions(+), 63 deletions(-) diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py index c172ab16..83571f83 100644 --- a/lib/bb/fetch2/gitsm.py +++ b/lib/bb/fetch2/gitsm.py @@ -150,68 +150,7 @@ class GitSM(Git): def download(self, ud, d): Git.download(self, ud, d) - - if not ud.shallow or ud.localpath != ud.fullshallow: - self.update_submodules(ud, d) - - def copy_submodules(self, submodules, ud, name, destdir, d): - if ud.bareclone: - repo_conf = destdir - else: - repo_conf = os.path.join(destdir, '.git') - - if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')): - os.mkdir(os.path.join(repo_conf, 'modules')) - - for module, md in submodules.items(): - srcpath = os.path.join(ud.clonedir, 'modules', md['path']) - modpath = os.path.join(repo_conf, 'modules', md['path']) - - # Check if the module is initialized - try: - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir) - except: - # If the command fails, we don't have a valid file to check. If it doesn't - # fail -- it still might be a failure, see next check... - module_hash = "" - - if not module_hash: - logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module) - continue - - if os.path.exists(srcpath): - if os.path.exists(os.path.join(srcpath, '.git')): - srcpath = os.path.join(srcpath, '.git') - - target = modpath - if os.path.exists(modpath): - target = os.path.dirname(modpath) - - os.makedirs(os.path.dirname(target), exist_ok=True) - runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d) - elif os.path.exists(modpath): - # Module already exists, likely unpacked from a shallow mirror clone - pass - else: - # This is fatal, as we do NOT want git-submodule to hit the network - raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath)) - - def clone_shallow_local(self, ud, dest, d): - super(GitSM, self).clone_shallow_local(ud, dest, d) - - # Copy over the submodules' fetched histories too. - repo_conf = os.path.join(dest, '.git') - - submodules = [] - for name in ud.names: - try: - gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest) - except: - # No submodules to update - continue - - submodules = self.parse_gitmodules(gitmodules) - self.copy_submodules(submodules, ud, name, dest, d) + self.update_submodules(ud, d) def unpack_submodules(self, repo_conf, ud, d): submodules = [] @@ -294,6 +233,9 @@ class GitSM(Git): # Correct the submodule references to the local download version... runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir) + if ud.shallow: + runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) + # Ensure the submodule repository is NOT set to bare, since we're checking it out... runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', paths[module])) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 311c7013..5fb5d04c 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -1316,6 +1316,7 @@ class GitShallowTest(FetcherTest): # fetch and unpack, from the shallow tarball bb.utils.remove(self.gitdir, recurse=True) bb.utils.remove(ud.clonedir, recurse=True) + bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) # confirm that the unpacked repo is used when no git clone or git # mirror tarball is available @@ -1470,6 +1471,7 @@ class GitShallowTest(FetcherTest): self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) self.add_empty_file('asub', cwd=smdir) + self.add_empty_file('bsub', cwd=smdir) self.git('submodule init', cwd=self.srcdir) self.git('submodule add file://%s' % smdir, cwd=self.srcdir) @@ -1479,10 +1481,16 @@ class GitShallowTest(FetcherTest): uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir fetcher, ud = self.fetch_shallow(uri) + # Verify the main repository is shallow self.assertRevCount(1) - assert './.git/modules/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] + + # Verify the gitsubmodule directory is present assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) + # Verify the submodule is also shallow + self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) + + if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): def test_shallow_annex(self): self.add_empty_file('a') -- 2.17.2