From: "Aníbal Limón" <anibal.limon@linaro.org>
To: bitbake-devel@lists.openembedded.org
Subject: [PATCHv2] lib/bb: Add BB_SIGNATURE_LOCAL_DIRS_EXCLUDE to speed-up taskhash on directories
Date: Fri, 15 Nov 2019 08:47:23 -0600 [thread overview]
Message-ID: <20191115144723.7452-1-anibal.limon@linaro.org> (raw)
The new BB_SIGNATURE_LOCAL_DIRS_EXCLUDE allows you to specify a list
of directories to exclude when making taskhash, our specific case
is using SRC_URI that points local VCS directory.
Use bb.fetch.module to set default to: "CVS .bzr .git .hg .osc .p4 .repo .svn"
Signed-off-by: Aníbal Limón <anibal.limon@linaro.org>
---
lib/bb/checksum.py | 5 +++--
lib/bb/fetch2/__init__.py | 4 ++--
lib/bb/siggen.py | 5 +++--
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/lib/bb/checksum.py b/lib/bb/checksum.py
index 5bc8a8fc..677020f4 100644
--- a/lib/bb/checksum.py
+++ b/lib/bb/checksum.py
@@ -74,7 +74,7 @@ class FileChecksumCache(MultiProcessCache):
else:
dest[0][h] = source[0][h]
- def get_checksums(self, filelist, pn):
+ def get_checksums(self, filelist, pn, localdirsexclude):
"""Get checksums for a list of files"""
def checksum_file(f):
@@ -90,7 +90,8 @@ class FileChecksumCache(MultiProcessCache):
if pth == "/":
bb.fatal("Refusing to checksum /")
dirchecksums = []
- for root, dirs, files in os.walk(pth):
+ for root, dirs, files in os.walk(pth, topdown=True):
+ [dirs.remove(d) for d in list(dirs) if d in localdirsexclude]
for name in files:
fullpth = os.path.join(root, name)
checksum = checksum_file(fullpth)
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 07de6c26..731c1608 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1197,14 +1197,14 @@ def get_checksum_file_list(d):
return " ".join(filelist)
-def get_file_checksums(filelist, pn):
+def get_file_checksums(filelist, pn, localdirsexclude):
"""Get a list of the checksums for a list of local files
Returns the checksums for a list of local files, caching the results as
it proceeds
"""
- return _checksum_cache.get_checksums(filelist, pn)
+ return _checksum_cache.get_checksums(filelist, pn, localdirsexclude)
class FetchData(object):
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index a4bb1ff7..2445a340 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -123,6 +123,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
self.unihash_cache = bb.cache.SimpleCache("1")
self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {})
+ self.localdirsexclude = (data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE") or "CVS .bzr .git .hg .osc .p4 .repo .svn").split()
def init_rundepcheck(self, data):
self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
@@ -221,9 +222,9 @@ class SignatureGeneratorBasic(SignatureGenerator):
if task in dataCache.file_checksums[fn]:
if self.checksum_cache:
- checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename)
+ checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude)
else:
- checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
+ checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude)
for (f,cs) in checksums:
self.file_checksum_values[tid].append((f,cs))
if cs:
--
2.24.0
next reply other threads:[~2019-11-15 14:47 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-11-15 14:47 Aníbal Limón [this message]
2019-11-15 16:29 ` [PATCHv2] lib/bb: Add BB_SIGNATURE_LOCAL_DIRS_EXCLUDE to speed-up taskhash on directories Peter Kjellerstedt
2019-11-15 17:12 ` Mark Hatle
2019-11-18 15:22 ` Anibal Limon
2019-12-16 18:55 ` Anibal Limon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191115144723.7452-1-anibal.limon@linaro.org \
--to=anibal.limon@linaro.org \
--cc=bitbake-devel@lists.openembedded.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.