From: Dongxiao Xu <dongxiao.xu@intel.com>
To: bitbake-devel@lists.openembedded.org
Subject: [PATCH 02/10] cache: Use configuration's hash value to validate cache
Date: Thu, 15 Dec 2011 15:14:53 +0800 [thread overview]
Message-ID: <1ef50ac57fc6ab1bf0f49c4c1ef476786a963577.1323933009.git.dongxiao.xu@intel.com> (raw)
In-Reply-To: <cover.1323933009.git.dongxiao.xu@intel.com>
In-Reply-To: <cover.1323933009.git.dongxiao.xu@intel.com>
Previously we use the file time stamp to judge if a cache is valid.
Here this commit introduce a new method, which calculates the total
hash value for a certain configuration's key/value paris, and tag
it into cache filename, for example, bb_cache.dat.xxxyyyzzz.
This mechanism also ensures the cache's correctness if user
dynamically setting variables from some frontend GUI, like HOB.
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
lib/bb/cache.py | 32 ++++++++++++--------------------
lib/bb/cooker.py | 4 +++-
lib/bb/data_smart.py | 17 +++++++++++++++++
3 files changed, 32 insertions(+), 21 deletions(-)
diff --git a/lib/bb/cache.py b/lib/bb/cache.py
index 6b7fa6f..955b6df 100644
--- a/lib/bb/cache.py
+++ b/lib/bb/cache.py
@@ -42,10 +42,10 @@ except ImportError:
logger.info("Importing cPickle failed. "
"Falling back to a very slow implementation.")
-__cache_version__ = "142"
+__cache_version__ = "143"
-def getCacheFile(path, filename):
- return os.path.join(path, filename)
+def getCacheFile(path, filename, data_hash):
+ return os.path.join(path, filename + "." + data_hash)
# RecipeInfoCommon defines common data retrieving methods
# from meta data for caches. CoreRecipeInfo as well as other
@@ -254,7 +254,7 @@ class Cache(object):
BitBake Cache implementation
"""
- def __init__(self, data, caches_array):
+ def __init__(self, data, data_hash, caches_array):
# Pass caches_array information into Cache Constructor
# It will be used in later for deciding whether we
# need extra cache file dump/load support
@@ -266,6 +266,7 @@ class Cache(object):
self.data = None
self.data_fn = None
self.cacheclean = True
+ self.data_hash = data_hash
if self.cachedir in [None, '']:
self.has_cache = False
@@ -274,26 +275,17 @@ class Cache(object):
return
self.has_cache = True
- self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat")
+ self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
logger.debug(1, "Using cache in '%s'", self.cachedir)
bb.utils.mkdirhier(self.cachedir)
- # If any of configuration.data's dependencies are newer than the
- # cache there isn't even any point in loading it...
- newest_mtime = 0
- deps = data.getVar("__base_depends")
-
- old_mtimes = [old_mtime for _, old_mtime in deps]
- old_mtimes.append(newest_mtime)
- newest_mtime = max(old_mtimes)
-
cache_ok = True
if self.caches_array:
for cache_class in self.caches_array:
if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
- cache_ok = cache_ok and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime)
+ cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+ cache_ok = cache_ok and os.path.exists(cachefile)
cache_class.init_cacheData(self)
if cache_ok:
self.load_cachefile()
@@ -327,7 +319,7 @@ class Cache(object):
# Calculate the correct cachesize of all those cache files
for cache_class in self.caches_array:
if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+ cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
with open(cachefile, "rb") as cachefile:
cachesize += os.fstat(cachefile.fileno()).st_size
@@ -335,7 +327,7 @@ class Cache(object):
for cache_class in self.caches_array:
if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+ cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
with open(cachefile, "rb") as cachefile:
pickled = pickle.Unpickler(cachefile)
while cachefile:
@@ -588,7 +580,7 @@ class Cache(object):
for cache_class in self.caches_array:
if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
cache_class_name = cache_class.__name__
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+ cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
file_dict[cache_class_name] = open(cachefile, "wb")
pickler_dict[cache_class_name] = pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
@@ -693,7 +685,7 @@ def init(cooker):
Files causing parsing errors are evicted from the cache.
"""
- return Cache(cooker.configuration.data)
+ return Cache(cooker.configuration.data, cooker.configuration.data_hash)
class CacheData(object):
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index cbe0d71..2c02e28 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -832,6 +832,7 @@ class BBCooker:
bb.parse.init_parser(data)
bb.event.fire(bb.event.ConfigParsed(), data)
self.configuration.data = data
+ self.configuration.data_hash = data.get_hash()
def handleCollections( self, collections ):
"""Handle collections"""
@@ -1399,6 +1400,7 @@ class CookerParser(object):
self.filelist = filelist
self.cooker = cooker
self.cfgdata = cooker.configuration.data
+ self.cfghash = cooker.configuration.data_hash
# Accounting statistics
self.parsed = 0
@@ -1414,7 +1416,7 @@ class CookerParser(object):
self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
multiprocessing.cpu_count())
- self.bb_cache = bb.cache.Cache(self.cfgdata, cooker.caches_array)
+ self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array)
self.fromcache = []
self.willparse = []
for filename in self.filelist:
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index ea13478..06d8d0b 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -31,6 +31,7 @@ BitBake build tools.
import copy, re
from collections import MutableMapping
import logging
+import hashlib
import bb, bb.codeparser
from bb import utils
from bb.COW import COWDictBase
@@ -459,3 +460,19 @@ class DataSmart(MutableMapping):
def __delitem__(self, var):
self.delVar(var)
+
+ def get_hash(self):
+ data = ""
+ keys = iter(self)
+ for key in keys:
+ if key == "TIME":
+ continue
+ if key == "__depends":
+ deps = list(self.getVar(key, False))
+ deps.sort()
+ value = [deps[i][0] for i in range(len(deps))]
+ else:
+ value = self.getVar(key, False) or ""
+ data = data + key + str(value)
+
+ return hashlib.md5(data).hexdigest()
--
1.7.0.4
next prev parent reply other threads:[~2011-12-15 7:21 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2011-12-15 7:14 [PATCH 00/10 v2] Hob2 related bitbake changes Dongxiao Xu
2011-12-15 7:14 ` [PATCH 01/10] command.py: Modify needcache value for certain functions Dongxiao Xu
2011-12-15 7:14 ` Dongxiao Xu [this message]
2011-12-15 7:14 ` [PATCH 03/10] cooker: user bb.configuration.data to inject events Dongxiao Xu
2011-12-15 7:14 ` [PATCH 04/10] command.py: add initCooker API Dongxiao Xu
2011-12-15 7:14 ` [PATCH 05/10] command.py: add parseConfigurationFiles API Dongxiao Xu
2011-12-15 7:14 ` [PATCH 06/10] command.py: add resolve option for generateTargetsTree API Dongxiao Xu
2011-12-15 7:14 ` [PATCH 07/10] event.py: Add a new event PackageInfo Dongxiao Xu
2011-12-15 7:14 ` [PATCH 08/10] xmlrpc: Change BitbakeServerInfo init function Dongxiao Xu
2011-12-15 7:15 ` [PATCH 09/10] cooker: remove command import in cooker.py Dongxiao Xu
2011-12-15 7:15 ` [PATCH 10/10] bitbake: add a new option "--server-only" Dongxiao Xu
2011-12-19 11:00 ` [PATCH 00/10 v2] Hob2 related bitbake changes Xu, Dongxiao
-- strict thread matches above, loose matches on Subject: below --
2011-12-12 2:20 [PATCH 00/10][PULL] " Dongxiao Xu
2011-12-12 2:20 ` [PATCH 02/10] cache: Use configuration's hash value to validate cache Dongxiao Xu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1ef50ac57fc6ab1bf0f49c4c1ef476786a963577.1323933009.git.dongxiao.xu@intel.com \
--to=dongxiao.xu@intel.com \
--cc=bitbake-devel@lists.openembedded.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.