bitbake/cooker/codeparser: Ensure the code parser cache is saved for each parsing process
Before this change, the codeparser cache was only being saved for the main server process. This is suboptimal as it leaves code being re-evaluated at task execution time and increases parse time. We use the multiprocess Finalize() functionality to ensure each process saves out its cache. We need to update the cache save function to be multiprocess friendly with locking. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
parent
9211fd9c37
commit
c8928e93dd
|
@ -70,8 +70,22 @@ def parser_cache_save(d):
|
|||
if not cachefile:
|
||||
return
|
||||
|
||||
lf = bb.utils.lockfile(cachefile + ".lock")
|
||||
|
||||
p = pickle.Unpickler(file(cachefile, "rb"))
|
||||
data, version = p.load()
|
||||
|
||||
if version == PARSERCACHE_VERSION:
|
||||
for h in data[0]:
|
||||
if h not in pythonparsecache:
|
||||
pythonparsecache[h] = data[0][h]
|
||||
for h in data[1]:
|
||||
if h not in pythonparsecache:
|
||||
shellparsecache[h] = data[1][h]
|
||||
|
||||
p = pickle.Pickler(file(cachefile, "wb"), -1)
|
||||
p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION])
|
||||
bb.utils.unlockfile(lf)
|
||||
|
||||
class PythonParser():
|
||||
class ValueVisitor():
|
||||
|
|
|
@ -1112,6 +1112,7 @@ class CookerParser(object):
|
|||
def start(self):
|
||||
def init(cfg):
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cooker.configuration.data, ), exitpriority=1)
|
||||
parse_file.cfg = cfg
|
||||
|
||||
bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
|
||||
|
@ -1137,10 +1138,6 @@ class CookerParser(object):
|
|||
sync.start()
|
||||
atexit.register(lambda: sync.join())
|
||||
|
||||
codesync = threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data))
|
||||
codesync.start()
|
||||
atexit.register(lambda: codesync.join())
|
||||
|
||||
def load_cached(self):
|
||||
for filename, appends in self.fromcache:
|
||||
cached, infos = self.bb_cache.load(filename, appends, self.cfgdata)
|
||||
|
|
Loading…
Reference in New Issue