From 43595fabbe0c9084a3878645aef8e3e1c2f79452 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Tue, 31 Aug 2010 14:49:43 +0100 Subject: [PATCH] bitbake: Implement signatures Includes functionality to find out what changes between two different singature data dumps. Signed-off-by: Richard Purdie --- bitbake/bin/bitbake | 3 +++ bitbake/lib/bb/cache.py | 8 ++++++- bitbake/lib/bb/cooker.py | 4 ++++ bitbake/lib/bb/parse/__init__.py | 4 ++++ bitbake/lib/bb/parse/ast.py | 6 ++--- bitbake/lib/bb/runqueue.py | 38 +++++++++++++++++++++++++++++++- bitbake/lib/bb/utils.py | 1 + 7 files changed, 59 insertions(+), 5 deletions(-) diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake index 7caa5d95e6..4e6815e46e 100755 --- a/bitbake/bin/bitbake +++ b/bitbake/bin/bitbake @@ -120,6 +120,9 @@ Default BBFILES are the .bb files in the current directory.""") parser.add_option("-n", "--dry-run", help = "don't execute, just go through the motions", action = "store_true", dest = "dry_run", default = False) + parser.add_option("-S", "--dump-signatures", help = "don't execute, just dump out the signature construction information", + action = "store_true", dest = "dump_signatures", default = False) + parser.add_option("-p", "--parse-only", help = "quit after parsing the BB files (developers only)", action = "store_true", dest = "parse_only", default = False) diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 21cbad915e..646fdecbeb 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -38,7 +38,7 @@ except ImportError: import pickle bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.") -__cache_version__ = "131" +__cache_version__ = "132" class Cache: """ @@ -377,6 +377,10 @@ class Cache: cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True) + cacheData.tasks[file_name] = self.getVar('__BBTASKS', file_name, True) + for t in cacheData.tasks[file_name]: + cacheData.basetaskhash[file_name + "." + t] = self.getVar("BB_BASEHASH_task-%s" % t, file_name, True) + # build FileName to PackageName lookup table cacheData.pkg_fn[file_name] = pn cacheData.pkg_pepvpr[file_name] = (pe, pv, pr) @@ -539,6 +543,8 @@ class CacheData: self.task_deps = {} self.stamp = {} self.preferred = {} + self.tasks = {} + self.basetaskhash = {} """ Indirect Cache variables diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index ea33693ddb..5cea9dba9d 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -496,6 +496,8 @@ class BBCooker: def parseConfigurationFiles(self, files): try: data = self.configuration.data + + bb.parse.init_parser(data) for f in files: data = bb.parse.handle(f, data) @@ -548,6 +550,8 @@ class BBCooker: bb.fetch.fetcher_init(self.configuration.data) bb.codeparser.parser_cache_init(self.configuration.data) + bb.parse.init_parser(data) + bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) except IOError as e: diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py index 95f372b00b..c7249ef050 100644 --- a/bitbake/lib/bb/parse/__init__.py +++ b/bitbake/lib/bb/parse/__init__.py @@ -28,6 +28,7 @@ handlers = [] import bb, os import bb.utils +import bb.siggen class ParseError(Exception): """Exception raised when parsing fails""" @@ -79,6 +80,9 @@ def init(fn, data): if h['supports'](fn): return h['init'](data) +def init_parser(d): + bb.parse.siggen = bb.siggen.init(d) + def resolve_file(fn, d): if not os.path.isabs(fn): bbpath = bb.data.getVar("BBPATH", d, True) diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py index b800569434..870ae65b0e 100644 --- a/bitbake/lib/bb/parse/ast.py +++ b/bitbake/lib/bb/parse/ast.py @@ -300,7 +300,7 @@ def handleInherit(statements, m): n = __word__.findall(files) statements.append(InheritNode(m.group(1))) -def finalize(fn, d): +def finalize(fn, d, variant = None): for lazykey in bb.data.getVar("__lazy_assigned", d) or (): if bb.data.getVar(lazykey, d) is None: val = bb.data.getVarFlag(lazykey, "defaultval", d) @@ -323,7 +323,7 @@ def finalize(fn, d): tasklist = bb.data.getVar('__BBTASKS', d) or [] bb.build.add_tasks(tasklist, d) - #bb.data.generate_dependencies(d) + bb.parse.siggen.finalise(fn, d, variant) bb.event.fire(bb.event.RecipeParsed(fn), d) @@ -433,7 +433,7 @@ def multi_finalize(fn, d): for variant, variant_d in datastores.iteritems(): if variant: try: - finalize(fn, variant_d) + finalize(fn, variant_d, variant) except bb.parse.SkipPackage: bb.data.setVar("__SKIPPED", True, variant_d) diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 1b974312b2..3d84bb17ee 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -178,6 +178,7 @@ class RunQueueData: self.runq_task = [] self.runq_depends = [] self.runq_revdeps = [] + self.runq_hash = [] def runq_depends_names(self, ids): import re @@ -477,6 +478,7 @@ class RunQueueData: self.runq_task.append(taskData.tasks_name[task]) self.runq_depends.append(set(depends)) self.runq_revdeps.append(set()) + self.runq_hash.append("") runq_build.append(0) runq_recrdepends.append(recrdepends) @@ -589,6 +591,7 @@ class RunQueueData: del self.runq_depends[listid-delcount] del runq_build[listid-delcount] del self.runq_revdeps[listid-delcount] + del self.runq_hash[listid-delcount] delcount = delcount + 1 maps.append(-1) @@ -686,6 +689,20 @@ class RunQueueData: #bb.note("Found setscene for %s %s" % (self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task])) self.runq_setscene.append(task) + # Interate over the task list and call into the siggen code + + dealtwith = set() + todeal = set(range(len(self.runq_fnid))) + while len(todeal) > 0: + for task in todeal.copy(): + if len(self.runq_depends[task] - dealtwith) == 0: + dealtwith.add(task) + todeal.remove(task) + procdep = [] + for dep in self.runq_depends[task]: + procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep]) + self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache) + return len(self.runq_fnid) def dump_data(self, taskQueue): @@ -885,7 +902,10 @@ class RunQueue: self.state = runQueueSceneInit if self.state is runQueueSceneInit: - self.rqexe = RunQueueExecuteScenequeue(self) + if self.cooker.configuration.dump_signatures: + self.dump_signatures() + else: + self.rqexe = RunQueueExecuteScenequeue(self) if self.state is runQueueSceneRun: self.rqexe.execute() @@ -926,6 +946,20 @@ class RunQueue: else: self.rqexe.finish() + def dump_signatures(self): + self.state = runQueueComplete + done = set() + bb.note("Reparsing files to collect dependency data") + for task in range(len(self.rqdata.runq_fnid)): + if self.rqdata.runq_fnid[task] not in done: + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data) + done.add(self.rqdata.runq_fnid[task]) + + bb.parse.siggen.dump_sigs(self.rqdata.dataCache) + + return + class RunQueueExecute: @@ -1007,6 +1041,8 @@ class RunQueueExecute: comps = var.split("=") env[comps[0]] = comps[1] + env['BB_TASKHASH'] = self.rqdata.runq_hash[task] + sys.stdout.flush() sys.stderr.flush() diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index 9fc56eaa0f..c2e6ff08ed 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -452,6 +452,7 @@ def preserved_envvars_list(): 'BB_PRESERVE_ENV', 'BB_ENV_WHITELIST', 'BB_ENV_EXTRAWHITE', + 'BB_TASKHASH', 'COLORTERM', 'DBUS_SESSION_BUS_ADDRESS', 'DESKTOP_SESSION',