sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: data/codeparser: Improve handling of contains functions 

One of the current frustrations with the sstate checksums is that
code like base_contains('X', 'y',...) adds a full dependency on X
and varies depend even on whitespace changes in X.

This patch adds special handling of the contains functions to expand
the first parameter and check for the flag specified by the second
parameter (assuming its a string).

The result is then appended to the value of the variable with a "Set"
or "Unset" status. If the flag is added/removed, the stored variable
value changes and hence the checksum changes. No dependency on X
is added so it is free to change with regard to other flags or
whitespace.

This code is far from ideal, ideally we'd have properly typed variables
however it fixes a major annoyance of the current checksums and
is of enough value its worth adding in a stopgap solution. It shouldn't
significantly restrict any propely typed variable implementation in
future.

(Bitbake rev: ed2d0a22a80299de0cfd377999950cf4b26c512e)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Christopher Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2013-11-25 22:59:39 +00:00
parent d061692372
commit a10301f6a9
3 changed files with 33 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bitbake/lib/bb/codeparser.py
View File

@ -1,6 +1,7 @@
import ast
import codegen
import logging
import collections
import os.path
import bb.utils, bb.data
from itertools import chain
@ -35,7 +36,7 @@ def check_indent(codestr):
class CodeParserCache(MultiProcessCache):
cache_file_name = "bb_codeparser.dat"
CACHE_VERSION = 3
CACHE_VERSION = 4
def __init__(self):
MultiProcessCache.__init__(self)
@ -122,7 +123,11 @@ class PythonParser():
name = self.called_node_name(node.func)
if name in self.getvars or name in self.containsfuncs:
if isinstance(node.args[0], ast.Str):
self.references.add(node.args[0].s)
varname = node.args[0].s
if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
self.contains[varname].add(node.args[1].s)
else:
self.references.add(node.args[0].s)
else:
self.warn(node.func, node.args[0])
elif name in self.execfuncs:
@ -148,6 +153,7 @@ class PythonParser():
def __init__(self, name, log):
self.var_execs = set()
self.contains = collections.defaultdict(set)
self.execs = set()
self.references = set()
self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log)
@ -161,14 +167,15 @@ class PythonParser():
if h in codeparsercache.pythoncache:
self.references = codeparsercache.pythoncache[h]["refs"]
self.execs = codeparsercache.pythoncache[h]["execs"]
self.contains = codeparsercache.pythoncache[h]["contains"]
return
if h in codeparsercache.pythoncacheextras:
self.references = codeparsercache.pythoncacheextras[h]["refs"]
self.execs = codeparsercache.pythoncacheextras[h]["execs"]
self.contains = codeparsercache.pythoncacheextras[h]["contains"]
return
code = compile(check_indent(str(node)), "<string>", "exec",
ast.PyCF_ONLY_AST)
@ -181,6 +188,7 @@ class PythonParser():
codeparsercache.pythoncacheextras[h] = {}
codeparsercache.pythoncacheextras[h]["refs"] = self.references
codeparsercache.pythoncacheextras[h]["execs"] = self.execs
codeparsercache.pythoncacheextras[h]["contains"] = self.contains
class ShellParser():
def __init__(self, name, log):

18
bitbake/lib/bb/data.py
View File

@ -299,6 +299,21 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
vardeps = varflags.get("vardeps")
value = d.getVar(key, False)
def handle_contains(value, contains, d):
newvalue = ""
for k in contains:
l = (d.getVar(k, True) or "").split()
for word in contains[k]:
if word in l:
newvalue += "\n%s{%s} = Set" % (k, word)
else:
newvalue += "\n%s{%s} = Unset" % (k, word)
if not newvalue:
return value
if not value:
return newvalue
return value + newvalue
if "vardepvalue" in varflags:
value = varflags.get("vardepvalue")
elif varflags.get("func"):
@ -309,6 +324,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
logger.warn("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True)))
parser.parse_python(parsedvar.value)
deps = deps | parser.references
value = handle_contains(value, parser.contains, d)
else:
parsedvar = d.expandWithRefs(value, key)
parser = bb.codeparser.ShellParser(key, logger)
@ -318,10 +334,12 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
parser.log.flush()
deps = deps | parsedvar.references
deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
value = handle_contains(value, parsedvar.contains, d)
else:
parser = d.expandWithRefs(value, key)
deps |= parser.references
deps = deps | (keys & parser.execs)
value = handle_contains(value, parser.contains, d)
# Add varflags, assuming an exclusion list is set
if varflagsexcl:

4
bitbake/lib/bb/data_smart.py
View File

@ -35,6 +35,7 @@ import hashlib
import bb, bb.codeparser
from bb import utils
from bb.COW import COWDictBase
import collections
logger = logging.getLogger("BitBake.Data")
@ -88,6 +89,7 @@ class VariableParse:
self.references = set()
self.execs = set()
self.contains = collections.defaultdict(set)
def var_sub(self, match):
key = match.group()[2:-1]
@ -120,6 +122,8 @@ class VariableParse:
self.references |= parser.references
self.execs |= parser.execs
for k in parser.contains:
self.contains[k].update(parser.contains[k])
value = utils.better_eval(codeobj, DataContext(self.d))
return str(value)