cache.py: Refactory Current Cache implementation
This patch is for refactorying current cache implementation, the main reason is for introducing extra cache fields requests for image creator as well as other users. The refactory parts include: Move cache data retrieve methods out of Cache Data Fields Definition. Since this retrieve methods will be shared for both CoreRecipeInfo as well as the new introduced extra RecipeInfo in the following patches. (Bitbake rev: f0f53506926a3f79181796dde177f11f0a396b75) Signed-off-by: Liping Ke <liping.ke@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Liping Ke
Richard Purdie
parent
911e8bb56a
commit
8df355140c
|
|
@ -30,7 +30,7 @@
|
|||
|
||||
import os
|
||||
import logging
|
||||
from collections import defaultdict, namedtuple
|
||||
from collections import defaultdict
|
||||
import bb.data
|
||||
import bb.utils
|
||||
|
||||
|
|
@ -45,46 +45,11 @@ except ImportError:
|
|||
|
||||
__cache_version__ = "138"
|
||||
|
||||
recipe_fields = (
|
||||
'pn',
|
||||
'pv',
|
||||
'pr',
|
||||
'pe',
|
||||
'defaultpref',
|
||||
'depends',
|
||||
'provides',
|
||||
'task_deps',
|
||||
'stamp',
|
||||
'stamp_extrainfo',
|
||||
'broken',
|
||||
'not_world',
|
||||
'skipped',
|
||||
'timestamp',
|
||||
'packages',
|
||||
'packages_dynamic',
|
||||
'rdepends',
|
||||
'rdepends_pkg',
|
||||
'rprovides',
|
||||
'rprovides_pkg',
|
||||
'rrecommends',
|
||||
'rrecommends_pkg',
|
||||
'nocache',
|
||||
'variants',
|
||||
'file_depends',
|
||||
'tasks',
|
||||
'basetaskhashes',
|
||||
'hashfilename',
|
||||
'inherits',
|
||||
'summary',
|
||||
'license',
|
||||
'section',
|
||||
'fakerootenv',
|
||||
'fakerootdirs'
|
||||
)
|
||||
|
||||
|
||||
class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
|
||||
__slots__ = ()
|
||||
# RecipeInfoCommon defines common data retrieving methods
|
||||
# from meta data for caches. CoreRecipeInfo as well as other
|
||||
# Extra RecipeInfo needs to inherit this class
|
||||
class RecipeInfoCommon(object):
|
||||
|
||||
@classmethod
|
||||
def listvar(cls, var, metadata):
|
||||
|
|
@ -117,69 +82,162 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
|
|||
def getvar(cls, var, metadata):
|
||||
return metadata.getVar(var, True) or ''
|
||||
|
||||
@classmethod
|
||||
def make_optional(cls, default=None, **kwargs):
|
||||
"""Construct the namedtuple from the specified keyword arguments,
|
||||
with every value considered optional, using the default value if
|
||||
it was not specified."""
|
||||
for field in cls._fields:
|
||||
kwargs[field] = kwargs.get(field, default)
|
||||
return cls(**kwargs)
|
||||
|
||||
class CoreRecipeInfo(RecipeInfoCommon):
|
||||
__slots__ = ()
|
||||
|
||||
def __init__(self, filename, metadata):
|
||||
self.name = "base"
|
||||
# please override this member with the correct data cache file
|
||||
# such as (bb_cache.dat, bb_extracache_hob.dat)
|
||||
self.cachefile = "bb_cache.dat"
|
||||
|
||||
self.file_depends = metadata.getVar('__depends', False)
|
||||
self.timestamp = bb.parse.cached_mtime(filename)
|
||||
self.variants = self.listvar('__VARIANTS', metadata) + ['']
|
||||
self.nocache = self.getvar('__BB_DONT_CACHE', metadata)
|
||||
|
||||
if self.getvar('__SKIPPED', metadata):
|
||||
self.skipped = True
|
||||
return
|
||||
|
||||
self.tasks = metadata.getVar('__BBTASKS', False)
|
||||
|
||||
self.pn = self.getvar('PN', metadata)
|
||||
self.packages = self.listvar('PACKAGES', metadata)
|
||||
if not self.pn in self.packages:
|
||||
self.packages.append(self.pn)
|
||||
|
||||
self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
|
||||
self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
|
||||
|
||||
self.file_depends = metadata.getVar('__depends', False)
|
||||
self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}
|
||||
|
||||
self.skipped = False
|
||||
self.pe = self.getvar('PE', metadata)
|
||||
self.pv = self.getvar('PV', metadata)
|
||||
self.pr = self.getvar('PR', metadata)
|
||||
self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata)
|
||||
self.broken = self.getvar('BROKEN', metadata)
|
||||
self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata)
|
||||
self.stamp = self.getvar('STAMP', metadata)
|
||||
self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
|
||||
self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
|
||||
self.depends = self.depvar('DEPENDS', metadata)
|
||||
self.provides = self.depvar('PROVIDES', metadata)
|
||||
self.rdepends = self.depvar('RDEPENDS', metadata)
|
||||
self.rprovides = self.depvar('RPROVIDES', metadata)
|
||||
self.rrecommends = self.depvar('RRECOMMENDS', metadata)
|
||||
self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata)
|
||||
self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata)
|
||||
self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata)
|
||||
self.inherits = self.getvar('__inherit_cache', metadata)
|
||||
self.summary = self.getvar('SUMMARY', metadata)
|
||||
self.license = self.getvar('LICENSE', metadata)
|
||||
self.section = self.getvar('SECTION', metadata)
|
||||
self.fakerootenv = self.getvar('FAKEROOTENV', metadata)
|
||||
self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata)
|
||||
|
||||
@classmethod
|
||||
def from_metadata(cls, filename, metadata):
|
||||
if cls.getvar('__SKIPPED', metadata):
|
||||
return cls.make_optional(skipped=True,
|
||||
file_depends=metadata.getVar('__depends', False),
|
||||
timestamp=bb.parse.cached_mtime(filename),
|
||||
variants=cls.listvar('__VARIANTS', metadata) + [''])
|
||||
def init_cacheData(cls, cachedata):
|
||||
# CacheData in Core RecipeInfo Class
|
||||
cachedata.task_deps = {}
|
||||
cachedata.pkg_fn = {}
|
||||
cachedata.pkg_pn = defaultdict(list)
|
||||
cachedata.pkg_pepvpr = {}
|
||||
cachedata.pkg_dp = {}
|
||||
|
||||
tasks = metadata.getVar('__BBTASKS', False)
|
||||
cachedata.stamp = {}
|
||||
cachedata.stamp_extrainfo = {}
|
||||
cachedata.fn_provides = {}
|
||||
cachedata.pn_provides = defaultdict(list)
|
||||
cachedata.all_depends = []
|
||||
|
||||
pn = cls.getvar('PN', metadata)
|
||||
packages = cls.listvar('PACKAGES', metadata)
|
||||
if not pn in packages:
|
||||
packages.append(pn)
|
||||
cachedata.deps = defaultdict(list)
|
||||
cachedata.packages = defaultdict(list)
|
||||
cachedata.providers = defaultdict(list)
|
||||
cachedata.rproviders = defaultdict(list)
|
||||
cachedata.packages_dynamic = defaultdict(list)
|
||||
|
||||
return RecipeInfo(
|
||||
tasks = tasks,
|
||||
basetaskhashes = cls.taskvar('BB_BASEHASH', tasks, metadata),
|
||||
hashfilename = cls.getvar('BB_HASHFILENAME', metadata),
|
||||
cachedata.rundeps = defaultdict(lambda: defaultdict(list))
|
||||
cachedata.runrecs = defaultdict(lambda: defaultdict(list))
|
||||
cachedata.possible_world = []
|
||||
cachedata.universe_target = []
|
||||
cachedata.hashfn = {}
|
||||
|
||||
file_depends = metadata.getVar('__depends', False),
|
||||
task_deps = metadata.getVar('_task_deps', False) or
|
||||
{'tasks': [], 'parents': {}},
|
||||
variants = cls.listvar('__VARIANTS', metadata) + [''],
|
||||
cachedata.basetaskhash = {}
|
||||
cachedata.inherits = {}
|
||||
cachedata.summary = {}
|
||||
cachedata.license = {}
|
||||
cachedata.section = {}
|
||||
cachedata.fakerootenv = {}
|
||||
cachedata.fakerootdirs = {}
|
||||
|
||||
def add_cacheData(self, cachedata, fn):
|
||||
cachedata.task_deps[fn] = self.task_deps
|
||||
cachedata.pkg_fn[fn] = self.pn
|
||||
cachedata.pkg_pn[self].append(fn)
|
||||
cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr)
|
||||
cachedata.pkg_dp[fn] = self.defaultpref
|
||||
cachedata.stamp[fn] = self.stamp
|
||||
cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo
|
||||
|
||||
provides = [self.pn]
|
||||
for provide in self.provides:
|
||||
if provide not in provides:
|
||||
provides.append(provide)
|
||||
cachedata.fn_provides[fn] = provides
|
||||
|
||||
for provide in provides:
|
||||
cachedata.providers[provide].append(fn)
|
||||
if provide not in cachedata.pn_provides[self.pn]:
|
||||
cachedata.pn_provides[self.pn].append(provide)
|
||||
|
||||
for dep in self.depends:
|
||||
if dep not in cachedata.deps[fn]:
|
||||
cachedata.deps[fn].append(dep)
|
||||
if dep not in cachedata.all_depends:
|
||||
cachedata.all_depends.append(dep)
|
||||
|
||||
rprovides = self.rprovides
|
||||
for package in self.packages:
|
||||
cachedata.packages[package].append(fn)
|
||||
rprovides += self.rprovides_pkg[package]
|
||||
|
||||
for rprovide in rprovides:
|
||||
cachedata.rproviders[rprovide].append(fn)
|
||||
|
||||
for package in self.packages_dynamic:
|
||||
cachedata.packages_dynamic[package].append(fn)
|
||||
|
||||
# Build hash of runtime depends and rececommends
|
||||
for package in self.packages + [self.pn]:
|
||||
cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package]
|
||||
cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package]
|
||||
|
||||
# Collect files we may need for possible world-dep
|
||||
# calculations
|
||||
if not self.broken and not self.not_world:
|
||||
cachedata.possible_world.append(fn)
|
||||
|
||||
# create a collection of all targets for sanity checking
|
||||
# tasks, such as upstream versions, license, and tools for
|
||||
# task and image creation.
|
||||
cachedata.universe_target.append(self.pn)
|
||||
|
||||
cachedata.hashfn[fn] = self.hashfilename
|
||||
for task, taskhash in self.basetaskhashes.iteritems():
|
||||
identifier = '%s.%s' % (fn, task)
|
||||
cachedata.basetaskhash[identifier] = taskhash
|
||||
|
||||
cachedata.inherits[fn] = self.inherits
|
||||
cachedata.summary[fn] = self.summary
|
||||
cachedata.license[fn] = self.license
|
||||
cachedata.section[fn] = self.section
|
||||
cachedata.fakerootenv[fn] = self.fakerootenv
|
||||
cachedata.fakerootdirs[fn] = self.fakerootdirs
|
||||
|
||||
skipped = False,
|
||||
timestamp = bb.parse.cached_mtime(filename),
|
||||
packages = cls.listvar('PACKAGES', metadata),
|
||||
pn = pn,
|
||||
pe = cls.getvar('PE', metadata),
|
||||
pv = cls.getvar('PV', metadata),
|
||||
pr = cls.getvar('PR', metadata),
|
||||
nocache = cls.getvar('__BB_DONT_CACHE', metadata),
|
||||
defaultpref = cls.intvar('DEFAULT_PREFERENCE', metadata),
|
||||
broken = cls.getvar('BROKEN', metadata),
|
||||
not_world = cls.getvar('EXCLUDE_FROM_WORLD', metadata),
|
||||
stamp = cls.getvar('STAMP', metadata),
|
||||
stamp_extrainfo = cls.flaglist('stamp-extra-info', tasks, metadata),
|
||||
packages_dynamic = cls.listvar('PACKAGES_DYNAMIC', metadata),
|
||||
depends = cls.depvar('DEPENDS', metadata),
|
||||
provides = cls.depvar('PROVIDES', metadata),
|
||||
rdepends = cls.depvar('RDEPENDS', metadata),
|
||||
rprovides = cls.depvar('RPROVIDES', metadata),
|
||||
rrecommends = cls.depvar('RRECOMMENDS', metadata),
|
||||
rprovides_pkg = cls.pkgvar('RPROVIDES', packages, metadata),
|
||||
rdepends_pkg = cls.pkgvar('RDEPENDS', packages, metadata),
|
||||
rrecommends_pkg = cls.pkgvar('RRECOMMENDS', packages, metadata),
|
||||
inherits = cls.getvar('__inherit_cache', metadata),
|
||||
summary = cls.getvar('SUMMARY', metadata),
|
||||
license = cls.getvar('LICENSE', metadata),
|
||||
section = cls.getvar('SECTION', metadata),
|
||||
fakerootenv = cls.getvar('FAKEROOTENV', metadata),
|
||||
fakerootdirs = cls.getvar('FAKEROOTDIRS', metadata),
|
||||
)
|
||||
|
||||
|
||||
class Cache(object):
|
||||
|
|
@ -314,7 +372,7 @@ class Cache(object):
|
|||
depends |= (data.getVar("__depends", False) or set())
|
||||
if depends and not variant:
|
||||
data.setVar("__depends", depends)
|
||||
info = RecipeInfo.from_metadata(filename, data)
|
||||
info = CoreRecipeInfo(filename, data)
|
||||
infos.append((virtualfn, info))
|
||||
return infos
|
||||
|
||||
|
|
@ -500,7 +558,7 @@ class Cache(object):
|
|||
"""
|
||||
|
||||
realfn = self.virtualfn2realfn(file_name)[0]
|
||||
info = RecipeInfo.from_metadata(realfn, data)
|
||||
info = CoreRecipeInfo(realfn, data)
|
||||
self.add_info(file_name, info, cacheData, parsed)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -566,38 +624,11 @@ class CacheData(object):
|
|||
"""
|
||||
|
||||
def __init__(self):
|
||||
CoreRecipeInfo.init_cacheData(self)
|
||||
# Direct cache variables
|
||||
self.providers = defaultdict(list)
|
||||
self.rproviders = defaultdict(list)
|
||||
self.packages = defaultdict(list)
|
||||
self.packages_dynamic = defaultdict(list)
|
||||
self.possible_world = []
|
||||
self.universe_target = []
|
||||
self.pkg_pn = defaultdict(list)
|
||||
self.pkg_fn = {}
|
||||
self.pkg_pepvpr = {}
|
||||
self.pkg_dp = {}
|
||||
self.pn_provides = defaultdict(list)
|
||||
self.fn_provides = {}
|
||||
self.all_depends = []
|
||||
self.deps = defaultdict(list)
|
||||
self.rundeps = defaultdict(lambda: defaultdict(list))
|
||||
self.runrecs = defaultdict(lambda: defaultdict(list))
|
||||
self.task_queues = {}
|
||||
self.task_deps = {}
|
||||
self.stamp = {}
|
||||
self.stamp_extrainfo = {}
|
||||
self.preferred = {}
|
||||
self.tasks = {}
|
||||
self.basetaskhash = {}
|
||||
self.hashfn = {}
|
||||
self.inherits = {}
|
||||
self.summary = {}
|
||||
self.license = {}
|
||||
self.section = {}
|
||||
self.fakerootenv = {}
|
||||
self.fakerootdirs = {}
|
||||
|
||||
# Indirect Cache variables (set elsewhere)
|
||||
self.ignored_dependencies = []
|
||||
self.world_target = set()
|
||||
|
|
@ -605,65 +636,6 @@ class CacheData(object):
|
|||
self.bbfile_config_priorities = []
|
||||
|
||||
def add_from_recipeinfo(self, fn, info):
|
||||
self.task_deps[fn] = info.task_deps
|
||||
self.pkg_fn[fn] = info.pn
|
||||
self.pkg_pn[info.pn].append(fn)
|
||||
self.pkg_pepvpr[fn] = (info.pe, info.pv, info.pr)
|
||||
self.pkg_dp[fn] = info.defaultpref
|
||||
self.stamp[fn] = info.stamp
|
||||
self.stamp_extrainfo[fn] = info.stamp_extrainfo
|
||||
info.add_cacheData(self, fn)
|
||||
|
||||
provides = [info.pn]
|
||||
for provide in info.provides:
|
||||
if provide not in provides:
|
||||
provides.append(provide)
|
||||
self.fn_provides[fn] = provides
|
||||
|
||||
for provide in provides:
|
||||
self.providers[provide].append(fn)
|
||||
if provide not in self.pn_provides[info.pn]:
|
||||
self.pn_provides[info.pn].append(provide)
|
||||
|
||||
for dep in info.depends:
|
||||
if dep not in self.deps[fn]:
|
||||
self.deps[fn].append(dep)
|
||||
if dep not in self.all_depends:
|
||||
self.all_depends.append(dep)
|
||||
|
||||
rprovides = info.rprovides
|
||||
for package in info.packages:
|
||||
self.packages[package].append(fn)
|
||||
rprovides += info.rprovides_pkg[package]
|
||||
|
||||
for rprovide in rprovides:
|
||||
self.rproviders[rprovide].append(fn)
|
||||
|
||||
for package in info.packages_dynamic:
|
||||
self.packages_dynamic[package].append(fn)
|
||||
|
||||
# Build hash of runtime depends and rececommends
|
||||
for package in info.packages + [info.pn]:
|
||||
self.rundeps[fn][package] = list(info.rdepends) + info.rdepends_pkg[package]
|
||||
self.runrecs[fn][package] = list(info.rrecommends) + info.rrecommends_pkg[package]
|
||||
|
||||
# Collect files we may need for possible world-dep
|
||||
# calculations
|
||||
if not info.broken and not info.not_world:
|
||||
self.possible_world.append(fn)
|
||||
|
||||
# create a collection of all targets for sanity checking
|
||||
# tasks, such as upstream versions, license, and tools for
|
||||
# task and image creation.
|
||||
self.universe_target.append(info.pn)
|
||||
|
||||
self.hashfn[fn] = info.hashfilename
|
||||
for task, taskhash in info.basetaskhashes.iteritems():
|
||||
identifier = '%s.%s' % (fn, task)
|
||||
self.basetaskhash[identifier] = taskhash
|
||||
|
||||
self.inherits[fn] = info.inherits
|
||||
self.summary[fn] = info.summary
|
||||
self.license[fn] = info.license
|
||||
self.section[fn] = info.section
|
||||
self.fakerootenv[fn] = info.fakerootenv
|
||||
self.fakerootdirs[fn] = info.fakerootdirs
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue