sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake/cache/runqueue.py: Move workload for recipe parsing to the child process 

Parsing the recipe in the parent before forking off the child worker
can mean the parent doesn't hit the idle loop and becomes a bottleneck
when lauching many short lived processes.

The reason we need this in the parent is to figure out the fakeroot
environmental options. To address this, add the fakeroot variables
to the cache and move recipe loadData into the child task.

For a poky-image-sato build this results in about a 2 minute speedup
(1.8%).

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2011-02-28 15:31:20 +00:00
parent d5e12a1bfb
commit 1b08a7eb8b
2 changed files with 32 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
bitbake/lib/bb/cache.py
View File

@ -43,7 +43,7 @@ except ImportError:
logger.info("Importing cPickle failed. "
"Falling back to a very slow implementation.")
__cache_version__ = "137"
__cache_version__ = "138"
recipe_fields = (
'pn',
@ -78,6 +78,8 @@ recipe_fields = (
'summary',
'license',
'section',
'fakerootenv',
'fakerootdirs'
)
@ -172,6 +174,8 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
summary = cls.getvar('SUMMARY', metadata),
license = cls.getvar('LICENSE', metadata),
section = cls.getvar('SECTION', metadata),
fakerootenv = cls.getvar('FAKEROOTENV', metadata),
fakerootdirs = cls.getvar('FAKEROOTDIRS', metadata),
)
@ -584,6 +588,8 @@ class CacheData(object):
self.summary = {}
self.license = {}
self.section = {}
self.fakerootenv = {}
self.fakerootdirs = {}
# Indirect Cache variables (set elsewhere)
self.ignored_dependencies = []
@ -647,3 +653,5 @@ class CacheData(object):
self.summary[fn] = info.summary
self.license[fn] = info.license
self.section[fn] = info.section
self.fakerootenv[fn] = info.fakerootenv
self.fakerootdirs[fn] = info.fakerootdirs

35
bitbake/lib/bb/runqueue.py
View File

@ -1060,27 +1060,23 @@ class RunQueueExecute:
return
def fork_off_task(self, fn, task, taskname, quieterrors=False):
the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data)
env = bb.data.export_vars(the_data)
env = bb.data.export_envvars(env, the_data)
envbackup = os.environ.copy()
env = {}
taskdep = self.rqdata.dataCache.task_deps[fn]
if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']:
envvars = the_data.getVar("FAKEROOTENV", True).split()
envvars = (self.rqdata.dataCache.fakerootenv[fn] or "").split()
for var in envvars:
comps = var.split("=")
env[comps[0]] = comps[1]
fakedirs = (the_data.getVar("FAKEROOTDIRS", True) or "").split()
fakedirs = (self.rqdata.dataCache.fakerootdirs[fn] or "").split()
for p in fakedirs:
bb.mkdirhier(p)
logger.debug(2, "Running %s:%s under fakeroot, state dir is %s" % (fn, taskname, fakedirs))
envbackup = os.environ.copy()
for e in envbackup:
os.unsetenv(e)
for e in env:
os.putenv(e, env[e])
for e in env:
os.putenv(e, env[e])
sys.stdout.flush()
sys.stderr.flush()
@ -1111,6 +1107,20 @@ class RunQueueExecute:
# No stdin
newsi = os.open(os.devnull, os.O_RDWR)
os.dup2(newsi, sys.stdin.fileno())
the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data)
env2 = bb.data.export_vars(the_data)
env2 = bb.data.export_envvars(env2, the_data)
for e in os.environ:
os.unsetenv(e)
for e in env2:
os.putenv(e, env2[e])
for e in env:
os.putenv(e, env[e])
if quieterrors:
the_data.setVarFlag(taskname, "quieterrors", "1")
@ -1137,7 +1147,8 @@ class RunQueueExecute:
for e in env:
os.unsetenv(e)
for e in envbackup:
os.putenv(e, envbackup[e])
if e in env:
os.putenv(e, envbackup[e])
return pid, pipein, pipeout