sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake-dev: Turn parsing into a server idle callback allowing the client to interrupt parsing and improving user interactvity. Also now specify whether async commands need the cache or not 

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
This commit is contained in:
Richard Purdie 2009-07-08 22:46:09 +01:00
parent 67d169aa1c
commit 433f50435e
2 changed files with 122 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
bitbake-dev/lib/bb/command.py
View File

@ -57,16 +57,18 @@ class Command:
async_cmds[command] = (method)
def runCommand(self, commandline):
bb.debug("Running command %s" % commandline)
try:
command = commandline.pop(0)
if command in CommandsSync.__dict__:
# Can run online commands straight away
# Can run synchronous commands straight away
return getattr(CommandsSync, command)(self.cmds_sync, self, commandline)
if self.currentAsyncCommand is not None:
return "Busy (%s in progress)" % self.currentAsyncCommand[0]
if command not in CommandsAsync.__dict__:
return "No such command"
self.currentAsyncCommand = (command, commandline)
self.cooker.server.register_idle_function(self.cooker.runCommands, self.cooker)
return True
except:
import traceback
@ -76,10 +78,20 @@ class Command:
try:
if self.currentAsyncCommand is not None:
(command, options) = self.currentAsyncCommand
getattr(CommandsAsync, command)(self.cmds_async, self, options)
commandmethod = getattr(CommandsAsync, command)
needcache = getattr( commandmethod, "needcache" )
if needcache and self.cooker.cookerState != bb.cooker.cookerParsed:
self.cooker.updateCache()
return True
else:
commandmethod(self.cmds_async, self, options)
return False
else:
return False
except:
import traceback
self.finishAsyncCommand(traceback.format_exc())
return False
def finishAsyncCommand(self, error = None):
if error:
@ -149,6 +161,7 @@ class CommandsAsync:
task = params[1]
command.cooker.buildFile(bfile, task)
buildFile.needcache = False
def buildTargets(self, command, params):
"""
@ -158,6 +171,7 @@ class CommandsAsync:
task = params[1]
command.cooker.buildTargets(pkgs_to_build, task)
buildTargets.needcache = True
def generateDepTreeEvent(self, command, params):
"""
@ -168,6 +182,7 @@ class CommandsAsync:
command.cooker.generateDepTreeEvent(pkgs_to_build, task)
command.finishAsyncCommand()
generateDepTreeEvent.needcache = True
def generateDotGraph(self, command, params):
"""
@ -178,6 +193,7 @@ class CommandsAsync:
command.cooker.generateDotGraphFiles(pkgs_to_build, task)
command.finishAsyncCommand()
generateDotGraph.needcache = True
def showVersions(self, command, params):
"""
@ -185,6 +201,7 @@ class CommandsAsync:
"""
command.cooker.showVersions()
command.finishAsyncCommand()
showVersions.needcache = True
def showEnvironment(self, command, params):
"""
@ -195,6 +212,7 @@ class CommandsAsync:
command.cooker.showEnvironment(bfile, pkg)
command.finishAsyncCommand()
showEnvironment.needcache = True
def parseFiles(self, command, params):
"""
@ -202,6 +220,7 @@ class CommandsAsync:
"""
command.cooker.updateCache()
command.finishAsyncCommand()
parseFiles.needcache = True
#
# Events

191
bitbake-dev/lib/bb/cooker.py
View File

@ -46,7 +46,8 @@ class NothingToBuild(Exception):
# Different states cooker can be in
cookerClean = 1
cookerParsed = 2
cookerParsing = 2
cookerParsed = 3
# Different action states the cooker can be in
cookerRun = 1 # Cooker is running normally
@ -116,10 +117,8 @@ class BBCooker:
termios.tcsetattr(fd, termios.TCSANOW, tcattr)
self.command = bb.command.Command(self)
self.cookerIdle = True
self.cookerState = cookerClean
self.cookerAction = cookerRun
self.server.register_idle_function(self.runCommands, self)
def parseConfiguration(self):
@ -172,11 +171,8 @@ class BBCooker:
This is done by the idle handler so it runs in true context rather than
tied to any UI.
"""
if self.cookerIdle and not abort:
self.command.runAsyncCommand()
# Always reschedule
return True
return self.command.runAsyncCommand()
def tryBuildPackage(self, fn, item, task, the_data):
"""
@ -675,12 +671,11 @@ class BBCooker:
failures = failures + 1
retval = False
if not retval:
self.cookerIdle = True
self.command.finishAsyncCommand()
bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
return retval
return False
return 0.5
self.cookerIdle = False
self.server.register_idle_function(buildFileIdle, rq)
def buildTargets(self, targets, task):
@ -712,10 +707,10 @@ class BBCooker:
failures = failures + 1
retval = False
if not retval:
self.cookerIdle = True
self.command.finishAsyncCommand()
bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
return retval
return None
return 0.5
self.buildSetVars()
@ -736,47 +731,54 @@ class BBCooker:
rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
self.cookerIdle = False
self.server.register_idle_function(buildTargetsIdle, rq)
def updateCache(self):
self.parseConfiguration ()
if self.cookerState == cookerParsed:
return
# Import Psyco if available and not disabled
import platform
if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
if not self.configuration.disable_psyco:
try:
import psyco
except ImportError:
bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
if self.cookerState != cookerParsing:
self.parseConfiguration ()
# Import Psyco if available and not disabled
import platform
if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
if not self.configuration.disable_psyco:
try:
import psyco
except ImportError:
bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
else:
psyco.bind( CookerParser.parse_next )
else:
psyco.bind( self.parse_bbfiles )
else:
bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
self.status = bb.cache.CacheData()
self.status = bb.cache.CacheData()
ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
self.status.ignored_dependencies = set(ignore.split())
ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
self.status.ignored_dependencies = set(ignore.split())
for dep in self.configuration.extra_assume_provided:
self.status.ignored_dependencies.add(dep)
self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
for dep in self.configuration.extra_assume_provided:
self.status.ignored_dependencies.add(dep)
bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
(filelist, masked) = self.collect_bbfiles()
bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
self.parser = CookerParser(self, filelist, masked)
self.cookerState = cookerParsing
bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
(filelist, masked) = self.collect_bbfiles()
bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
self.parse_bbfiles(filelist, masked)
bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
if not self.parser.parse_next():
bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
self.buildDepgraph()
self.cookerState = cookerParsed
return None
self.buildDepgraph()
self.cookerState = cookerParsed
return 0.00001
def checkPackages(self, pkgs_to_build):
@ -861,57 +863,6 @@ class BBCooker:
return (finalfiles, masked)
def parse_bbfiles(self, filelist, masked):
parsed, cached, skipped, error, total = 0, 0, 0, 0, len(filelist)
for i in xrange(total):
f = filelist[i]
#bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
# read a file's metadata
try:
fromCache, skip = self.bb_cache.loadData(f, self.configuration.data, self.status)
if skip:
skipped += 1
bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
self.bb_cache.skip(f)
continue
elif fromCache: cached += 1
else: parsed += 1
# Disabled by RP as was no longer functional
# allow metadata files to add items to BBFILES
#data.update_data(self.pkgdata[f])
#addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
#if addbbfiles:
# for aof in addbbfiles.split():
# if not files.count(aof):
# if not os.path.isabs(aof):
# aof = os.path.join(os.path.dirname(f),aof)
# files.append(aof)
except IOError, e:
error += 1
self.bb_cache.remove(f)
bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
pass
except KeyboardInterrupt:
self.bb_cache.sync()
raise
except Exception, e:
error += 1
self.bb_cache.remove(f)
bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
except:
self.bb_cache.remove(f)
raise
finally:
bb.event.fire(bb.event.ParseProgress(self.configuration.event_data, cached, parsed, skipped, masked, error, total))
self.bb_cache.sync()
if error > 0:
raise ParsingErrorsFound
def serve(self):
# Empty the environment. The environment will be populated as
@ -955,3 +906,63 @@ class CookerExit(bb.event.Event):
def __init__(self, d):
bb.event.Event.__init__(self, d)
class CookerParser:
def __init__(self, cooker, filelist, masked):
# Internal data
self.filelist = filelist
self.cooker = cooker
# Accounting statistics
self.parsed = 0
self.cached = 0
self.skipped = 0
self.error = 0
self.masked = masked
self.total = len(filelist)
# Pointer to the next file to parse
self.pointer = 0
def parse_next(self):
print "Pointer %d" % self.pointer
f = self.filelist[self.pointer]
cooker = self.cooker
try:
fromCache, skip = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
if skip:
self.skipped += 1
bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
cooker.bb_cache.skip(f)
elif fromCache: self.cached += 1
else: self.parsed += 1
except IOError, e:
self.error += 1
cooker.bb_cache.remove(f)
bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
pass
except KeyboardInterrupt:
cooker.bb_cache.remove(f)
cooker.bb_cache.sync()
raise
except Exception, e:
self.error += 1
cooker.bb_cache.remove(f)
bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
except:
cooker.bb_cache.remove(f)
raise
finally:
bb.event.fire(bb.event.ParseProgress(cooker.configuration.event_data, self.cached, self.parsed, self.skipped, self.masked, self.error, self.total))
self.pointer += 1
if self.pointer >= self.total:
cooker.bb_cache.sync()
if self.error > 0:
raise ParsingErrorsFound
return False
return True