sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: runqueue: Change buildable/running lists to sets 

Using positions in lists for flags is an odd choice and makes the code
hard to maintain. Maintaining a list is slow since list searches are
slow (watch bitbake -n slow massively with it) but we can use a set()
instead.

This patch uses python sets to maintain the lists of tasks in each state
and this prepares for changing the task IDs from being integers.

(Bitbake rev: 8c1ed57f6ea475b714eca6673b48e8e5f5f0f9c3)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2016-06-12 23:55:08 +01:00
parent 7e54da0581
commit 29b4fb1796
1 changed files with 27 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
bitbake/lib/bb/runqueue.py
View File

@ -108,7 +108,7 @@ class RunQueueScheduler(object):
fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]]
taskname = self.rqdata.runq_task[taskid]
self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
if self.rq.runq_buildable[taskid] == 1:
if taskid in self.rq.runq_buildable:
self.buildable.append(taskid)
self.rev_prio_map = None
@ -117,7 +117,7 @@ class RunQueueScheduler(object):
"""
Return the id of the first task we find that is buildable
"""
self.buildable = [x for x in self.buildable if not self.rq.runq_running[x] == 1]
self.buildable = [x for x in self.buildable if x not in self.rq.runq_running]
if not self.buildable:
return None
if len(self.buildable) == 1:
@ -1361,9 +1361,9 @@ class RunQueueExecute:
self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1)
self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed"
self.runq_buildable = []
self.runq_running = []
self.runq_complete = []
self.runq_buildable = set()
self.runq_running = set()
self.runq_complete = set()
self.build_stamps = {}
self.build_stamps2 = []
@ -1464,12 +1464,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
# Mark initial buildable tasks
for task in range(self.stats.total):
self.runq_running.append(0)
self.runq_complete.append(0)
if len(self.rqdata.runq_depends[task]) == 0:
self.runq_buildable.append(1)
else:
self.runq_buildable.append(0)
self.runq_buildable.add(task)
if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered):
self.rq.scenequeue_covered.add(task)
@ -1574,7 +1570,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
return schedulers
def setbuildable(self, task):
self.runq_buildable[task] = 1
self.runq_buildable.add(task)
self.sched.newbuilable(task)
def task_completeoutright(self, task):
@ -1583,15 +1579,15 @@ class RunQueueExecuteTasks(RunQueueExecute):
Look at the reverse dependencies and mark any task with
completed dependencies as buildable
"""
self.runq_complete[task] = 1
self.runq_complete.add(task)
for revdep in self.rqdata.runq_revdeps[task]:
if self.runq_running[revdep] == 1:
if revdep in self.runq_running:
continue
if self.runq_buildable[revdep] == 1:
if revdep in self.runq_buildable:
continue
alldeps = 1
for dep in self.rqdata.runq_depends[revdep]:
if self.runq_complete[dep] != 1:
if dep not in self.runq_complete:
alldeps = 0
if alldeps == 1:
self.setbuildable(revdep)
@ -1617,7 +1613,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
self.rq.state = runQueueCleanUp
def task_skip(self, task, reason):
self.runq_running[task] = 1
self.runq_running.add(task)
self.setbuildable(task)
bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData)
self.task_completeoutright(task)
@ -1658,7 +1654,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
startevent = runQueueTaskStarted(task, self.stats, self.rq,
noexec=True)
bb.event.fire(startevent, self.cfgData)
self.runq_running[task] = 1
self.runq_running.add(task)
self.stats.taskActive()
if not self.cooker.configuration.dry_run:
bb.build.make_stamp(taskname, self.rqdata.dataCache, fn)
@ -1687,7 +1683,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
self.build_stamps2.append(self.build_stamps[task])
self.runq_running[task] = 1
self.runq_running.add(task)
self.stats.taskActive()
if self.stats.active < self.number_tasks:
return True
@ -1702,11 +1698,11 @@ class RunQueueExecuteTasks(RunQueueExecute):
# Sanity Checks
for task in range(self.stats.total):
if self.runq_buildable[task] == 0:
if task not in self.runq_buildable:
logger.error("Task %s never buildable!", task)
if self.runq_running[task] == 0:
if task not in self.runq_running:
logger.error("Task %s never ran!", task)
if self.runq_complete[task] == 0:
if task not in self.runq_complete:
logger.error("Task %s never completed!", task)
self.rq.state = runQueueComplete
@ -1759,11 +1755,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
# therefore aims to collapse the huge runqueue dependency tree into a smaller one
# only containing the setscene functions.
for task in range(self.stats.total):
self.runq_running.append(0)
self.runq_complete.append(0)
self.runq_buildable.append(0)
# First process the chains up to the first setscene task.
endpoints = {}
for task in range(len(self.rqdata.runq_fnid)):
@ -1894,7 +1885,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
for task in range(len(self.sq_revdeps)):
if len(self.sq_revdeps[task]) == 0:
self.runq_buildable[task] = 1
self.runq_buildable.add(task)
self.outrightfail = []
if self.rq.hashvalidate:
@ -1966,7 +1957,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
continue
self.sq_revdeps2[dep].remove(task)
if len(self.sq_revdeps2[dep]) == 0:
self.runq_buildable[dep] = 1
self.runq_buildable.add(dep)
def task_completeoutright(self, task):
"""
@ -1994,8 +1985,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.scenequeue_updatecounters(task, True)
def task_failoutright(self, task):
self.runq_running[task] = 1
self.runq_buildable[task] = 1
self.runq_running.add(task)
self.runq_buildable.add(task)
self.stats.taskCompleted()
self.stats.taskSkipped()
index = self.rqdata.runq_setscene[task]
@ -2003,8 +1994,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.scenequeue_updatecounters(task, True)
def task_skip(self, task):
self.runq_running[task] = 1
self.runq_buildable[task] = 1
self.runq_running.add(task)
self.runq_buildable.add(task)
self.task_completeoutright(task)
self.stats.taskCompleted()
self.stats.taskSkipped()
@ -2020,7 +2011,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
if self.stats.active < self.number_tasks:
# Find the next setscene to run
for nexttask in range(self.stats.total):
if self.runq_buildable[nexttask] == 1 and self.runq_running[nexttask] != 1:
if nexttask in self.runq_buildable and nexttask not in self.runq_running:
if nexttask in self.unskippable:
logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask]))
if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True):
@ -2077,7 +2068,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
self.rq.worker.stdin.flush()
self.runq_running[task] = 1
self.runq_running.add(task)
self.stats.taskActive()
if self.stats.active < self.number_tasks:
return True
@ -2087,8 +2078,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
return self.rq.active_fds()
#for task in range(self.stats.total):
# if self.runq_running[task] != 1:
# buildable = self.runq_buildable[task]
# if task not in self.runq_running:
# buildable = task in self.runq_buildable
# revdeps = self.sq_revdeps[task]
# bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task])))