bitbake-dev: Turn parsing into a server idle callback allowing the client to interrupt parsing and improving user interactvity. Also now specify whether async commands need the cache or not

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

bitbake-dev/lib/bb/command.py

@@ -57,16 +57,18 @@ class Command:
             async_cmds[command] = (method)
 
     def runCommand(self, commandline):
+        bb.debug("Running command %s" % commandline)
         try:
             command = commandline.pop(0)
             if command in CommandsSync.__dict__:
-                # Can run online commands straight away            
+                # Can run synchronous commands straight away            
                 return getattr(CommandsSync, command)(self.cmds_sync, self, commandline)
             if self.currentAsyncCommand is not None:
                 return "Busy (%s in progress)" % self.currentAsyncCommand[0]
             if command not in CommandsAsync.__dict__:
                 return "No such command"
             self.currentAsyncCommand = (command, commandline)
+            self.cooker.server.register_idle_function(self.cooker.runCommands, self.cooker)
             return True
         except:
             import traceback
@@ -76,10 +78,20 @@ class Command:
         try:
             if self.currentAsyncCommand is not None:
                 (command, options) = self.currentAsyncCommand
-                getattr(CommandsAsync, command)(self.cmds_async, self, options)
+                commandmethod = getattr(CommandsAsync, command)
+                needcache = getattr( commandmethod, "needcache" )
+                if needcache and self.cooker.cookerState != bb.cooker.cookerParsed:
+                    self.cooker.updateCache()
+                    return True
+                else:
+                    commandmethod(self.cmds_async, self, options)
+                    return False
+            else:
+                return False
         except:
             import traceback
             self.finishAsyncCommand(traceback.format_exc())
+            return False
 
     def finishAsyncCommand(self, error = None):
         if error:
@@ -149,6 +161,7 @@ class CommandsAsync:
         task = params[1]
 
         command.cooker.buildFile(bfile, task)
+    buildFile.needcache = False
 
     def buildTargets(self, command, params):
         """
@@ -158,6 +171,7 @@ class CommandsAsync:
         task = params[1]
 
         command.cooker.buildTargets(pkgs_to_build, task)
+    buildTargets.needcache = True
 
     def generateDepTreeEvent(self, command, params):
         """
@@ -168,6 +182,7 @@ class CommandsAsync:
 
         command.cooker.generateDepTreeEvent(pkgs_to_build, task)
         command.finishAsyncCommand()
+    generateDepTreeEvent.needcache = True
 
     def generateDotGraph(self, command, params):
         """
@@ -178,6 +193,7 @@ class CommandsAsync:
 
         command.cooker.generateDotGraphFiles(pkgs_to_build, task)
         command.finishAsyncCommand()
+    generateDotGraph.needcache = True
 
     def showVersions(self, command, params):
         """
@@ -185,6 +201,7 @@ class CommandsAsync:
         """
         command.cooker.showVersions()
         command.finishAsyncCommand()
+    showVersions.needcache = True
 
     def showEnvironment(self, command, params):
         """
@@ -195,6 +212,7 @@ class CommandsAsync:
 
         command.cooker.showEnvironment(bfile, pkg)
         command.finishAsyncCommand()
+    showEnvironment.needcache = True
 
     def parseFiles(self, command, params):
         """
@@ -202,6 +220,7 @@ class CommandsAsync:
         """
         command.cooker.updateCache()
         command.finishAsyncCommand()
+    parseFiles.needcache = True
 
 #
 # Events

bitbake-dev/lib/bb/cooker.py

@@ -46,7 +46,8 @@ class NothingToBuild(Exception):
 
 # Different states cooker can be in
 cookerClean = 1
-cookerParsed = 2
+cookerParsing = 2
+cookerParsed = 3
 
 # Different action states the cooker can be in
 cookerRun = 1           # Cooker is running normally
@@ -116,10 +117,8 @@ class BBCooker:
                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
 
         self.command = bb.command.Command(self)
-        self.cookerIdle = True
         self.cookerState = cookerClean
         self.cookerAction = cookerRun
-        self.server.register_idle_function(self.runCommands, self)
 
     def parseConfiguration(self):
 
@@ -172,11 +171,8 @@ class BBCooker:
         This is done by the idle handler so it runs in true context rather than
         tied to any UI.
         """
-        if self.cookerIdle and not abort:
-            self.command.runAsyncCommand()
 
-        # Always reschedule
+        return self.command.runAsyncCommand()
-        return True
 
     def tryBuildPackage(self, fn, item, task, the_data):
         """
@@ -675,12 +671,11 @@ class BBCooker:
                     failures = failures + 1
                 retval = False
             if not retval:
-                self.cookerIdle = True
                 self.command.finishAsyncCommand()
                 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
-            return retval
+                return False
+            return 0.5
 
-        self.cookerIdle = False
         self.server.register_idle_function(buildFileIdle, rq)
 
     def buildTargets(self, targets, task):
@@ -712,10 +707,10 @@ class BBCooker:
                     failures = failures + 1
                 retval = False
             if not retval:
-                self.cookerIdle = True
                 self.command.finishAsyncCommand()
                 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
-            return retval
+                return None
+            return 0.5
 
         self.buildSetVars()
 
@@ -736,47 +731,54 @@ class BBCooker:
 
         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
 
-        self.cookerIdle = False
         self.server.register_idle_function(buildTargetsIdle, rq)
 
     def updateCache(self):
 
-        self.parseConfiguration ()
         if self.cookerState == cookerParsed:
             return
 
-        # Import Psyco if available and not disabled
+        if self.cookerState != cookerParsing:
-        import platform
+
-        if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
+            self.parseConfiguration ()
-            if not self.configuration.disable_psyco:
+
-                try:
+            # Import Psyco if available and not disabled
-                    import psyco
+            import platform
-                except ImportError:
+            if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
-                    bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
+                if not self.configuration.disable_psyco:
+                    try:
+                        import psyco
+                    except ImportError:
+                        bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
+                    else:
+                        psyco.bind( CookerParser.parse_next )
                 else:
-                    psyco.bind( self.parse_bbfiles )
+                    bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
-            else:
-                bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
 
-        self.status = bb.cache.CacheData()
+            self.status = bb.cache.CacheData()
 
-        ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
+            ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
-        self.status.ignored_dependencies = set(ignore.split())
+            self.status.ignored_dependencies = set(ignore.split())
+    
+            for dep in self.configuration.extra_assume_provided:
+                self.status.ignored_dependencies.add(dep)
+    
+            self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
 
-        for dep in self.configuration.extra_assume_provided:
+            bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
-            self.status.ignored_dependencies.add(dep)
+            (filelist, masked) = self.collect_bbfiles()
+            bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
 
-        self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
+            self.parser = CookerParser(self, filelist, masked)
+            self.cookerState = cookerParsing
 
-        bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
+        if not self.parser.parse_next():
-        (filelist, masked) = self.collect_bbfiles()
+            bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
-        bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
+            self.buildDepgraph()
-        self.parse_bbfiles(filelist, masked)
+            self.cookerState = cookerParsed
-        bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
+            return None
 
-        self.buildDepgraph()
+        return 0.00001
-
-        self.cookerState = cookerParsed
 
     def checkPackages(self, pkgs_to_build):
 
@@ -861,57 +863,6 @@ class BBCooker:
 
         return (finalfiles, masked)
 
-    def parse_bbfiles(self, filelist, masked):
-        parsed, cached, skipped, error, total = 0, 0, 0, 0, len(filelist)
-        for i in xrange(total):
-            f = filelist[i]
-
-            #bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
-
-            # read a file's metadata
-            try:
-                fromCache, skip = self.bb_cache.loadData(f, self.configuration.data, self.status)
-                if skip:
-                    skipped += 1
-                    bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
-                    self.bb_cache.skip(f)
-                    continue
-                elif fromCache: cached += 1
-                else: parsed += 1
-
-                # Disabled by RP as was no longer functional
-                # allow metadata files to add items to BBFILES
-                #data.update_data(self.pkgdata[f])
-                #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
-                #if addbbfiles:
-                #    for aof in addbbfiles.split():
-                #        if not files.count(aof):
-                #            if not os.path.isabs(aof):
-                #                aof = os.path.join(os.path.dirname(f),aof)
-                #            files.append(aof)
-
-            except IOError, e:
-                error += 1
-                self.bb_cache.remove(f)
-                bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
-                pass
-            except KeyboardInterrupt:
-                self.bb_cache.sync()
-                raise
-            except Exception, e:
-                error += 1
-                self.bb_cache.remove(f)
-                bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
-            except:
-                self.bb_cache.remove(f)
-                raise
-            finally:
-                bb.event.fire(bb.event.ParseProgress(self.configuration.event_data, cached, parsed, skipped, masked, error, total))
-
-        self.bb_cache.sync()
-        if error > 0:
-             raise ParsingErrorsFound
-
     def serve(self):
 
         # Empty the environment. The environment will be populated as
@@ -955,3 +906,63 @@ class CookerExit(bb.event.Event):
     def __init__(self, d):
         bb.event.Event.__init__(self, d)
 
+
+class CookerParser:
+    def __init__(self, cooker, filelist, masked):
+        # Internal data
+        self.filelist = filelist
+        self.cooker = cooker
+
+        # Accounting statistics
+        self.parsed = 0
+        self.cached = 0
+        self.skipped = 0
+        self.error = 0
+        self.masked = masked
+        self.total = len(filelist)
+
+        # Pointer to the next file to parse
+        self.pointer = 0
+
+    def parse_next(self):
+        print "Pointer %d" % self.pointer
+        f = self.filelist[self.pointer]
+        cooker = self.cooker
+
+        try:
+            fromCache, skip = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
+            if skip:
+                self.skipped += 1
+                bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
+                cooker.bb_cache.skip(f)
+            elif fromCache: self.cached += 1
+            else: self.parsed += 1
+
+        except IOError, e:
+            self.error += 1
+            cooker.bb_cache.remove(f)
+            bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
+            pass
+        except KeyboardInterrupt:
+            cooker.bb_cache.remove(f)
+            cooker.bb_cache.sync()
+            raise
+        except Exception, e:
+            self.error += 1
+            cooker.bb_cache.remove(f)
+            bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
+        except:
+            cooker.bb_cache.remove(f)
+            raise
+        finally:
+            bb.event.fire(bb.event.ParseProgress(cooker.configuration.event_data, self.cached, self.parsed, self.skipped, self.masked, self.error, self.total))
+
+        self.pointer += 1
+
+        if self.pointer >= self.total:
+            cooker.bb_cache.sync()
+            if self.error > 0:
+                raise ParsingErrorsFound
+            return False
+        return True
+