bitbake: Add support for .bbappend files (see mailing lists for detais)

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

bitbake/lib/bb/cache.py

@@ -165,7 +165,7 @@ class Cache:
         #bb.msg.debug(2, bb.msg.domain.Cache, "realfn2virtual %s and %s to %s" % (realfn, cls, "virtual:" + cls + ":" + realfn))
         return "virtual:" + cls + ":" + realfn
 
-    def loadDataFull(self, virtualfn, cfgData):
+    def loadDataFull(self, virtualfn, appends, cfgData):
         """
         Return a complete set of data for fn.
         To do this, we need to parse the file.
@@ -175,10 +175,10 @@ class Cache:
 
         bb.msg.debug(1, bb.msg.domain.Cache, "Parsing %s (full)" % fn)
 
-        bb_data = self.load_bbfile(fn, cfgData)
+        bb_data = self.load_bbfile(fn, appends, cfgData)
         return bb_data[cls]
 
-    def loadData(self, fn, cfgData, cacheData):
+    def loadData(self, fn, appends, cfgData, cacheData):
         """
         Load a subset of data for fn.
         If the cached data is valid we do nothing,
@@ -206,7 +206,7 @@ class Cache:
 
         bb.msg.debug(1, bb.msg.domain.Cache, "Parsing %s" % fn)
 
-        bb_data = self.load_bbfile(fn, cfgData)
+        bb_data = self.load_bbfile(fn, appends, cfgData)
 
         for data in bb_data:
             virtualfn = self.realfn2virtual(fn, data)
@@ -439,7 +439,7 @@ class Cache:
         self.getVar('__BB_DONT_CACHE', file_name, True)
         self.getVar('__VARIANTS', file_name, True)
 
-    def load_bbfile( self, bbfile, config):
+    def load_bbfile(self, bbfile, appends, config):
         """
         Load and parse one .bb build file
         Return the data and whether parsing resulted in the file being skipped
@@ -463,6 +463,8 @@ class Cache:
             chdir_back = True
             data.setVar('TOPDIR', bbfile_loc, bb_data)
         try:
+            if appends:
+                data.setVar('__BBAPPEND', " ".join(appends), bb_data)
             bb_data = parse.handle(bbfile, bb_data) # read .bb data
             if chdir_back: os.chdir(oldpath)
             return bb_data

bitbake/lib/bb/cooker.py

@@ -226,7 +226,7 @@ class BBCooker:
 
         if fn:
             try:
-                envdata = self.bb_cache.loadDataFull(fn, self.configuration.data)
+                envdata = self.bb_cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data)
             except IOError as e:
                 bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
                 raise
@@ -637,7 +637,7 @@ class BBCooker:
         self.buildSetVars()
 
         # Load data into the cache for fn and parse the loaded cache data
-        the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
+        the_data = self.bb_cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data)
         self.bb_cache.setData(fn, buildfile, the_data)
         self.bb_cache.handle_data(fn, self.status)
 
@@ -778,7 +778,6 @@ class BBCooker:
 
             self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
 
-            bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
             (filelist, masked) = self.collect_bbfiles()
             bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
 
@@ -817,7 +816,7 @@ class BBCooker:
         return bbfiles
 
     def find_bbfiles( self, path ):
-        """Find all the .bb files in a directory"""
+        """Find all the .bb and .bbappend files in a directory"""
         from os.path import join
 
         found = []
@@ -825,7 +824,7 @@ class BBCooker:
             for ignored in ('SCCS', 'CVS', '.svn'):
                 if ignored in dirs:
                     dirs.remove(ignored)
-            found += [join(dir, f) for f in files if f.endswith('.bb')]
+            found += [join(dir, f) for f in files if (f.endswith('.bb') or f.endswith('.bbappend'))]
 
         return found
 
@@ -834,6 +833,8 @@ class BBCooker:
         parsed, cached, skipped, masked = 0, 0, 0, 0
         self.bb_cache = bb.cache.init(self)
 
+        bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
+
         files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
         data.setVar("BBFILES", " ".join(files), self.configuration.data)
 
@@ -848,9 +849,7 @@ class BBCooker:
         for f in files:
             if os.path.isdir(f):
                 dirfiles = self.find_bbfiles(f)
-                if dirfiles:
+                newfiles.update(dirfiles)
-                    newfiles.update(dirfiles)
-                    continue
             else:
                 globbed = glob.glob(f)
                 if not globbed and os.path.exists(f):
@@ -859,23 +858,45 @@ class BBCooker:
 
         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
 
-        if not bbmask:
+        if bbmask:
-            return (list(newfiles), 0)
+            try:
+                bbmask_compiled = re.compile(bbmask)
+            except sre_constants.error:
+                bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
 
-        try:
+        bbfiles = []
-            bbmask_compiled = re.compile(bbmask)
+        bbappend = []
-        except sre_constants.error:
-            bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
-
-        finalfiles = []
         for f in newfiles:
-            if bbmask_compiled.search(f):
+            if bbmask and bbmask_compiled.search(f):
                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
                 masked += 1
                 continue
-            finalfiles.append(f)
+            if f.endswith('.bb'):
+                bbfiles.append(f)
+            elif f.endswith('.bbappend'):
+                bbappend.append(f)
+            else:
+                bb.msg.note(1, bb.msg.domain.Collection, "File %s of unknown filetype in BBFILES? Ignorning..." % f)
 
-        return (finalfiles, masked)
+        # Build a list of .bbappend files for each .bb file
+        self.appendlist = {}
+        for f in bbappend:
+            base = os.path.basename(f).replace('.bbappend', '.bb')
+            if not base in self.appendlist:
+               self.appendlist[base] = []
+            self.appendlist[base].append(f)
+ 
+        return (bbfiles, masked)
+
+    def get_file_appends(self, fn):
+        """
+        Returns a list of .bbappend files to apply to fn
+        NB: collect_files() must have been called prior to this
+        """
+        f = os.path.basename(fn)
+        if f in self.appendlist:
+            return self.appendlist[f] 
+        return []
 
     def serve(self):
 
@@ -945,7 +966,7 @@ class CookerParser:
             f = self.filelist[self.pointer]
 
             try:
-                fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
+                fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.get_file_appends(f), cooker.configuration.data, cooker.status)
                 if fromCache:
                     self.cached += 1
                 else:

bitbake/lib/bb/parse/ast.py

@@ -357,8 +357,12 @@ def _expand_versions(versions):
             versions = itertools.chain(newversions, versions)
 
 def multi_finalize(fn, d):
-    safe_d = d
+    appends = (d.getVar("__BBAPPEND", True) or "").split()
+    for append in appends:
+        bb.msg.debug(2, bb.msg.domain.Parsing, "Appending .bbappend file " + append + " to " + fn)
+        bb.parse.BBHandler.handle(append, d, True)
 
+    safe_d = d
     d = bb.data.createCopy(safe_d)
     try:
         finalize(fn, d)

bitbake/lib/bb/runqueue.py

@@ -1067,7 +1067,7 @@ class RunQueue:
             bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", self, self.cooker.configuration.data)
             bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY2", fn, self.cooker.configuration.data)
             try:
-                the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.configuration.data)
+                the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data)
 
                 if not self.cooker.configuration.dry_run:
                     bb.build.exec_task(taskname, the_data)