Update bitbake to latest bitbake svn

git-svn-id: https://svn.o-hand.com/repos/poky/trunk@262 311d38ba-8fff-0310-9ca6-ca027cbcb966

bitbake/ChangeLog

@@ -1,3 +1,20 @@
+Changes in BitBake 1.3.?:
+	- Create a new Fetcher module to ease the
+	  development of new Fetchers.
+	  Issue #438 fixed by rpurdie@openedhand.com
+	- Make the Subversion fetcher honor the SRC Date
+	  (CVSDATE).
+	  Issue #555 fixed by chris@openedhand.com
+	- Expand PREFERRED_PROVIDER properly
+	  Issue #436 fixed by rprudie@openedhand.com
+	- Typo fix for Issue #531 Philipp Zabel for the
+	  BitBake Shell
+	- Introduce a new special variable SRCDATE as
+	  a generic naming to take over CVSDATE.
+	- Introduce a new keyword 'required' In contrast
+	  to include parsing will fail if a to be included
+	  can not be found.
+
 Changes in BitBake 1.3.2:
 	- reintegration of make.py into BitBake
 	- bbread is gone, use bitbake -e

bitbake/MANIFEST

@@ -9,7 +9,13 @@ lib/bb/build.py
 lib/bb/data.py
 lib/bb/data_smart.py
 lib/bb/event.py
-lib/bb/fetch.py
+lib/bb/fetch/bk.py
+lib/bb/fetch/cvs.py
+lib/bb/fetch/git.py
+lib/bb/fetch/__init__.py
+lib/bb/fetch/local.py
+lib/bb/fetch/svn.py
+lib/bb/fetch/wget.py
 lib/bb/manifest.py
 lib/bb/parse/__init__.py
 lib/bb/parse/parse_py/BBHandler.py

bitbake/bin/bitbake

@@ -46,9 +46,12 @@ class BBParsingStatus:
     def __init__(self):
         self.cache_dirty = False
         self.providers   = {}
+        self.rproviders = {}
+        self.packages = {}
+        self.packages_dynamic = {}
         self.bbfile_priority = {}
         self.bbfile_config_priorities = []
-        self.ignored_depedencies = None
+        self.ignored_dependencies = None
         self.possible_world = []
         self.world_target = Set()
         self.pkg_pn = {}
@@ -74,7 +77,10 @@ class BBParsingStatus:
         pr       = bb.data.getVar('PR', bb_data, True)
         dp       = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
         provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
+        rprovides = (bb.data.getVar("RPROVIDES", bb_data, 1) or "").split()
         depends  = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
+        packages = (bb.data.getVar('PACKAGES', bb_data, True) or "").split()
+        packages_dynamic = (bb.data.getVar('PACKAGES_DYNAMIC', bb_data, True) or "").split()
 
 
         # build PackageName to FileName lookup table
@@ -102,6 +108,24 @@ class BBParsingStatus:
         for dep in depends:
             self.all_depends.add(dep)
 
+        # Build reverse hash for PACKAGES, so runtime dependencies 
+        # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
+    
+        for package in packages:
+            if not package in self.packages:
+                self.packages[package] = []
+            self.packages[package].append(file_name)
+
+        for package in packages_dynamic:
+            if not package in self.packages_dynamic:
+                self.packages_dynamic[package] = []
+            self.packages_dynamic[package].append(file_name)
+
+        for rprovide in rprovides:
+            if not rprovide in self.rproviders:
+                self.rproviders[rprovide] = []
+            self.rproviders[rprovide].append(file_name)
+
         # Collect files we may need for possible world-dep
         # calculations
         if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
@@ -158,6 +182,7 @@ class BBCooker:
     def __init__( self ):
         self.build_cache_fail = []
         self.build_cache = []
+        self.rbuild_cache = []
         self.building_list = []
         self.build_path = []
         self.consider_msgs_cache = []
@@ -194,14 +219,29 @@ class BBCooker:
             self.build_cache_fail.append(fn)
             raise
 
-    def tryBuild( self, fn, virtual ):
+    def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []):
-        """Build a provider and its dependencies"""
+        """
-        if fn in self.building_list:
+        Build a provider and its dependencies. 
+        build_depends is a list of previous build dependencies (not runtime)
+        If build_depends is empty, we're dealing with a runtime depends
+        """
+
+        the_data = self.pkgdata[fn]
+
+        if not buildAllDeps:
+            buildAllDeps = bb.data.getVar('BUILD_ALL_DEPS', the_data, True) or False
+
+        # Error on build time dependency loops
+        if build_depends and build_depends.count(fn) > 1:
             bb.error("%s depends on itself (eventually)" % fn)
             bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
             return False
 
-        the_data = self.pkgdata[fn]
+        # See if this is a runtime dependency we've already built
+	# Or a build dependency being handled in a different build chain
+        if fn in self.building_list:
+            return self.addRunDeps(fn, virtual , buildAllDeps)
+
         item = self.status.pkg_fn[fn]
 
         self.building_list.append(fn)
@@ -209,7 +249,8 @@ class BBCooker:
         pathstr = "%s (%s)" % (item, virtual)
         self.build_path.append(pathstr)
 
-        depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
+        depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
+
         if self.configuration.verbose:
             bb.note("current path: %s" % (" -> ".join(self.build_path)))
             bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
@@ -234,7 +275,7 @@ class BBCooker:
                     continue
                 if not depcmd:
                     continue
-                if self.buildProvider( dependency ) == 0:
+                if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
                     bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
                     failed = True
                     if self.configuration.abort:
@@ -247,6 +288,9 @@ class BBCooker:
                 self.stats.deps += 1
                 return False
 
+            if not self.addRunDeps(fn, virtual , buildAllDeps):
+                return False
+
             if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
                 self.build_cache.append(fn)
                 return True
@@ -285,7 +329,7 @@ class BBCooker:
         bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
         bb.data.update_data(localdata)
 
-        preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, 1)
+        preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
         if preferred_v:
             m = re.match('(.*)_(.*)', preferred_v)
             if m:
@@ -379,28 +423,17 @@ class BBCooker:
             if data.getVarFlag( e, 'python', self.configuration.data ):
                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
 
-    def buildProvider( self, item ):
+    def filterProviders(self, providers, item):
-        fn = None
+        """
-
+        Take a list of providers and filter/reorder according to the 
-        discriminated = False
+        environment variables and previous build results
-
+        """
-        if item not in self.status.providers:
-            bb.error("Nothing provides %s" % item)
-            return 0
-
-        all_p = self.status.providers[item]
-
-        for p in all_p:
-            if p in self.build_cache:
-                bb.debug(1, "already built %s in this run\n" % p)
-                return 1
-
         eligible = []
         preferred_versions = {}
 
         # Collate providers by PN
         pkg_pn = {}
-        for p in all_p:
+        for p in providers:
             pn = self.status.pkg_fn[p]
             if pn not in pkg_pn:
                 pkg_pn[pn] = []
@@ -423,7 +456,7 @@ class BBCooker:
 
         # look to see if one of them is already staged, or marked as preferred.
         # if so, bump it to the head of the queue
-        for p in all_p:
+        for p in providers:
             the_data = self.pkgdata[p]
             pn = bb.data.getVar('PN', the_data, 1)
             pv = bb.data.getVar('PV', the_data, 1)
@@ -448,6 +481,33 @@ class BBCooker:
                 discriminated = True
                 break
 
+        return eligible
+
+    def buildProvider( self, item , buildAllDeps , build_depends = [] ):
+        """
+        Build something to provide a named build requirement
+        (takes item names from DEPENDS namespace)
+        """
+
+        fn = None
+        discriminated = False
+
+        if not item in self.status.providers:
+            bb.error("Nothing provides dependency %s" % item)
+            return 0
+
+        all_p = self.status.providers[item]
+
+        for p in all_p:
+            if p in self.build_cache:
+                bb.debug(1, "already built %s in this run\n" % p)
+                return 1
+
+        eligible = self.filterProviders(all_p, item)
+
+        if not eligible:
+            return 0
+
         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
         if prefervar:
             self.preferred[item] = prefervar
@@ -476,12 +536,140 @@ class BBCooker:
         # run through the list until we find one that we can build
         for fn in eligible:
             bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
-            if self.tryBuild(fn, item):
+            if self.tryBuild(fn, item, buildAllDeps, build_depends + [fn]):
                 return 1
 
         bb.note("no buildable providers for %s" % item)
         return 0
 
+    def buildRProvider( self, item , buildAllDeps ):
+        """
+        Build something to provide a named runtime requirement
+        (takes item names from RDEPENDS/PACKAGES namespace)
+        """
+
+        fn = None
+        all_p = []
+        discriminated = False
+
+        if not buildAllDeps:
+            return True
+
+        all_p = self.getProvidersRun(item)
+
+        if not all_p:
+            bb.error("Nothing provides runtime dependency %s" % (item))
+            return False
+
+        for p in all_p:
+            if p in self.rbuild_cache:
+                bb.debug(2, "Already built %s providing runtime %s\n" % (p,item))
+                return True
+            if p in self.build_cache:
+                bb.debug(2, "Already built %s but adding any further RDEPENDS for %s\n" % (p, item))
+                return self.addRunDeps(p, item , buildAllDeps)
+
+        eligible = self.filterProviders(all_p, item)
+        if not eligible:
+            return 0
+
+        preferred = []
+        for p in eligible:
+            pn = self.status.pkg_fn[p]
+            provides = self.status.pn_provides[pn]
+            for provide in provides:
+                prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, self.configuration.data, 1)
+                if prefervar == pn:
+                    if self.configuration.verbose:
+                        bb.note("selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
+                    eligible.remove(p)
+                    eligible = [p] + eligible
+                    preferred.append(p)
+
+        if len(eligible) > 1 and len(preferred) == 0:
+            if item not in self.consider_msgs_cache:
+                providers_list = []
+                for fn in eligible:
+                    providers_list.append(self.status.pkg_fn[fn])
+                bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
+                bb.note("consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
+            self.consider_msgs_cache.append(item)
+
+        if len(preferred) > 1:
+            if item not in self.consider_msgs_cache:
+                providers_list = []
+                for fn in preferred:
+                    providers_list.append(self.status.pkg_fn[fn])
+                bb.note("multiple preferred providers are available (%s);" % ", ".join(providers_list))
+                bb.note("consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
+            self.consider_msgs_cache.append(item)
+
+        # run through the list until we find one that we can build
+        for fn in eligible:
+            bb.debug(2, "selecting %s to satisfy runtime %s" % (fn, item))
+            if self.tryBuild(fn, item, buildAllDeps):
+                return True
+
+        bb.error("No buildable providers for runtime %s" % item)
+        return False
+
+    def getProvidersRun(self, rdepend):
+        """
+        Return any potential providers of runtime rdepend
+        """
+        rproviders = []
+
+        if rdepend in self.status.rproviders:
+            rproviders += self.status.rproviders[rdepend]
+
+        if rdepend in self.status.packages:
+            rproviders += self.status.packages[rdepend]
+
+        if rproviders:
+            return rproviders
+
+        # Only search dynamic packages if we can't find anything in other variables
+        for pattern in self.status.packages_dynamic:
+            regexp = re.compile(pattern)
+            if regexp.match(rdepend):
+                rproviders += self.status.packages_dynamic[pattern]
+
+        return rproviders
+
+    def addRunDeps(self , fn, item , buildAllDeps):
+        """
+        Add any runtime dependencies of runtime item provided by fn 
+        as long as item has't previously been processed by this function.
+        """
+
+        if item in self.rbuild_cache:
+            return True
+
+        if not buildAllDeps:
+            return True
+
+        rdepends = []
+        self.rbuild_cache.append(item)
+        the_data = self.pkgdata[fn]
+        pn = self.status.pkg_fn[fn]
+
+        if (item == pn):
+            rdepends += bb.utils.explode_deps(bb.data.getVar('RDEPENDS', the_data, True) or "")
+            rdepends += bb.utils.explode_deps(bb.data.getVar('RRECOMMENDS', the_data, True) or "")
+        else:
+            packages = (bb.data.getVar('PACKAGES', the_data, 1).split() or "")
+            for package in packages:
+                if package == item:
+                    rdepends += bb.utils.explode_deps(bb.data.getVar("RDEPENDS_%s" % package, the_data, True) or "")
+                    rdepends += bb.utils.explode_deps(bb.data.getVar("RRECOMMENDS_%s" % package, the_data, True) or "")
+
+        bb.debug(2, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends)))
+
+        for rdepend in rdepends:  
+            if not self.buildRProvider(rdepend, buildAllDeps):
+                return False
+        return True
+
     def buildDepgraph( self ):
         all_depends = self.status.all_depends
         pn_provides = self.status.pn_provides
@@ -694,7 +882,7 @@ class BBCooker:
             for k in pkgs_to_build:
                 failed = False
                 try:
-                    if self.buildProvider( k ) == 0:
+                    if self.buildProvider( k , False ) == 0:
                         # already diagnosed
                         failed = True
                 except bb.build.EventException:

bitbake/doc/manual/usermanual.xml

@@ -138,6 +138,12 @@ will be introduced.</para>
                 <title>Inclusion</title>
                 <para>Next, there is the <literal>include</literal> directive, which causes BitBake to parse in whatever file you specify, and insert it at that location, which is not unlike <command>make</command>.  However, if the path specified on the <literal>include</literal> line is a relative path, BitBake will locate the first one it can find within <envar>BBPATH</envar>.</para>
             </section>
+            <section>
+                <title>Requiring Inclusion</title>
+                <para>In contrast to the <literal>include</literal> directive, <literal>require</literal> will
+raise an ParseError if the to be included file can not be found. Otherwise it will behave just like the <literal>
+include</literal> directive.</para>
+            </section>
             <section>
                 <title>Python variable expansion</title>
                 <para><screen><varname>DATE</varname> = "${@time.strftime('%Y%m%d',time.gmtime())}"</screen></para>

bitbake/lib/bb/__init__.py

@@ -23,7 +23,7 @@ this program; if not, write to the Free Software Foundation, Inc., 59 Temple
 Place, Suite 330, Boston, MA 02111-1307 USA.
 """
 
-__version__ = "1.3.2"
+__version__ = "1.3.2.1"
 
 __all__ = [
 

bitbake/lib/bb/parse/__init__.py

@@ -26,6 +26,8 @@ __all__ = [ 'ParseError', 'SkipPackage', 'cached_mtime', 'mark_dependency',
             'supports', 'handle', 'init' ]
 handlers = []
 
+import bb, os
+
 class ParseError(Exception):
     """Exception raised when parsing fails"""
 
@@ -34,13 +36,14 @@ class SkipPackage(Exception):
 
 __mtime_cache = {}
 def cached_mtime(f):
-    import os
     if not __mtime_cache.has_key(f):
-        __mtime_cache[f] = os.stat(f)[8]
+        update_mtime(f)
     return __mtime_cache[f]
 
+def update_mtime(f):
+    __mtime_cache[f] = os.stat(f)[8]
+
 def mark_dependency(d, f):
-    import bb, os
     if f.startswith('./'):
         f = "%s/%s" % (os.getcwd(), f[2:])
     deps = (bb.data.getVar('__depends', d) or "").split()

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -28,6 +28,7 @@ from bb.parse import ParseError
 #__config_regexp__  = re.compile( r"(?P<exp>export\s*)?(?P<var>[a-zA-Z0-9\-_+.${}]+)\s*(?P<colon>:)?(?P<ques>\?)?=\s*(?P<apo>['\"]?)(?P<value>.*)(?P=apo)$")
 __config_regexp__  = re.compile( r"(?P<exp>export\s*)?(?P<var>[a-zA-Z0-9\-_+.${}/]+)(\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?\s*((?P<colon>:=)|(?P<ques>\?=)|(?P<append>\+=)|(?P<prepend>=\+)|(?P<predot>=\.)|(?P<postdot>\.=)|=)\s*(?P<apo>['\"]?)(?P<value>.*)(?P=apo)$")
 __include_regexp__ = re.compile( r"include\s+(.+)" )
+__require_regexp__ = re.compile( r"require\s+(.+)" )
 
 def init(data):
     if not bb.data.getVar('TOPDIR', data):
@@ -83,7 +84,11 @@ def obtain(fn, data = bb.data.init()):
     return localfn
 
 
-def include(oldfn, fn, data = bb.data.init()):
+def include(oldfn, fn, data = bb.data.init(), error_out = False):
+    """
+
+    error_out If True a ParseError will be reaised if the to be included
+    """
     if oldfn == fn: # prevent infinate recursion
         return None
 
@@ -93,8 +98,10 @@ def include(oldfn, fn, data = bb.data.init()):
 
     from bb.parse import handle
     try:
-        ret = handle(fn, data, 1)
+        ret = handle(fn, data, True)
     except IOError:
+        if error_out:
+            raise ParseError("Could not include required file %(fn)s" % vars() )
         debug(2, "CONF file '%s' not found" % fn)
 
 def handle(fn, data = bb.data.init(), include = 0):
@@ -125,7 +132,7 @@ def handle(fn, data = bb.data.init(), include = 0):
                 debug(1, "CONF %s %s" % (inc_string, currname))
                 break
         if f is None:
-            raise IOError("file not found")
+            raise IOError("file '%s' not found" % fn)
     else:
         f = open(fn,'r')
         debug(1, "CONF %s %s" % (inc_string,fn))
@@ -191,6 +198,12 @@ def feeder(lineno, s, fn, data = bb.data.init()):
         include(fn, s, data)
         return
 
+    m = __require_regexp__.match(s)
+    if m:
+        s = bb.data.expand(m.group(1), data)
+        include(fn, s, data, True)
+        return
+
     raise ParseError("%s:%d: unparsed line: '%s'" % (fn, lineno, s));
 
 # Add us to the handlers list

bitbake/lib/bb/shell.py

@@ -18,6 +18,12 @@
 # Place, Suite 330, Boston, MA 02111-1307 USA.
 #
 ##########################################################################
+#
+# Thanks to:
+# * Holger Freyther <zecke@handhelds.org>
+# * Justin Patrin <papercrane@reversefold.com>
+#
+##########################################################################
 
 """
 BitBake Shell
@@ -53,7 +59,7 @@ import sys, os, imp, readline, socket, httplib, urllib, commands, popen2, copy,
 imp.load_source( "bitbake", os.path.dirname( sys.argv[0] )+"/bitbake" )
 from bb import data, parse, build, fatal
 
-__version__ = "0.5.2"
+__version__ = "0.5.3"
 __credits__ = """BitBake Shell Version %s (C) 2005 Michael 'Mickey' Lauer <mickey@Vanille.de>
 Type 'help' for more information, press CTRL-D to exit.""" % __version__
 
@@ -151,7 +157,7 @@ class BitBakeShellCommands:
 
         for name in names:
             try:
-                cooker.buildProvider( name )
+                cooker.buildProvider( name, data.getVar("BUILD_ALL_DEPS", cooker.configuration.data, True) )
             except build.EventException, e:
                 print "ERROR: Couldn't build '%s'" % name
                 global last_exception
@@ -252,6 +258,19 @@ class BitBakeShellCommands:
         self.fileBuild( params )
     fileRebuild.usage = "<bbfile>"
 
+    def fileReparse( self, params ):
+        """(re)Parse a bb file"""
+        bbfile = params[0]
+        print "SHELL: Parsing '%s'" % bbfile
+        parse.update_mtime( bbfile )
+        bb_data, fromCache = cooker.load_bbfile( bbfile )
+        cooker.pkgdata[bbfile] = bb_data
+        if fromCache:
+            print "SHELL: File has not been updated, not reparsing"
+        else:
+            print "SHELL: Parsed"
+    fileReparse.usage = "<bbfile>"
+
     def force( self, params ):
         """Toggle force task execution flag (see bitbake -f)"""
         cooker.configuration.force = not cooker.configuration.force
@@ -391,6 +410,16 @@ SRC_URI = ""
         parsed = True
         print
 
+    def reparse( self, params ):
+        """(re)Parse a providee's bb file"""
+        bbfile = self._findProvider( params[0] )
+        if bbfile is not None:
+            print "SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] )
+            self.fileReparse( [ bbfile ] )
+        else:
+            print "ERROR: Nothing provides '%s'" % params[0]
+    reparse.usage = "<providee>"
+
     def getvar( self, params ):
         """Dump the contents of an outer BitBake environment variable"""
         var = params[0]

bitbake/lib/bb/utils.py

@@ -69,3 +69,27 @@ def vercmp(ta, tb):
     if (r == 0):
         r = vercmp_part(ra, rb)
     return r
+
+def explode_deps(s):
+    """
+    Take an RDEPENDS style string of format:
+    "DEPEND1 (optional version) DEPEND2 (optional version) ..."
+    and return a list of dependencies.
+    Version information is ignored.
+    """
+    r = []
+    l = s.split()
+    flag = False
+    for i in l:
+        if i[0] == '(':
+            flag = True
+            j = []
+        if flag:
+            j.append(i)
+        if i.endswith(')'):
+            flag = False
+            # Ignore version
+            #r[-1] += ' ' + ' '.join(j)
+        else:
+            r.append(i)
+    return r

bitbake/setup.py

@@ -55,7 +55,7 @@ setup(name='bitbake',
       long_description='BitBake is a simple tool for the execution of tasks. It is derived from Portage, which is the package management system used by the Gentoo Linux distribution. It is most commonly used to build packages, as it can easily use its rudamentary inheritence to abstract common operations, such as fetching sources, unpacking them, patching them, compiling them, and so on.  It is the basis of the OpenEmbedded project, which is being used for OpenZaurus, Familiar, and a number of other Linux distributions.',
       author='Chris Larson',
       author_email='clarson@elinux.org',
-      packages=['bb', 'bb.parse', 'bb.parse.parse_py'],
+      packages=['bb', 'bb.fetch', 'bb.parse', 'bb.parse.parse_py'],
       package_dir={'bb': os.path.join('lib', 'bb')},
       scripts=[os.path.join('bin', 'bitbake'),
                os.path.join('bin', 'bbimage')],