bitbake: Update to latest bitbake-1.8 branch

git-svn-id: https://svn.o-hand.com/repos/poky/trunk@2651 311d38ba-8fff-0310-9ca6-ca027cbcb966

bitbake/ChangeLog

@@ -1,4 +1,13 @@
 Changes in BitBake 1.8.x:
+	- Psyco is available only for x86 - do not use it on other architectures.
+	- Fix a bug in bb.decodeurl where http://some.where.com/somefile.tgz decoded to host="" (#1530)
+	- Warn about malformed PREFERRED_PROVIDERS (#1072)
+	- Add support for BB_NICE_LEVEL option (#1627)
+	- Sort initial providers list by default preference (#1145, #2024)
+	- Improve provider sorting so prefered versions have preference over latest versions (#768)
+	- Detect builds of tasks with overlapping providers and warn (will become a fatal error) (#1359)
+	- Add MULTI_PROVIDER_WHITELIST variable to allow known safe multiple providers to be listed
+	- Handle paths in svn fetcher module parameter
 
 Changes in Bitbake 1.8.8:
 	- Rewrite svn fetcher to make adding extra operations easier 

bitbake/lib/bb/__init__.py

@@ -283,10 +283,11 @@ def decodeurl(url):
         raise MalformedUrl(url)
     user = m.group('user')
     parm = m.group('parm')
-    m = re.compile('(?P<host>[^/;]+)(?P<path>/[^;]+)').match(location)
+
-    if m:
+    locidx = location.find('/')
-        host = m.group('host')
+    if locidx != -1:
-        path = m.group('path')
+        host = location[:locidx]
+        path = location[locidx:]
     else:
         host = ""
         path = location

bitbake/lib/bb/cache.py

@@ -39,7 +39,7 @@ except ImportError:
     import pickle
     bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
 
-__cache_version__ = "126"
+__cache_version__ = "127"
 
 class Cache:
     """
@@ -286,10 +286,12 @@ class Cache:
             cacheData.pn_provides[pn] = Set()
         cacheData.pn_provides[pn] |= provides
 
+        cacheData.fn_provides[file_name] = Set()
         for provide in provides:
             if provide not in cacheData.providers:
                 cacheData.providers[provide] = []
             cacheData.providers[provide].append(file_name)
+            cacheData.fn_provides[file_name].add(provide)
 
         cacheData.deps[file_name] = Set()
         for dep in depends:
@@ -414,6 +416,7 @@ class CacheData:
         self.pkg_pepvpr = {}
         self.pkg_dp = {}
         self.pn_provides = {}
+        self.fn_provides = {}
         self.all_depends = Set()
         self.deps = {}
         self.rundeps = {}

bitbake/lib/bb/cooker.py

@@ -85,6 +85,13 @@ class BBCooker:
                 tcattr[3] = tcattr[3] & ~termios.TOSTOP
                 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
 
+        # Change nice level if we're asked to
+        nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
+        if nice:
+            curnice = os.nice(0)
+            nice = int(nice) - curnice
+            bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
+ 
 
     def tryBuildPackage(self, fn, item, task, the_data, build_depends):
         """
@@ -270,7 +277,11 @@ class BBCooker:
 
         # Handle PREFERRED_PROVIDERS
         for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
-            (providee, provider) = p.split(':')
+            try:
+                (providee, provider) = p.split(':')
+            except:
+                bb.msg.error(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
+                continue
             if providee in self.status.preferred and self.status.preferred[providee] != provider:
                 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
             self.status.preferred[providee] = provider

bitbake/lib/bb/fetch/svn.py

@@ -163,7 +163,7 @@ class Svn(Fetch):
         os.chdir(ud.pkgdir)
         # tar them up to a defined filename
         try:
-            runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)), d)
+            runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
         except:
             t, v, tb = sys.exc_info()
             try:

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -161,7 +161,7 @@ def handle(fn, d, include = 0):
                 if t:
                     data.setVar('T', t, d)
             except Exception, e:
-                bb.msg.debug(1, bb.msg.domain.Parsing, "executing anonymous function: %s" % e)
+                bb.msg.debug(1, bb.msg.domain.Parsing, "Exception when executing anonymous function: %s" % e)
                 raise
             data.delVar("__anonqueue", d)
             data.delVar("__anonfunc", d)

bitbake/lib/bb/providers.py

@@ -31,12 +31,12 @@ class NoProvider(Exception):
 class NoRProvider(Exception):
     """Exception raised when no provider of a runtime dependency can be found"""
 
-def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
+
+def sortPriorities(pn, dataCache, pkg_pn = None):
     """
-    If there is a PREFERRED_VERSION, find the highest-priority bbfile
+    Reorder pkg_pn by file priority and default preference
-    providing that version.  If not, find the latest version provided by
-    an bbfile in the highest-priority set.
     """
+
     if not pkg_pn:
         pkg_pn = dataCache.pkg_pn
 
@@ -44,14 +44,30 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     priorities = {}
     for f in files:
         priority = dataCache.bbfile_priority[f]
+        preference = dataCache.pkg_dp[f]
         if priority not in priorities:
-            priorities[priority] = []
+            priorities[priority] = {}
-        priorities[priority].append(f)
+        if preference not in priorities[priority]:
-    p_list = priorities.keys()
+            priorities[priority][preference] = []
-    p_list.sort(lambda a, b: a - b)
+        priorities[priority][preference].append(f)
+    pri_list = priorities.keys()
+    pri_list.sort(lambda a, b: a - b)
     tmp_pn = []
-    for p in p_list:
+    for pri in pri_list:
-        tmp_pn = [priorities[p]] + tmp_pn
+        pref_list = priorities[pri].keys()
+        pref_list.sort(lambda a, b: b - a)
+        tmp_pref = []
+        for pref in pref_list:
+            tmp_pref.extend(priorities[pri][pref])
+        tmp_pn = [tmp_pref] + tmp_pn
+
+    return tmp_pn
+
+
+def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
+    """
+    Find the first provider in pkg_pn with a PREFERRED_VERSION set.
+    """
 
     preferred_file = None
 
@@ -76,7 +92,7 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
             preferred_e = None
             preferred_r = None
 
-        for file_set in tmp_pn:
+        for file_set in pkg_pn:
             for f in file_set:
                 pe,pv,pr = dataCache.pkg_pepvpr[f]
                 if preferred_v == pv and (preferred_r == pr or preferred_r == None) and (preferred_e == pe or preferred_e == None):
@@ -99,14 +115,18 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
         else:
             bb.msg.debug(1, bb.msg.domain.Provider, "selecting %s as PREFERRED_VERSION %s of package %s%s" % (preferred_file, pv_str, pn, itemstr))
 
-    del localdata
+    return (preferred_v, preferred_file)
 
-    # get highest priority file set
+
-    files = tmp_pn[0]
+def findLatestProvider(pn, cfgData, dataCache, file_set):
+    """
+    Return the highest version of the providers in file_set.
+    Take default preferences into account.
+    """
     latest = None
     latest_p = 0
     latest_f = None
-    for file_name in files:
+    for file_name in file_set:
         pe,pv,pr = dataCache.pkg_pepvpr[file_name]
         dp = dataCache.pkg_dp[file_name]
 
@@ -114,11 +134,29 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
             latest = (pe, pv, pr)
             latest_f = file_name
             latest_p = dp
+
+    return (latest, latest_f)
+
+
+def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
+    """
+    If there is a PREFERRED_VERSION, find the highest-priority bbfile
+    providing that version.  If not, find the latest version provided by
+    an bbfile in the highest-priority set.
+    """
+
+    sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
+    # Find the highest priority provider with a PREFERRED_VERSION set
+    (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
+    # Find the latest version of the highest priority provider
+    (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
+
     if preferred_file is None:
         preferred_file = latest_f
         preferred_ver = latest
 
-    return (latest,latest_f,preferred_ver, preferred_file)
+    return (latest, latest_f, preferred_ver, preferred_file)
+
 
 def _filterProviders(providers, item, cfgData, dataCache):
     """
@@ -127,6 +165,7 @@ def _filterProviders(providers, item, cfgData, dataCache):
     """
     eligible = []
     preferred_versions = {}
+    sortpkg_pn = {}
 
     # The order of providers depends on the order of the files on the disk 
     # up to here. Sort pkg_pn to make dependency issues reproducible rather
@@ -143,15 +182,24 @@ def _filterProviders(providers, item, cfgData, dataCache):
 
     bb.msg.debug(1, bb.msg.domain.Provider, "providers for %s are: %s" % (item, pkg_pn.keys()))
 
+    # First add PREFERRED_VERSIONS
     for pn in pkg_pn.keys():
-        preferred_versions[pn] = bb.providers.findBestProvider(pn, cfgData, dataCache, pkg_pn, item)[2:4]
+        sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
+        preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        if preferred_versions[pn][1]:
+            eligible.append(preferred_versions[pn][1])
+
+    # Now add latest verisons
+    for pn in pkg_pn.keys():
+        if pn in preferred_versions and preferred_versions[pn][1]:
+            continue
+        preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
         eligible.append(preferred_versions[pn][1])
 
     if len(eligible) == 0:
         bb.msg.error(bb.msg.domain.Provider, "no eligible providers for %s" % item)
         return 0
 
-
     # If pn == item, give it a slight default preference
     # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
     for p in providers:
@@ -192,7 +240,7 @@ def _filterProviders(providers, item, cfgData, dataCache):
             eligible = [fn] + eligible
             break
 
-    return eligible, preferred_versions
+    return eligible
 
 
 def filterProviders(providers, item, cfgData, dataCache):
@@ -202,7 +250,7 @@ def filterProviders(providers, item, cfgData, dataCache):
     Takes a "normal" target item
     """
 
-    eligible, pref_vers = _filterProviders(providers, item, cfgData, dataCache)
+    eligible = _filterProviders(providers, item, cfgData, dataCache)
 
     prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
     if prefervar:
@@ -219,6 +267,8 @@ def filterProviders(providers, item, cfgData, dataCache):
                 foundUnique = True
                 break
 
+    bb.msg.debug(1, bb.msg.domain.Provider, "sorted providers for %s are: %s" % (item, eligible))
+
     return eligible, foundUnique
 
 def filterProvidersRunTime(providers, item, cfgData, dataCache):
@@ -228,7 +278,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
     Takes a "runtime" target item
     """
 
-    eligible, pref_vers = _filterProviders(providers, item, cfgData, dataCache)
+    eligible = _filterProviders(providers, item, cfgData, dataCache)
 
     # Should use dataCache.preferred here?
     preferred = []
@@ -246,6 +296,8 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
 
     numberPreferred = len(preferred)
 
+    bb.msg.debug(1, bb.msg.domain.Provider, "sorted providers for %s are: %s" % (item, eligible))
+
     return eligible, numberPreferred
 
 def getRuntimeProviders(dataCache, rdepend):

bitbake/lib/bb/runqueue.py

@@ -7,7 +7,7 @@ BitBake 'RunQueue' implementation
 Handles preparation and execution of a queue of tasks
 """
 
-# Copyright (C) 2006  Richard Purdie
+# Copyright (C) 2006-2007  Richard Purdie
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -63,6 +63,7 @@ class RunQueue:
         self.targets = targets
 
         self.number_tasks = int(bb.data.getVar("BB_NUMBER_THREADS", cfgData) or 1)
+        self.multi_provider_whitelist = (bb.data.getVar("MULTI_PROVIDER_WHITELIST", cfgData) or "").split()
 
     def reset_runqueue(self):
 
@@ -373,6 +374,29 @@ class RunQueue:
             if runq_weight1[task] != 0:
                 bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) count not zero!" % (task, self.get_user_idstring(task)))
 
+
+        # Check for mulitple taska building the same provider
+        prov_list = {}
+        seen_fn = []
+        for task in range(len(self.runq_fnid)):
+            fn = taskData.fn_index[self.runq_fnid[task]]
+            if fn in seen_fn:
+                continue
+            seen_fn.append(fn)
+            for prov in self.dataCache.fn_provides[fn]:
+                if prov not in prov_list:
+                    prov_list[prov] = [fn]
+                elif fn not in prov_list[prov]: 
+                    prov_list[prov].append(fn)
+        error = False
+        for prov in prov_list:
+            if len(prov_list[prov]) > 1 and prov not in self.multi_provider_whitelist:
+                error = True
+                bb.msg.error(bb.msg.domain.RunQueue, "Multiple files due to be built which all provide %s (%s)" % (prov, " ".join(prov_list[prov])))
+        #if error:
+        #    bb.msg.fatal(bb.msg.domain.RunQueue, "Corrupted metadata configuration detected, aborting...")
+
+
         # Make a weight sorted map
         from copy import deepcopy