diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index ab4658bc3b..ccb60de59c 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py @@ -185,7 +185,18 @@ def go(d, urls = None): pass bb.utils.unlockfile(lf) continue - m.go(u, ud, d) + + # First try fetching uri, u, from PREMIRRORS + mirrors = [ i.split() for i in (bb.data.getVar('PREMIRRORS', d, 1) or "").split('\n') if i ] + if not try_mirrors(d, u, mirrors): + # Next try fetching from the original uri, u + try: + m.go(u, ud, d) + except: + # Finally, try fetching uri, u, from MIRRORS + mirrors = [ i.split() for i in (bb.data.getVar('MIRRORS', d, 1) or "").split('\n') if i ] + try_mirrors (d, u, mirrors) + if ud.localfile: if not m.forcefetch(u, ud, d): Fetch.write_md5sum(u, ud, d) @@ -332,6 +343,44 @@ def runfetchcmd(cmd, d, quiet = False): return output +def try_mirrors(d, uri, mirrors): + """ + Try to use a mirrored version of the sources. + This method will be automatically called before the fetchers go. + + d Is a bb.data instance + uri is the original uri we're trying to download + mirrors is the list of mirrors we're going to try + """ + fpath = os.path.join(data.getVar("DL_DIR", d, 1), os.path.basename(uri)) + if os.access(fpath, os.R_OK): + bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % fpath) + return True + + ld = d.createCopy() + for (find, replace) in mirrors: + newuri = uri_replace(uri, find, replace, ld) + if newuri != uri: + try: + ud = FetchData(newuri, ld) + except bb.fetch.NoMethodError: + bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url) + continue + + ud.setup_localpath(ld) + + try: + ud.method.go(newuri, ud, ld) + return True + except (bb.fetch.MissingParameterError, + bb.fetch.FetchError, + bb.fetch.MD5SumError): + import sys + (type, value, traceback) = sys.exc_info() + bb.msg.debug(2, bb.msg.domain.Fetcher, "Mirror fetch failure: %s" % value) + return False + + class FetchData(object): """ A class which represents the fetcher state for a given URI. @@ -489,49 +538,6 @@ class Fetch(object): localcount_internal_helper = staticmethod(localcount_internal_helper) - def try_mirror(d, tarfn): - """ - Try to use a mirrored version of the sources. We do this - to avoid massive loads on foreign cvs and svn servers. - This method will be used by the different fetcher - implementations. - - d Is a bb.data instance - tarfn is the name of the tarball - """ - tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn) - if os.access(tarpath, os.R_OK): - bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn) - return True - - pn = data.getVar('PN', d, True) - src_tarball_stash = None - if pn: - src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split() - - ld = d.createCopy() - for stash in src_tarball_stash: - url = stash + tarfn - try: - ud = FetchData(url, ld) - except bb.fetch.NoMethodError: - bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url) - continue - - ud.setup_localpath(ld) - - try: - ud.method.go(url, ud, ld) - return True - except (bb.fetch.MissingParameterError, - bb.fetch.FetchError, - bb.fetch.MD5SumError): - import sys - (type, value, traceback) = sys.exc_info() - bb.msg.debug(2, bb.msg.domain.Fetcher, "Tarball stash fetch failure: %s" % value) - return False - try_mirror = staticmethod(try_mirror) - def verify_md5sum(ud, got_sum): """ Verify the md5sum we wanted with the one we got diff --git a/bitbake/lib/bb/fetch/bzr.py b/bitbake/lib/bb/fetch/bzr.py index b27fb63d07..c6e33c3343 100644 --- a/bitbake/lib/bb/fetch/bzr.py +++ b/bitbake/lib/bb/fetch/bzr.py @@ -91,11 +91,6 @@ class Bzr(Fetch): def go(self, loc, ud, d): """Fetch url""" - # try to use the tarball stash - if Fetch.try_mirror(d, ud.localfile): - bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping bzr checkout." % ud.localpath) - return - if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): bzrcmd = self._buildbzrcommand(ud, d, "update") bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc) diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py index 90a006500e..443f521317 100644 --- a/bitbake/lib/bb/fetch/cvs.py +++ b/bitbake/lib/bb/fetch/cvs.py @@ -77,11 +77,6 @@ class Cvs(Fetch): def go(self, loc, ud, d): - # try to use the tarball stash - if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile): - bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath) - return - method = "pserver" if "method" in ud.parm: method = ud.parm["method"] diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py index 9430582d55..10b396bb00 100644 --- a/bitbake/lib/bb/fetch/git.py +++ b/bitbake/lib/bb/fetch/git.py @@ -82,10 +82,6 @@ class Git(Fetch): def go(self, loc, ud, d): """Fetch url""" - if Fetch.try_mirror(d, ud.localfile): - bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath) - return - if ud.user: username = ud.user + '@' else: @@ -97,11 +93,12 @@ class Git(Fetch): codir = os.path.join(ud.clonedir, coname) if not os.path.exists(ud.clonedir): - if Fetch.try_mirror(d, ud.mirrortarball): + try: + Fetch.try_mirrors(ud.mirrortarball) bb.mkdirhier(ud.clonedir) os.chdir(ud.clonedir) runfetchcmd("tar -xzf %s" % (repofile), d) - else: + except: runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.clonedir), d) os.chdir(ud.clonedir) diff --git a/bitbake/lib/bb/fetch/hg.py b/bitbake/lib/bb/fetch/hg.py index 08cb61fc28..d0756382f8 100644 --- a/bitbake/lib/bb/fetch/hg.py +++ b/bitbake/lib/bb/fetch/hg.py @@ -116,11 +116,6 @@ class Hg(Fetch): def go(self, loc, ud, d): """Fetch url""" - # try to use the tarball stash - if Fetch.try_mirror(d, ud.localfile): - bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath) - return - bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): diff --git a/bitbake/lib/bb/fetch/osc.py b/bitbake/lib/bb/fetch/osc.py index 2c34caf6c9..548dd9d074 100644 --- a/bitbake/lib/bb/fetch/osc.py +++ b/bitbake/lib/bb/fetch/osc.py @@ -91,11 +91,6 @@ class Osc(Fetch): Fetch url """ - # Try to use the tarball stash - if Fetch.try_mirror(d, ud.localfile): - bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping osc checkout." % ud.localpath) - return - bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK): diff --git a/bitbake/lib/bb/fetch/perforce.py b/bitbake/lib/bb/fetch/perforce.py index 394f5a2253..e2c3421089 100644 --- a/bitbake/lib/bb/fetch/perforce.py +++ b/bitbake/lib/bb/fetch/perforce.py @@ -124,11 +124,6 @@ class Perforce(Fetch): Fetch urls """ - # try to use the tarball stash - if Fetch.try_mirror(d, ud.localfile): - bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath) - return - (host,depot,user,pswd,parm) = Perforce.doparse(loc, d) if depot.find('/...') != -1: diff --git a/bitbake/lib/bb/fetch/svk.py b/bitbake/lib/bb/fetch/svk.py index 120dad9d4e..a17ac04d21 100644 --- a/bitbake/lib/bb/fetch/svk.py +++ b/bitbake/lib/bb/fetch/svk.py @@ -62,9 +62,6 @@ class Svk(Fetch): def go(self, loc, ud, d): """Fetch urls""" - if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile): - return - svkroot = ud.host + ud.path svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module) diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py index eef9862a84..ba9f6ab109 100644 --- a/bitbake/lib/bb/fetch/svn.py +++ b/bitbake/lib/bb/fetch/svn.py @@ -136,11 +136,6 @@ class Svn(Fetch): def go(self, loc, ud, d): """Fetch url""" - # try to use the tarball stash - if Fetch.try_mirror(d, ud.localfile): - bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath) - return - bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py index fd93c7ec46..ae1c6ad136 100644 --- a/bitbake/lib/bb/fetch/wget.py +++ b/bitbake/lib/bb/fetch/wget.py @@ -30,7 +30,6 @@ import bb from bb import data from bb.fetch import Fetch from bb.fetch import FetchError -from bb.fetch import uri_replace class Wget(Fetch): """Class to fetch urls via 'wget'""" @@ -105,24 +104,9 @@ class Wget(Fetch): data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) data.update_data(localdata) - premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ] - for (find, replace) in premirrors: - newuri = uri_replace(uri, find, replace, d) - if newuri != uri: - if fetch_uri(newuri, ud, localdata): - return True - if fetch_uri(uri, ud, localdata): return True - # try mirrors - mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ] - for (find, replace) in mirrors: - newuri = uri_replace(uri, find, replace, d) - if newuri != uri: - if fetch_uri(newuri, ud, localdata): - return True - raise FetchError(uri)