sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: unify mirror support and make it independant of the fetcher 

This patch serves two purposes. Firstly it unifies the concept of mirrors into
PREMIRRORS and MIRRORS. PREMIRRORS are tried before the SRC_URI defined in the
recipe whereas MIRRORS are tried only if that fails.
The tarball stash was conceptually inline with a PREMIRROR only with special
handling within the wget fetcher and therefore only worked with certain
fetch types.
Secondly the patch removes the need for individual fetch implementations to
worry about mirror handling.

With this patch, the base fetch implementation will first try to use a
PREMIRROR to fetch the desired object, if this fails the native fetch method
for the object will be tried and if this fails will try to fetch a copy from
one of the MIRRORS.

Signed-off-by: Joshua Lock <josh@linux.intel.com>
This commit is contained in:
Joshua Lock 2010-02-01 16:56:16 +00:00
parent 0b52537f5c
commit 0737552c1d
10 changed files with 53 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
bitbake/lib/bb/fetch/__init__.py
View File

@ -185,7 +185,18 @@ def go(d, urls = None):
pass
bb.utils.unlockfile(lf)
continue
m.go(u, ud, d)
# First try fetching uri, u, from PREMIRRORS
mirrors = [ i.split() for i in (bb.data.getVar('PREMIRRORS', d, 1) or "").split('\n') if i ]
if not try_mirrors(d, u, mirrors):
# Next try fetching from the original uri, u
try:
m.go(u, ud, d)
except:
# Finally, try fetching uri, u, from MIRRORS
mirrors = [ i.split() for i in (bb.data.getVar('MIRRORS', d, 1) or "").split('\n') if i ]
try_mirrors (d, u, mirrors)
if ud.localfile:
if not m.forcefetch(u, ud, d):
Fetch.write_md5sum(u, ud, d)
@ -332,6 +343,44 @@ def runfetchcmd(cmd, d, quiet = False):
return output
def try_mirrors(d, uri, mirrors):
"""
Try to use a mirrored version of the sources.
This method will be automatically called before the fetchers go.
d Is a bb.data instance
uri is the original uri we're trying to download
mirrors is the list of mirrors we're going to try
"""
fpath = os.path.join(data.getVar("DL_DIR", d, 1), os.path.basename(uri))
if os.access(fpath, os.R_OK):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % fpath)
return True
ld = d.createCopy()
for (find, replace) in mirrors:
newuri = uri_replace(uri, find, replace, ld)
if newuri != uri:
try:
ud = FetchData(newuri, ld)
except bb.fetch.NoMethodError:
bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
continue
ud.setup_localpath(ld)
try:
ud.method.go(newuri, ud, ld)
return True
except (bb.fetch.MissingParameterError,
bb.fetch.FetchError,
bb.fetch.MD5SumError):
import sys
(type, value, traceback) = sys.exc_info()
bb.msg.debug(2, bb.msg.domain.Fetcher, "Mirror fetch failure: %s" % value)
return False
class FetchData(object):
"""
A class which represents the fetcher state for a given URI.
@ -489,49 +538,6 @@ class Fetch(object):
localcount_internal_helper = staticmethod(localcount_internal_helper)
def try_mirror(d, tarfn):
"""
Try to use a mirrored version of the sources. We do this
to avoid massive loads on foreign cvs and svn servers.
This method will be used by the different fetcher
implementations.
d Is a bb.data instance
tarfn is the name of the tarball
"""
tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
if os.access(tarpath, os.R_OK):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
return True
pn = data.getVar('PN', d, True)
src_tarball_stash = None
if pn:
src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
ld = d.createCopy()
for stash in src_tarball_stash:
url = stash + tarfn
try:
ud = FetchData(url, ld)
except bb.fetch.NoMethodError:
bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
continue
ud.setup_localpath(ld)
try:
ud.method.go(url, ud, ld)
return True
except (bb.fetch.MissingParameterError,
bb.fetch.FetchError,
bb.fetch.MD5SumError):
import sys
(type, value, traceback) = sys.exc_info()
bb.msg.debug(2, bb.msg.domain.Fetcher, "Tarball stash fetch failure: %s" % value)
return False
try_mirror = staticmethod(try_mirror)
def verify_md5sum(ud, got_sum):
"""
Verify the md5sum we wanted with the one we got

5
bitbake/lib/bb/fetch/bzr.py
View File

@ -91,11 +91,6 @@ class Bzr(Fetch):
def go(self, loc, ud, d):
"""Fetch url"""
# try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping bzr checkout." % ud.localpath)
return
if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
bzrcmd = self._buildbzrcommand(ud, d, "update")
bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc)

5
bitbake/lib/bb/fetch/cvs.py
View File

@ -77,11 +77,6 @@ class Cvs(Fetch):
def go(self, loc, ud, d):
# try to use the tarball stash
if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath)
return
method = "pserver"
if "method" in ud.parm:
method = ud.parm["method"]

9
bitbake/lib/bb/fetch/git.py
View File

@ -82,10 +82,6 @@ class Git(Fetch):
def go(self, loc, ud, d):
"""Fetch url"""
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
return
if ud.user:
username = ud.user + '@'
else:
@ -97,11 +93,12 @@ class Git(Fetch):
codir = os.path.join(ud.clonedir, coname)
if not os.path.exists(ud.clonedir):
if Fetch.try_mirror(d, ud.mirrortarball):
try:
Fetch.try_mirrors(ud.mirrortarball)
bb.mkdirhier(ud.clonedir)
os.chdir(ud.clonedir)
runfetchcmd("tar -xzf %s" % (repofile), d)
else:
except:
runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.clonedir), d)
os.chdir(ud.clonedir)

5
bitbake/lib/bb/fetch/hg.py
View File

@ -116,11 +116,6 @@ class Hg(Fetch):
def go(self, loc, ud, d):
"""Fetch url"""
# try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath)
return
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):

5
bitbake/lib/bb/fetch/osc.py
View File

@ -91,11 +91,6 @@ class Osc(Fetch):
Fetch url
"""
# Try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping osc checkout." % ud.localpath)
return
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):

5
bitbake/lib/bb/fetch/perforce.py
View File

@ -124,11 +124,6 @@ class Perforce(Fetch):
Fetch urls
"""
# try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath)
return
(host,depot,user,pswd,parm) = Perforce.doparse(loc, d)
if depot.find('/...') != -1:

3
bitbake/lib/bb/fetch/svk.py
View File

@ -62,9 +62,6 @@ class Svk(Fetch):
def go(self, loc, ud, d):
"""Fetch urls"""
if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
return
svkroot = ud.host + ud.path
svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module)

5
bitbake/lib/bb/fetch/svn.py
View File

@ -136,11 +136,6 @@ class Svn(Fetch):
def go(self, loc, ud, d):
"""Fetch url"""
# try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath)
return
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):

16
bitbake/lib/bb/fetch/wget.py
View File

@ -30,7 +30,6 @@ import bb
from bb import data
from bb.fetch import Fetch
from bb.fetch import FetchError
from bb.fetch import uri_replace
class Wget(Fetch):
"""Class to fetch urls via 'wget'"""
@ -105,24 +104,9 @@ class Wget(Fetch):
data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
data.update_data(localdata)
premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
for (find, replace) in premirrors:
newuri = uri_replace(uri, find, replace, d)
if newuri != uri:
if fetch_uri(newuri, ud, localdata):
return True
if fetch_uri(uri, ud, localdata):
return True
# try mirrors
mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
for (find, replace) in mirrors:
newuri = uri_replace(uri, find, replace, d)
if newuri != uri:
if fetch_uri(newuri, ud, localdata):
return True
raise FetchError(uri)