289 lines
9.5 KiB
Python
289 lines
9.5 KiB
Python
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
"""
|
|
BitBake 'Fetch' implementations
|
|
|
|
Classes for obtaining upstream sources for the
|
|
BitBake build tools.
|
|
"""
|
|
|
|
# Copyright (C) 2003, 2004 Chris Larson
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
#
|
|
# Based on functions from the base bb module, Copyright 2003 Holger Schurig
|
|
|
|
import os, re
|
|
import bb
|
|
from bb import data
|
|
|
|
class FetchError(Exception):
|
|
"""Exception raised when a download fails"""
|
|
|
|
class NoMethodError(Exception):
|
|
"""Exception raised when there is no method to obtain a supplied url or set of urls"""
|
|
|
|
class MissingParameterError(Exception):
|
|
"""Exception raised when a fetch method is missing a critical parameter in the url"""
|
|
|
|
class ParameterError(Exception):
|
|
"""Exception raised when a url cannot be proccessed due to invalid parameters."""
|
|
|
|
class MD5SumError(Exception):
|
|
"""Exception raised when a MD5SUM of a file does not match the expected one"""
|
|
|
|
def uri_replace(uri, uri_find, uri_replace, d):
|
|
# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: operating on %s" % uri)
|
|
if not uri or not uri_find or not uri_replace:
|
|
bb.msg.debug(1, bb.msg.domain.Fetcher, "uri_replace: passed an undefined value, not replacing")
|
|
uri_decoded = list(bb.decodeurl(uri))
|
|
uri_find_decoded = list(bb.decodeurl(uri_find))
|
|
uri_replace_decoded = list(bb.decodeurl(uri_replace))
|
|
result_decoded = ['','','','','',{}]
|
|
for i in uri_find_decoded:
|
|
loc = uri_find_decoded.index(i)
|
|
result_decoded[loc] = uri_decoded[loc]
|
|
import types
|
|
if type(i) == types.StringType:
|
|
import re
|
|
if (re.match(i, uri_decoded[loc])):
|
|
result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
|
|
if uri_find_decoded.index(i) == 2:
|
|
if d:
|
|
localfn = bb.fetch.localpath(uri, d)
|
|
if localfn:
|
|
result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d))
|
|
# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
|
|
else:
|
|
# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: no match")
|
|
return uri
|
|
# else:
|
|
# for j in i.keys():
|
|
# FIXME: apply replacements against options
|
|
return bb.encodeurl(result_decoded)
|
|
|
|
methods = []
|
|
urldata = {}
|
|
|
|
def init(urls = [], d = None):
|
|
if d == None:
|
|
bb.msg.debug(2, bb.msg.domain.Fetcher, "BUG init called with None as data object!!!")
|
|
return
|
|
|
|
for m in methods:
|
|
m.urls = []
|
|
|
|
for u in urls:
|
|
ud = initdata(u, d)
|
|
if ud.method:
|
|
ud.method.urls.append(u)
|
|
|
|
def initdata(url, d):
|
|
fn = bb.data.getVar('FILE', d, 1)
|
|
if fn not in urldata:
|
|
urldata[fn] = {}
|
|
if url not in urldata[fn]:
|
|
ud = FetchData()
|
|
(ud.type, ud.host, ud.path, ud.user, ud.pswd, ud.parm) = bb.decodeurl(data.expand(url, d))
|
|
ud.date = Fetch.getSRCDate(ud, d)
|
|
for m in methods:
|
|
if m.supports(url, ud, d):
|
|
ud.localpath = m.localpath(url, ud, d)
|
|
ud.md5 = ud.localpath + '.md5'
|
|
# if user sets localpath for file, use it instead.
|
|
if "localpath" in ud.parm:
|
|
ud.localpath = ud.parm["localpath"]
|
|
ud.method = m
|
|
break
|
|
urldata[fn][url] = ud
|
|
return urldata[fn][url]
|
|
|
|
def go(d):
|
|
"""Fetch all urls"""
|
|
fn = bb.data.getVar('FILE', d, 1)
|
|
for m in methods:
|
|
for u in m.urls:
|
|
ud = urldata[fn][u]
|
|
if ud.localfile and not m.forcefetch(u, ud, d) and os.path.exists(urldata[fn][u].md5):
|
|
# File already present along with md5 stamp file
|
|
# Touch md5 file to show activity
|
|
os.utime(ud.md5, None)
|
|
continue
|
|
# RP - is olddir needed?
|
|
# olddir = os.path.abspath(os.getcwd())
|
|
m.go(u, ud , d)
|
|
# os.chdir(olddir)
|
|
if ud.localfile and not m.forcefetch(u, ud, d):
|
|
Fetch.write_md5sum(u, ud, d)
|
|
|
|
def localpaths(d):
|
|
"""Return a list of the local filenames, assuming successful fetch"""
|
|
local = []
|
|
fn = bb.data.getVar('FILE', d, 1)
|
|
for m in methods:
|
|
for u in m.urls:
|
|
local.append(urldata[fn][u].localpath)
|
|
return local
|
|
|
|
def localpath(url, d):
|
|
ud = initdata(url, d)
|
|
if ud.method:
|
|
return ud.localpath
|
|
return url
|
|
|
|
class FetchData(object):
|
|
"""Class for fetcher variable store"""
|
|
def __init__(self):
|
|
self.localfile = ""
|
|
|
|
|
|
class Fetch(object):
|
|
"""Base class for 'fetch'ing data"""
|
|
|
|
def __init__(self, urls = []):
|
|
self.urls = []
|
|
|
|
def supports(self, url, urldata, d):
|
|
"""
|
|
Check to see if this fetch class supports a given url.
|
|
"""
|
|
return 0
|
|
|
|
def localpath(self, url, urldata, d):
|
|
"""
|
|
Return the local filename of a given url assuming a successful fetch.
|
|
Can also setup variables in urldata for use in go (saving code duplication
|
|
and duplicate code execution)
|
|
"""
|
|
return url
|
|
|
|
def setUrls(self, urls):
|
|
self.__urls = urls
|
|
|
|
def getUrls(self):
|
|
return self.__urls
|
|
|
|
urls = property(getUrls, setUrls, None, "Urls property")
|
|
|
|
def forcefetch(self, url, urldata, d):
|
|
"""
|
|
Force a fetch, even if localpath exists?
|
|
"""
|
|
return False
|
|
|
|
def go(self, url, urldata, d):
|
|
"""
|
|
Fetch urls
|
|
Assumes localpath was called first
|
|
"""
|
|
raise NoMethodError("Missing implementation for url")
|
|
|
|
def getSRCDate(urldata, d):
|
|
"""
|
|
Return the SRC Date for the component
|
|
|
|
d the bb.data module
|
|
"""
|
|
if "srcdate" in urldata.parm:
|
|
return urldata.parm['srcdate']
|
|
|
|
pn = data.getVar("PN", d, 1)
|
|
|
|
if pn:
|
|
return data.getVar("SRCDATE_%s" % pn, d, 1) or data.getVar("CVSDATE_%s" % pn, d, 1) or data.getVar("DATE", d, 1)
|
|
|
|
return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
|
|
getSRCDate = staticmethod(getSRCDate)
|
|
|
|
def try_mirror(d, tarfn):
|
|
"""
|
|
Try to use a mirrored version of the sources. We do this
|
|
to avoid massive loads on foreign cvs and svn servers.
|
|
This method will be used by the different fetcher
|
|
implementations.
|
|
|
|
d Is a bb.data instance
|
|
tarfn is the name of the tarball
|
|
"""
|
|
tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
|
|
if os.access(tarpath, os.R_OK):
|
|
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
|
|
return True
|
|
|
|
pn = data.getVar('PN', d, True)
|
|
src_tarball_stash = None
|
|
if pn:
|
|
src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
|
|
|
|
for stash in src_tarball_stash:
|
|
fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True)
|
|
uri = stash + tarfn
|
|
bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
|
|
fetchcmd = fetchcmd.replace("${URI}", uri)
|
|
ret = os.system(fetchcmd)
|
|
if ret == 0:
|
|
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetched %s from tarball stash, skipping checkout" % tarfn)
|
|
return True
|
|
return False
|
|
try_mirror = staticmethod(try_mirror)
|
|
|
|
def verify_md5sum(ud, got_sum):
|
|
"""
|
|
Verify the md5sum we wanted with the one we got
|
|
"""
|
|
wanted_sum = None
|
|
if 'md5sum' in ud.parm:
|
|
wanted_sum = ud.parm['md5sum']
|
|
if not wanted_sum:
|
|
return True
|
|
|
|
return wanted_sum == got_sum
|
|
verify_md5sum = staticmethod(verify_md5sum)
|
|
|
|
def write_md5sum(url, ud, d):
|
|
if bb.which(data.getVar('PATH', d), 'md5sum'):
|
|
try:
|
|
md5pipe = os.popen('md5sum ' + ud.localpath)
|
|
md5data = (md5pipe.readline().split() or [ "" ])[0]
|
|
md5pipe.close()
|
|
except OSError:
|
|
md5data = ""
|
|
|
|
# verify the md5sum
|
|
if not Fetch.verify_md5sum(ud, md5data):
|
|
raise MD5SumError(url)
|
|
|
|
md5out = file(ud.md5, 'w')
|
|
md5out.write(md5data)
|
|
md5out.close()
|
|
write_md5sum = staticmethod(write_md5sum)
|
|
|
|
import cvs
|
|
import git
|
|
import local
|
|
import svn
|
|
import wget
|
|
import svk
|
|
import ssh
|
|
import perforce
|
|
|
|
methods.append(cvs.Cvs())
|
|
methods.append(git.Git())
|
|
methods.append(local.Local())
|
|
methods.append(svn.Svn())
|
|
methods.append(wget.Wget())
|
|
methods.append(svk.Svk())
|
|
methods.append(ssh.SSH())
|
|
methods.append(perforce.Perforce())
|