meta/classes: Update classes to use new fetcher API
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
parent
ca7adf7529
commit
984e90f4d7
|
@ -116,163 +116,38 @@ addtask setscene before do_fetch
|
||||||
addtask fetch
|
addtask fetch
|
||||||
do_fetch[dirs] = "${DL_DIR}"
|
do_fetch[dirs] = "${DL_DIR}"
|
||||||
python base_do_fetch() {
|
python base_do_fetch() {
|
||||||
import sys
|
|
||||||
|
src_uri = (bb.data.getVar('SRC_URI', d, True) or "").split()
|
||||||
|
if len(src_uri) == 0:
|
||||||
|
return
|
||||||
|
|
||||||
localdata = bb.data.createCopy(d)
|
localdata = bb.data.createCopy(d)
|
||||||
bb.data.update_data(localdata)
|
bb.data.update_data(localdata)
|
||||||
|
|
||||||
src_uri = bb.data.getVar('SRC_URI', localdata, 1)
|
try:
|
||||||
if not src_uri:
|
fetcher = bb.fetch2.Fetch(src_uri, localdata)
|
||||||
return 1
|
fetcher.download()
|
||||||
|
except bb.fetch2.BBFetchException, e:
|
||||||
try:
|
raise bb.build.FuncFailed(e)
|
||||||
bb.fetch.init(src_uri.split(),d)
|
|
||||||
except bb.fetch.NoMethodError:
|
|
||||||
(type, value, traceback) = sys.exc_info()
|
|
||||||
raise bb.build.FuncFailed("No method: %s" % value)
|
|
||||||
except bb.MalformedUrl:
|
|
||||||
(type, value, traceback) = sys.exc_info()
|
|
||||||
raise bb.build.FuncFailed("Malformed URL: %s" % value)
|
|
||||||
|
|
||||||
try:
|
|
||||||
if bb.fetch.__version__ == "1":
|
|
||||||
bb.fetch.go(localdata)
|
|
||||||
else:
|
|
||||||
bb.fetch.download(localdata)
|
|
||||||
except bb.fetch.MissingParameterError:
|
|
||||||
(type, value, traceback) = sys.exc_info()
|
|
||||||
raise bb.build.FuncFailed("Missing parameters: %s" % value)
|
|
||||||
except bb.fetch.FetchError:
|
|
||||||
(type, value, traceback) = sys.exc_info()
|
|
||||||
raise bb.build.FuncFailed("Fetch failed: %s" % value)
|
|
||||||
except bb.fetch.MD5SumError:
|
|
||||||
(type, value, traceback) = sys.exc_info()
|
|
||||||
raise bb.build.FuncFailed("MD5 failed: %s" % value)
|
|
||||||
except:
|
|
||||||
(type, value, traceback) = sys.exc_info()
|
|
||||||
raise bb.build.FuncFailed("Unknown fetch Error: %s" % value)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def subprocess_setup():
|
|
||||||
import signal
|
|
||||||
# Python installs a SIGPIPE handler by default. This is usually not what
|
|
||||||
# non-Python subprocesses expect.
|
|
||||||
# SIGPIPE errors are known issues with gzip/bash
|
|
||||||
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
|
||||||
|
|
||||||
def oe_unpack_file(file, data, url = None):
|
|
||||||
import subprocess
|
|
||||||
if not url:
|
|
||||||
url = "file://%s" % file
|
|
||||||
dots = file.split(".")
|
|
||||||
if dots[-1] in ['gz', 'bz2', 'Z']:
|
|
||||||
efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
|
|
||||||
else:
|
|
||||||
efile = file
|
|
||||||
cmd = None
|
|
||||||
if file.endswith('.tar'):
|
|
||||||
cmd = 'tar x --no-same-owner -f %s' % file
|
|
||||||
elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
|
|
||||||
cmd = 'tar xz --no-same-owner -f %s' % file
|
|
||||||
elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
|
|
||||||
cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
|
|
||||||
elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
|
|
||||||
cmd = 'gzip -dc %s > %s' % (file, efile)
|
|
||||||
elif file.endswith('.bz2'):
|
|
||||||
cmd = 'bzip2 -dc %s > %s' % (file, efile)
|
|
||||||
elif file.endswith('.tar.xz'):
|
|
||||||
cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
|
|
||||||
elif file.endswith('.xz'):
|
|
||||||
cmd = 'xz -dc %s > %s' % (file, efile)
|
|
||||||
elif file.endswith('.zip') or file.endswith('.jar'):
|
|
||||||
cmd = 'unzip -q -o'
|
|
||||||
(type, host, path, user, pswd, parm) = bb.decodeurl(url)
|
|
||||||
if 'dos' in parm:
|
|
||||||
cmd = '%s -a' % cmd
|
|
||||||
cmd = "%s '%s'" % (cmd, file)
|
|
||||||
elif os.path.isdir(file):
|
|
||||||
filesdir = os.path.realpath(bb.data.getVar("FILESDIR", data, 1))
|
|
||||||
destdir = "."
|
|
||||||
if file[0:len(filesdir)] == filesdir:
|
|
||||||
destdir = file[len(filesdir):file.rfind('/')]
|
|
||||||
destdir = destdir.strip('/')
|
|
||||||
if len(destdir) < 1:
|
|
||||||
destdir = "."
|
|
||||||
elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
|
|
||||||
os.makedirs("%s/%s" % (os.getcwd(), destdir))
|
|
||||||
cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
|
|
||||||
else:
|
|
||||||
(type, host, path, user, pswd, parm) = bb.decodeurl(url)
|
|
||||||
if not 'patch' in parm:
|
|
||||||
# The "destdir" handling was specifically done for FILESPATH
|
|
||||||
# items. So, only do so for file:// entries.
|
|
||||||
if type == "file" and path.find("/") != -1:
|
|
||||||
destdir = path.rsplit("/", 1)[0]
|
|
||||||
else:
|
|
||||||
destdir = "."
|
|
||||||
bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
|
|
||||||
cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
|
|
||||||
|
|
||||||
if not cmd:
|
|
||||||
return True
|
|
||||||
|
|
||||||
dest = os.path.join(os.getcwd(), os.path.basename(file))
|
|
||||||
if os.path.exists(dest):
|
|
||||||
if os.path.samefile(file, dest):
|
|
||||||
return True
|
|
||||||
|
|
||||||
# Change to subdir before executing command
|
|
||||||
save_cwd = os.getcwd();
|
|
||||||
parm = bb.decodeurl(url)[5]
|
|
||||||
if 'subdir' in parm:
|
|
||||||
newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
|
|
||||||
bb.mkdirhier(newdir)
|
|
||||||
os.chdir(newdir)
|
|
||||||
|
|
||||||
cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd)
|
|
||||||
bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
|
|
||||||
ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
|
|
||||||
|
|
||||||
os.chdir(save_cwd)
|
|
||||||
|
|
||||||
return ret == 0
|
|
||||||
|
|
||||||
addtask unpack after do_fetch
|
addtask unpack after do_fetch
|
||||||
do_unpack[dirs] = "${WORKDIR}"
|
do_unpack[dirs] = "${WORKDIR}"
|
||||||
python base_do_unpack() {
|
python base_do_unpack() {
|
||||||
import re
|
src_uri = (bb.data.getVar('SRC_URI', d, True) or "").split()
|
||||||
|
if len(src_uri) == 0:
|
||||||
|
return
|
||||||
|
|
||||||
localdata = bb.data.createCopy(d)
|
localdata = bb.data.createCopy(d)
|
||||||
bb.data.update_data(localdata)
|
bb.data.update_data(localdata)
|
||||||
|
|
||||||
urldata = bb.fetch.init([], localdata)
|
rootdir = bb.data.getVar('WORKDIR', localdata, True)
|
||||||
|
|
||||||
src_uri = bb.data.getVar('SRC_URI', localdata, True)
|
try:
|
||||||
if not src_uri:
|
fetcher = bb.fetch2.Fetch(src_uri, localdata)
|
||||||
return
|
fetcher.unpack(rootdir)
|
||||||
for url in src_uri.split():
|
except bb.fetch2.BBFetchException, e:
|
||||||
try:
|
raise bb.build.FuncFailed(e)
|
||||||
local = bb.data.expand(bb.fetch.localpath(url, localdata), localdata)
|
|
||||||
except bb.MalformedUrl, e:
|
|
||||||
raise FuncFailed('Unable to generate local path for malformed uri: %s' % e)
|
|
||||||
if local is None:
|
|
||||||
continue
|
|
||||||
local = os.path.realpath(local)
|
|
||||||
lockfile = urldata[url].lockfile
|
|
||||||
if lockfile:
|
|
||||||
lf = bb.utils.lockfile(urldata[url].lockfile)
|
|
||||||
|
|
||||||
if bb.fetch.__version__ == "1":
|
|
||||||
ret = oe_unpack_file(local, localdata, url)
|
|
||||||
else:
|
|
||||||
# use bb.fetch2 unpack API
|
|
||||||
ud = urldata[url]
|
|
||||||
rootdir = bb.data.getVar('WORKDIR', localdata, True)
|
|
||||||
ret = ud.method.unpack(ud, rootdir, localdata)
|
|
||||||
if lockfile:
|
|
||||||
bb.utils.unlockfile(lf)
|
|
||||||
if not ret:
|
|
||||||
raise bb.build.FuncFailed("oe_unpack_file failed with return value %s" % ret)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
GIT_CONFIG = "${STAGING_DIR_NATIVE}/usr/etc/gitconfig"
|
GIT_CONFIG = "${STAGING_DIR_NATIVE}/usr/etc/gitconfig"
|
||||||
|
@ -550,7 +425,8 @@ python () {
|
||||||
for s in srcuri.split():
|
for s in srcuri.split():
|
||||||
if not s.startswith("file://"):
|
if not s.startswith("file://"):
|
||||||
continue
|
continue
|
||||||
local = bb.data.expand(bb.fetch.localpath(s, d), d)
|
fetcher = bb.fetch2.Fetch([s], d)
|
||||||
|
local = fetcher.localpath(s)
|
||||||
for mp in paths:
|
for mp in paths:
|
||||||
if local.startswith(mp):
|
if local.startswith(mp):
|
||||||
#bb.note("overriding PACKAGE_ARCH from %s to %s" % (pkg_arch, mach_arch))
|
#bb.note("overriding PACKAGE_ARCH from %s to %s" % (pkg_arch, mach_arch))
|
||||||
|
@ -594,14 +470,12 @@ python do_cleanall() {
|
||||||
dl_dir = bb.data.getVar('DL_DIR', localdata, True)
|
dl_dir = bb.data.getVar('DL_DIR', localdata, True)
|
||||||
dl_dir = os.path.realpath(dl_dir)
|
dl_dir = os.path.realpath(dl_dir)
|
||||||
|
|
||||||
src_uri = bb.data.getVar('SRC_URI', localdata, True)
|
src_uri = (bb.data.getVar('SRC_URI', localdata, True) or "").split()
|
||||||
if not src_uri:
|
if len(src_uri) == 0:
|
||||||
return
|
return
|
||||||
for url in src_uri.split():
|
fetcher = bb.fetch2.Fetch(src_uri, localdata)
|
||||||
try:
|
for url in src_uri:
|
||||||
local = bb.data.expand(bb.fetch.localpath(url, localdata), localdata)
|
local = fetcher.localpath(url)
|
||||||
except bb.MalformedUrl, e:
|
|
||||||
raise FuncFailed('Unable to generate local path for malformed uri: %s' % e)
|
|
||||||
if local is None:
|
if local is None:
|
||||||
continue
|
continue
|
||||||
local = os.path.realpath(local)
|
local = os.path.realpath(local)
|
||||||
|
|
|
@ -58,9 +58,8 @@ python patch_do_patch() {
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not local:
|
if not local:
|
||||||
bb.fetch.init([url],d)
|
|
||||||
url = bb.encodeurl((type, host, path, user, pswd, []))
|
url = bb.encodeurl((type, host, path, user, pswd, []))
|
||||||
local = os.path.join('/', bb.fetch.localpath(url, d))
|
local = os.path.join('/', bb.fetch2.localpath(url, d))
|
||||||
local = bb.data.expand(local, d)
|
local = bb.data.expand(local, d)
|
||||||
|
|
||||||
if "striplevel" in parm:
|
if "striplevel" in parm:
|
||||||
|
|
|
@ -326,40 +326,40 @@ def sstate_package(ss, d):
|
||||||
return
|
return
|
||||||
|
|
||||||
def pstaging_fetch(sstatepkg, d):
|
def pstaging_fetch(sstatepkg, d):
|
||||||
import bb.fetch
|
|
||||||
|
|
||||||
# only try and fetch if the user has configured a mirror
|
|
||||||
|
|
||||||
|
# Only try and fetch if the user has configured a mirror
|
||||||
mirrors = bb.data.getVar('SSTATE_MIRRORS', d, True)
|
mirrors = bb.data.getVar('SSTATE_MIRRORS', d, True)
|
||||||
if mirrors:
|
if not mirrors:
|
||||||
# Copy the data object and override DL_DIR and SRC_URI
|
return
|
||||||
localdata = bb.data.createCopy(d)
|
|
||||||
bb.data.update_data(localdata)
|
|
||||||
|
|
||||||
dldir = bb.data.expand("${SSTATE_DIR}", localdata)
|
import bb.fetch2
|
||||||
srcuri = "file://" + os.path.basename(sstatepkg)
|
# Copy the data object and override DL_DIR and SRC_URI
|
||||||
|
localdata = bb.data.createCopy(d)
|
||||||
|
bb.data.update_data(localdata)
|
||||||
|
|
||||||
bb.mkdirhier(dldir)
|
dldir = bb.data.expand("${SSTATE_DIR}", localdata)
|
||||||
|
srcuri = "file://" + os.path.basename(sstatepkg)
|
||||||
|
|
||||||
bb.data.setVar('DL_DIR', dldir, localdata)
|
bb.mkdirhier(dldir)
|
||||||
bb.data.setVar('PREMIRRORS', mirrors, localdata)
|
|
||||||
bb.data.setVar('SRC_URI', srcuri, localdata)
|
|
||||||
|
|
||||||
# Try a fetch from the sstate mirror, if it fails just return and
|
bb.data.setVar('DL_DIR', dldir, localdata)
|
||||||
# we will build the package
|
bb.data.setVar('PREMIRRORS', mirrors, localdata)
|
||||||
try:
|
bb.data.setVar('SRC_URI', srcuri, localdata)
|
||||||
bb.fetch.init([srcuri], localdata)
|
|
||||||
if bb.fetch.__version__ == "1":
|
# Try a fetch from the sstate mirror, if it fails just return and
|
||||||
bb.fetch.go(localdata, [srcuri])
|
# we will build the package
|
||||||
else:
|
try:
|
||||||
bb.fetch.download(localdata, [srcuri])
|
fetcher = bb.fetch2.Fetch([srcuri], localdata)
|
||||||
# Need to optimise this, if using file:// urls, the fetcher just changes the local path
|
fetcher.download()
|
||||||
# For now work around by symlinking
|
|
||||||
localpath = bb.data.expand(bb.fetch.localpath(srcuri, localdata), localdata)
|
# Need to optimise this, if using file:// urls, the fetcher just changes the local path
|
||||||
if localpath != sstatepkg and os.path.exists(localpath):
|
# For now work around by symlinking
|
||||||
os.symlink(localpath, sstatepkg)
|
localpath = bb.data.expand(fetcher.localpath(srcuri), localdata)
|
||||||
except:
|
if localpath != sstatepkg and os.path.exists(localpath) and not os.path.exists(sstatepkg):
|
||||||
pass
|
os.symlink(localpath, sstatepkg)
|
||||||
|
|
||||||
|
except bb.fetch2.BBFetchException:
|
||||||
|
pass
|
||||||
|
|
||||||
def sstate_setscene(d):
|
def sstate_setscene(d):
|
||||||
shared_state = sstate_state_fromvars(d)
|
shared_state = sstate_state_fromvars(d)
|
||||||
|
|
|
@ -51,11 +51,12 @@ def machine_paths(d):
|
||||||
def is_machine_specific(d):
|
def is_machine_specific(d):
|
||||||
"""Determine whether the current recipe is machine specific"""
|
"""Determine whether the current recipe is machine specific"""
|
||||||
machinepaths = set(machine_paths(d))
|
machinepaths = set(machine_paths(d))
|
||||||
urldatadict = bb.fetch.init(d.getVar("SRC_URI", True).split(), d, True)
|
srcuri = d.getVar("SRC_URI", True).split()
|
||||||
for urldata in (urldata for urldata in urldatadict.itervalues()
|
for url in srcuri:
|
||||||
if urldata.type == "file"):
|
fetcher = bb.fetch2.Fetch([srcuri], d)
|
||||||
if any(urldata.localpath.startswith(mp + "/") for mp in machinepaths):
|
if url.startswith("file://"):
|
||||||
return True
|
if any(fetcher.localpath(url).startswith(mp + "/") for mp in machinepaths):
|
||||||
|
return True
|
||||||
|
|
||||||
def oe_popen_env(d):
|
def oe_popen_env(d):
|
||||||
env = d.getVar("__oe_popen_env", False)
|
env = d.getVar("__oe_popen_env", False)
|
||||||
|
|
|
@ -72,14 +72,14 @@ class PatchSet(object):
|
||||||
if not patch.get("remote"):
|
if not patch.get("remote"):
|
||||||
raise PatchError("Patch file must be specified in patch import.")
|
raise PatchError("Patch file must be specified in patch import.")
|
||||||
else:
|
else:
|
||||||
patch["file"] = bb.fetch.localpath(patch["remote"], self.d)
|
patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
|
||||||
|
|
||||||
for param in PatchSet.defaults:
|
for param in PatchSet.defaults:
|
||||||
if not patch.get(param):
|
if not patch.get(param):
|
||||||
patch[param] = PatchSet.defaults[param]
|
patch[param] = PatchSet.defaults[param]
|
||||||
|
|
||||||
if patch.get("remote"):
|
if patch.get("remote"):
|
||||||
patch["file"] = bb.data.expand(bb.fetch.localpath(patch["remote"], self.d), self.d)
|
patch["file"] = bb.data.expand(bb.fetch2.localpath(patch["remote"], self.d), self.d)
|
||||||
|
|
||||||
patch["filemd5"] = bb.utils.md5_file(patch["file"])
|
patch["filemd5"] = bb.utils.md5_file(patch["file"])
|
||||||
|
|
||||||
|
@ -293,7 +293,7 @@ class QuiltTree(PatchSet):
|
||||||
if type == "file":
|
if type == "file":
|
||||||
import shutil
|
import shutil
|
||||||
if not patch.get("file") and patch.get("remote"):
|
if not patch.get("file") and patch.get("remote"):
|
||||||
patch["file"] = bb.fetch.localpath(patch["remote"], self.d)
|
patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
|
||||||
|
|
||||||
shutil.copyfile(patch["quiltfile"], patch["file"])
|
shutil.copyfile(patch["quiltfile"], patch["file"])
|
||||||
else:
|
else:
|
||||||
|
|
Loading…
Reference in New Issue