From f7c50a146093bd0a64b418cad84fdd4dc360e163 Mon Sep 17 00:00:00 2001 From: Maximilian Attems Date: Thu, 1 May 2008 07:54:28 +0000 Subject: [PATCH] Revert "debian/bin, debian/lib: Infrastructure was declared irrelevant, drop it." as announced revert trunk sabotage This reverts commit ccf3463f401b082e89a76d3475eff5b30c05622a. svn path=/dists/trunk/linux-2.6/; revision=11215 --- debian/bin/abicheck.py | 101 ++++++ debian/bin/abiupdate.py | 182 ++++++++++ debian/bin/check-patches.sh | 13 + debian/bin/gencontrol.py | 320 +++++++++++++++++ debian/bin/genorig.py | 117 +++++++ debian/bin/kconfig.py | 13 + debian/lib/python/debian_linux/__init__.py | 1 + debian/lib/python/debian_linux/abi.py | 51 +++ debian/lib/python/debian_linux/config.py | 238 +++++++++++++ debian/lib/python/debian_linux/debian.py | 348 +++++++++++++++++++ debian/lib/python/debian_linux/gencontrol.py | 310 +++++++++++++++++ debian/lib/python/debian_linux/kconfig.py | 70 ++++ debian/lib/python/debian_linux/patches.py | 206 +++++++++++ debian/lib/python/debian_linux/utils.py | 108 ++++++ 14 files changed, 2078 insertions(+) create mode 100755 debian/bin/abicheck.py create mode 100755 debian/bin/abiupdate.py create mode 100755 debian/bin/check-patches.sh create mode 100755 debian/bin/gencontrol.py create mode 100755 debian/bin/genorig.py create mode 100755 debian/bin/kconfig.py create mode 100644 debian/lib/python/debian_linux/__init__.py create mode 100644 debian/lib/python/debian_linux/abi.py create mode 100644 debian/lib/python/debian_linux/config.py create mode 100644 debian/lib/python/debian_linux/debian.py create mode 100644 debian/lib/python/debian_linux/gencontrol.py create mode 100644 debian/lib/python/debian_linux/kconfig.py create mode 100644 debian/lib/python/debian_linux/patches.py create mode 100644 debian/lib/python/debian_linux/utils.py diff --git a/debian/bin/abicheck.py b/debian/bin/abicheck.py new file mode 100755 index 000000000..3dea267cb --- /dev/null +++ b/debian/bin/abicheck.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +import sys +sys.path.append('debian/lib/python') + +from debian_linux.abi import * +from debian_linux.config import ConfigCoreDump +from debian_linux.debian import * + +class checker(object): + def __init__(self, dir, arch, featureset, flavour): + self.arch, self.featureset, self.flavour = arch, featureset, flavour + self.config = ConfigCoreDump(fp = file("debian/config.defines.dump")) + self.filename_new = "%s/Module.symvers" % dir + + changelog = Changelog(version = VersionLinux)[0] + version = changelog.version.linux_version + abiname = self.config['abi',]['abiname'] + self.filename_ref = "debian/abi/%s-%s/%s_%s_%s" % (version, abiname, arch, featureset, flavour) + + def __call__(self, out): + ret = 0 + + new = symbols(self.filename_new) + try: + ref = symbols(self.filename_ref) + except IOError: + out.write("Can't read ABI reference. ABI not checked! Continuing.\n") + return 0 + + add_info, change_info, remove_info = ref.cmp(new) + add = set(add_info.keys()) + change = set(change_info.keys()) + remove = set(remove_info.keys()) + ignore = self._ignore(add_info, change_info, remove_info) + + add_effective = add - ignore + change_effective = change - ignore + remove_effective = remove - ignore + + if change_effective or remove_effective: + out.write("ABI has changed! Refusing to continue.\n") + ret = 1 + elif change or remove: + out.write("ABI has changed but all changes have been ignored. Continuing.\n") + elif add_effective: + out.write("New symbols have been added. Continuing.\n") + elif add: + out.write("New symbols have been added but have been ignored. Continuing.\n") + else: + out.write("No ABI changes.\n") + if add: + out.write("\nAdded symbols:\n") + t = list(add) + t.sort() + for symbol in t: + info = [] + if symbol in ignore: + info.append("ignored") + for i in ('module', 'version', 'export'): + info.append("%s: %s" % (i, add_info[symbol][i])) + out.write("%-48s %s\n" % (symbol, ", ".join(info))) + if change: + out.write("\nChanged symbols:\n") + t = list(change) + t.sort() + for symbol in t: + info = [] + if symbol in ignore: + info.append("ignored") + s = change_info[symbol] + changes = s['changes'] + for i in ('module', 'version', 'export'): + if changes.has_key(i): + info.append("%s: %s -> %s" % (i, s['ref'][i], s['new'][i])) + else: + info.append("%s: %s" % (i, new[symbol][i])) + out.write("%-48s %s\n" % (symbol, ", ".join(info))) + if remove: + out.write("\nRemoved symbols:\n") + t = list(remove) + t.sort() + for symbol in t: + info = [] + if symbol in ignore: + info.append("ignored") + for i in ('module', 'version', 'export'): + info.append("%s: %s" % (i, remove_info[symbol][i])) + out.write("%-48s %s\n" % (symbol, ", ".join(info))) + + return ret + + def _ignore(self, add, change, remove): + config = self.config.merge('abi', self.arch, self.featureset, self.flavour) + ignores = config.get('ignore-changes', None) + if ignores is None: + return set() + return set(ignores.split()) + +if __name__ == '__main__': + sys.exit(checker(*sys.argv[1:])(sys.stdout)) diff --git a/debian/bin/abiupdate.py b/debian/bin/abiupdate.py new file mode 100755 index 000000000..cbe305ab8 --- /dev/null +++ b/debian/bin/abiupdate.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python + +import sys +sys.path.append(sys.path[0] + "/../lib/python") + +import optparse, os, shutil, tempfile, urllib2 +from debian_linux.abi import * +from debian_linux.config import * +from debian_linux.debian import * + +default_url_base = "http://ftp.de.debian.org/debian/" +default_url_base_incoming = "http://incoming.debian.org/" + +class url_debian_flat(object): + def __init__(self, base): + self.base = base + + def __call__(self, source, filename): + return self.base + filename + +class url_debian_pool(object): + def __init__(self, base): + self.base = base + + def __call__(self, source, filename): + return self.base + "pool/main/" + source[0] + "/" + source + "/" + filename + +class main(object): + dir = None + + def __init__(self, url, url_config = None, arch = None, featureset = None, flavour = None): + self.log = sys.stdout.write + + self.url = self.url_config = url + if url_config is not None: + self.url_config = url_config + self.override_arch = arch + self.override_featureset = featureset + self.override_flavour = flavour + + changelog = Changelog(version = VersionLinux) + while changelog[0].distribution == 'UNRELEASED': + changelog.pop(0) + changelog = changelog[0] + + self.source = changelog.source + self.version = changelog.version.linux_version + self.version_source = changelog.version.complete + + local_config = ConfigCoreDump(fp = file("debian/config.defines.dump")) + + self.version_abi = self.version + '-' + local_config['abi',]['abiname'] + + def __call__(self): + self.dir = tempfile.mkdtemp(prefix = 'abiupdate') + try: + self.log("Retreive config\n") + config = self.get_config() + if self.override_arch: + arches = [self.override_arch] + else: + arches = config[('base',)]['arches'] + for arch in arches: + self.update_arch(config, arch) + finally: + shutil.rmtree(self.dir) + + def extract_package(self, filename, base): + base_out = self.dir + "/" + base + os.mkdir(base_out) + os.system("dpkg-deb --extract %s %s" % (filename, base_out)) + return base_out + + def get_abi(self, arch, prefix): + filename = "linux-headers-%s-%s_%s_%s.deb" % (self.version_abi, prefix, self.version_source, arch) + f = self.retrieve_package(self.url, filename) + d = self.extract_package(f, "linux-headers-%s_%s" % (prefix, arch)) + f1 = d + "/usr/src/linux-headers-%s-%s/Module.symvers" % (self.version_abi, prefix) + s = symbols(f1) + shutil.rmtree(d) + return s + + def get_config(self): + filename = "linux-support-%s_%s_all.deb" % (self.version_abi, self.version_source) + f = self.retrieve_package(self.url_config, filename) + d = self.extract_package(f, "linux-support") + c = d + "/usr/src/linux-support-" + self.version_abi + "/config.defines.dump" + config = ConfigCoreDump(fp = file(c)) + shutil.rmtree(d) + return config + + def retrieve_package(self, url, filename): + u = url(self.source, filename) + filename_out = self.dir + "/" + filename + f_in = urllib2.urlopen(u) + f_out = file(filename_out, 'w') + while 1: + r = f_in.read() + if not r: + break + f_out.write(r) + return filename_out + + def save_abi(self, symbols, arch, featureset, flavour): + dir = "debian/abi/%s" % self.version_abi + if not os.path.exists(dir): + os.makedirs(dir) + out = "%s/%s_%s_%s" % (dir, arch, featureset, flavour) + symbols.write(file(out, 'w')) + + def update_arch(self, config, arch): + if self.override_featureset: + featuresets = [self.override_featureset] + else: + featuresets = config[('base', arch)]['featuresets'] + for featureset in featuresets: + self.update_featureset(config, arch, featureset) + + def update_featureset(self, config, arch, featureset): + config_base = config.merge('base', arch, featureset) + + if not config_base.get('enabled', True): + return + + if self.override_flavour: + flavours = [self.override_flavour] + else: + flavours = config_base['flavours'] + for flavour in flavours: + self.update_flavour(config, arch, featureset, flavour) + + def update_flavour(self, config, arch, featureset, flavour): + config_base = config.merge('base', arch, featureset, flavour) + + if not config_base.get('modules', True): + return + + self.log("Updating ABI for arch %s, featureset %s, flavour %s: " % (arch, featureset, flavour)) + try: + if featureset == 'none': + localversion = flavour + else: + localversion = featureset + '-' + flavour + + abi = self.get_abi(arch, localversion) + self.save_abi(abi, arch, featureset, flavour) + self.log("Ok.\n") + except KeyboardInterrupt: + self.log("Interrupted!\n") + sys.exit(1) + except Exception, e: + self.log("FAILED! (%s)\n" % str(e)) + +if __name__ == '__main__': + options = optparse.OptionParser() + options.add_option("-i", "--incoming", action = "store_true", dest = "incoming") + options.add_option("--incoming-config", action = "store_true", dest = "incoming_config") + options.add_option("-u", "--url-base", dest = "url_base", default = default_url_base) + options.add_option("--url-base-incoming", dest = "url_base_incoming", default = default_url_base_incoming) + + opts, args = options.parse_args() + + kw = {} + if len(args) >= 1: + kw['arch'] =args[0] + if len(args) >= 2: + kw['featureset'] =args[1] + if len(args) >= 3: + kw['flavour'] =args[2] + + url_base = url_debian_pool(opts.url_base) + url_base_incoming = url_debian_flat(opts.url_base_incoming) + if opts.incoming_config: + url = url_config = url_base_incoming + else: + url_config = url_base + if opts.incoming: + url = url_base_incoming + else: + url = url_base + + main(url, url_config, **kw)() diff --git a/debian/bin/check-patches.sh b/debian/bin/check-patches.sh new file mode 100755 index 000000000..3a733fae3 --- /dev/null +++ b/debian/bin/check-patches.sh @@ -0,0 +1,13 @@ +#!/bin/sh -e + +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR" EXIT +grep -v "^#" debian/patches/series/* | awk '{if (NF >= 2) print "debian/patches/" $2}' | sort -u > $TMPDIR/used +find debian/patches ! -path '*/series*' -type f -name "*.diff" -o -name "*.patch" -printf "%p\n" | sort > $TMPDIR/avail +echo "Used patches" +echo "==============" +cat $TMPDIR/used +echo +echo "Unused patches" +echo "==============" +fgrep -v -f $TMPDIR/used $TMPDIR/avail diff --git a/debian/bin/gencontrol.py b/debian/bin/gencontrol.py new file mode 100755 index 000000000..42290ac18 --- /dev/null +++ b/debian/bin/gencontrol.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python + +import os, sys +sys.path.append("debian/lib/python") + +from debian_linux.config import ConfigCoreHierarchy +from debian_linux.debian import * +from debian_linux.gencontrol import Gencontrol as Base +from debian_linux.utils import Templates + +class Gencontrol(Base): + def __init__(self, config_dirs = ["debian/config"], template_dirs = ["debian/templates"]): + super(Gencontrol, self).__init__(ConfigCoreHierarchy(config_dirs), Templates(template_dirs), VersionLinux) + self.process_changelog() + self.config_dirs = config_dirs + + def do_main_setup(self, vars, makeflags, extra): + super(Gencontrol, self).do_main_setup(vars, makeflags, extra) + vars.update(self.config['image',]) + makeflags.update({ + 'MAJOR': self.version.linux_major, + 'VERSION': self.version.linux_version, + 'UPSTREAMVERSION': self.version.linux_upstream, + 'ABINAME': self.abiname, + 'SOURCEVERSION': self.version.complete, + }) + + def do_main_packages(self, packages, extra): + packages.extend(self.process_packages(self.templates["control.main"], self.vars)) + packages.append(self.process_real_tree(self.templates["control.tree"][0], self.vars)) + packages.extend(self.process_packages(self.templates["control.support"], self.vars)) + + def do_arch_setup(self, vars, makeflags, arch, extra): + config_base = self.config.merge('base', arch) + vars.update(self.config.merge('image', arch)) + config_libc_dev = self.config.merge('libc-dev', arch) + makeflags['LIBC_DEV_ARCH'] = config_libc_dev.get('arch', config_base.get('kernel-arch')) + + def do_arch_packages(self, packages, makefile, arch, vars, makeflags, extra): + headers_arch = self.templates["control.headers.arch"] + packages_headers_arch = self.process_packages(headers_arch, vars) + + libc_dev = self.templates["control.libc-dev"] + packages_headers_arch[0:0] = self.process_packages(libc_dev, {}) + + extra['headers_arch_depends'] = packages_headers_arch[-1]['Depends'] = PackageRelation() + + for package in packages_headers_arch: + name = package['Package'] + if packages.has_key(name): + package = packages.get(name) + package['Architecture'].append(arch) + else: + package['Architecture'] = [arch] + packages.append(package) + + cmds_binary_arch = ["$(MAKE) -f debian/rules.real binary-arch-arch %s" % makeflags] + cmds_source = ["$(MAKE) -f debian/rules.real source-arch %s" % makeflags] + makefile.add('binary-arch_%s_real' % arch, cmds = cmds_binary_arch) + makefile.add('source_%s_real' % arch, cmds = cmds_source) + + def do_featureset_setup(self, vars, makeflags, arch, featureset, extra): + vars.update(self.config.merge('image', arch, featureset)) + makeflags['LOCALVERSION_HEADERS'] = vars['localversion_headers'] = vars['localversion'] + makeflags['KERNEL_HEADER_DIRS'] = vars.get('kernel-header-dirs', vars.get('kernel-arch')) + + def do_featureset_packages(self, packages, makefile, arch, featureset, vars, makeflags, extra): + headers_featureset = self.templates["control.headers.featureset"] + package_headers = self.process_package(headers_featureset[0], vars) + + name = package_headers['Package'] + if packages.has_key(name): + package_headers = packages.get(name) + package_headers['Architecture'].append(arch) + else: + package_headers['Architecture'] = [arch] + packages.append(package_headers) + + cmds_binary_arch = ["$(MAKE) -f debian/rules.real binary-arch-featureset %s" % makeflags] + cmds_source = ["$(MAKE) -f debian/rules.real source-featureset %s" % makeflags] + makefile.add('binary-arch_%s_%s_real' % (arch, featureset), cmds = cmds_binary_arch) + makefile.add('source_%s_%s_real' % (arch, featureset), cmds = cmds_source) + + def do_flavour_setup(self, vars, makeflags, arch, featureset, flavour, extra): + config_image = self.config.merge('image', arch, featureset, flavour) + + vars.update(config_image) + + vars['localversion-image'] = vars['localversion'] + override_localversion = config_image.get('override-localversion', None) + if override_localversion is not None: + vars['localversion-image'] = vars['localversion_headers'] + '-' + override_localversion + + for i in ( + ('compiler', 'COMPILER'), + ('kernel-arch', 'KERNEL_ARCH'), + ('localversion', 'LOCALVERSION'), + ('type', 'TYPE'), + ): + makeflags[i[1]] = vars[i[0]] + for i in ( + ('cflags', 'CFLAGS'), + ('initramfs', 'INITRAMFS'), + ('kpkg-arch', 'KPKG_ARCH'), + ('kpkg-subarch', 'KPKG_SUBARCH'), + ('localversion-image', 'LOCALVERSION_IMAGE'), + ('override-host-type', 'OVERRIDE_HOST_TYPE'), + ): + if vars.has_key(i[0]): + makeflags[i[1]] = vars[i[0]] + makeflags['KERNEL_HEADER_DIRS'] = vars.get('kernel-header-dirs', vars.get('kernel-arch')) + + def do_flavour_packages(self, packages, makefile, arch, featureset, flavour, vars, makeflags, extra): + headers = self.templates["control.headers"] + + config_entry_base = self.config.merge('base', arch, featureset, flavour) + config_entry_relations = self.config.merge('relations', arch, featureset, flavour) + + compiler = config_entry_base.get('compiler', 'gcc') + relations_compiler = PackageRelation(config_entry_relations[compiler]) + relations_compiler_build_dep = PackageRelation(config_entry_relations[compiler]) + for group in relations_compiler_build_dep: + for item in group: + item.arches = [arch] + packages['source']['Build-Depends'].extend(relations_compiler_build_dep) + + image_relations = { + 'conflicts': PackageRelation(), + 'depends': PackageRelation(), + } + if vars.get('initramfs', True): + generators = vars['initramfs-generators'] + config_entry_commands_initramfs = self.config.merge('commands-image-initramfs-generators', arch, featureset, flavour) + commands = [config_entry_commands_initramfs[i] for i in generators if config_entry_commands_initramfs.has_key(i)] + makeflags['INITRD_CMD'] = ' '.join(commands) + l_depends = PackageRelationGroup() + for i in generators: + i = config_entry_relations.get(i, i) + l_depends.append(i) + a = PackageRelationEntry(i) + if a.operator is not None: + a.operator = -a.operator + image_relations['conflicts'].append(PackageRelationGroup([a])) + image_relations['depends'].append(l_depends) + + packages_dummy = [] + packages_own = [] + + if vars['type'] == 'plain-s390-tape': + image = self.templates["control.image.type-standalone"] + build_modules = False + elif vars['type'] == 'plain-xen': + image = self.templates["control.image.type-modulesextra"] + build_modules = True + config_entry_xen = self.config.merge('xen', arch, featureset, flavour) + if config_entry_xen.get('dom0-support', True): + p = self.process_packages(self.templates['control.xen-linux-system'], vars) + l = PackageRelationGroup() + for version in config_entry_xen['versions']: + l.append("xen-hypervisor-%s-%s" % (version, config_entry_xen['flavour'])) + makeflags['XEN_VERSIONS'] = ' '.join(['%s-%s' % (i, config_entry_xen['flavour']) for i in config_entry_xen['versions']]) + p[0]['Depends'].append(l) + packages_dummy.extend(p) + else: + build_modules = True + image = self.templates["control.image.type-%s" % vars['type']] + #image = self.templates["control.image.type-modulesinline"] + + if not vars.has_key('desc'): + vars['desc'] = None + + packages_own.append(self.process_real_image(image[0], image_relations, config_entry_relations, vars)) + packages_own.extend(self.process_packages(image[1:], vars)) + + if build_modules: + makeflags['MODULES'] = True + package_headers = self.process_package(headers[0], vars) + package_headers['Depends'].extend(relations_compiler) + packages_own.append(package_headers) + extra['headers_arch_depends'].append('%s (= ${Source-Version})' % packages_own[-1]['Package']) + + for package in packages_own + packages_dummy: + name = package['Package'] + if packages.has_key(name): + package = packages.get(name) + package['Architecture'].append(arch) + else: + package['Architecture'] = [arch] + packages.append(package) + + if vars['type'] == 'plain-xen': + for i in ('postinst', 'postrm', 'prerm'): + j = self.substitute(self.templates["image.xen.%s" % i], vars) + file("debian/%s.%s" % (packages_own[0]['Package'], i), 'w').write(j) + + def get_config(*entry_name): + entry_real = ('image',) + entry_name + entry = self.config.get(entry_real, None) + if entry is None: + return None + return entry.get('configs', None) + + def check_config_default(fail, f): + for d in self.config_dirs[::-1]: + f1 = d + '/' + f + if os.path.exists(f1): + return [f1] + if fail: + raise RuntimeError("%s unavailable" % f) + return [] + + def check_config_files(files): + ret = [] + for f in files: + for d in self.config_dirs[::-1]: + f1 = d + '/' + f + if os.path.exists(f1): + ret.append(f1) + break + else: + raise RuntimeError("%s unavailable" % f) + return ret + + def check_config(default, fail, *entry_name): + configs = get_config(*entry_name) + if configs is None: + return check_config_default(fail, default) + return check_config_files(configs) + + kconfig = check_config('config', True) + kconfig.extend(check_config("%s/config" % arch, True, arch)) + kconfig.extend(check_config("%s/config.%s" % (arch, flavour), False, arch, None, flavour)) + kconfig.extend(check_config("featureset-%s/config" % featureset, False, None, featureset)) + kconfig.extend(check_config("%s/%s/config" % (arch, featureset), False, arch, featureset)) + kconfig.extend(check_config("%s/%s/config.%s" % (arch, featureset, flavour), False, arch, featureset, flavour)) + makeflags['KCONFIG'] = ' '.join(kconfig) + + cmds_binary_arch = [] + cmds_binary_arch.append("$(MAKE) -f debian/rules.real binary-arch-flavour %s" % makeflags) + if packages_dummy: + cmds_binary_arch.append("$(MAKE) -f debian/rules.real install-dummy DH_OPTIONS='%s' %s" % (' '.join(["-p%s" % i['Package'] for i in packages_dummy]), makeflags)) + cmds_build = ["$(MAKE) -f debian/rules.real build %s" % makeflags] + cmds_setup = ["$(MAKE) -f debian/rules.real setup-flavour %s" % makeflags] + makefile.add('binary-arch_%s_%s_%s_real' % (arch, featureset, flavour), cmds = cmds_binary_arch) + makefile.add('build_%s_%s_%s_real' % (arch, featureset, flavour), cmds = cmds_build) + makefile.add('setup_%s_%s_%s_real' % (arch, featureset, flavour), cmds = cmds_setup) + + def do_extra(self, packages, makefile): + apply = self.templates['patch.apply'] + + vars = { + 'revisions': 'orig ' + ' '.join([i.debian for i in self.versions[::-1]]), + 'upstream': self.version.upstream, + 'linux_upstream': self.version.linux_upstream, + 'abiname': self.abiname, + } + + apply = self.substitute(apply, vars) + + file('debian/bin/patch.apply', 'w').write(apply) + + def process_changelog(self): + act_upstream = self.changelog[0].version.linux_upstream + versions = [] + for i in self.changelog: + if i.version.linux_upstream != act_upstream: + break + versions.append(i.version) + self.versions = versions + self.version = self.changelog[0].version + if self.version.linux_modifier is not None: + self.abiname = '' + else: + self.abiname = '-%s' % self.config['abi',]['abiname'] + self.vars = { + 'upstreamversion': self.version.linux_upstream, + 'version': self.version.linux_version, + 'source_upstream': self.version.upstream, + 'major': self.version.linux_major, + 'abiname': self.abiname, + } + self.config['version',] = {'source': self.version.complete, 'abiname': self.abiname} + + def process_real_image(self, in_entry, relations, config, vars): + entry = self.process_package(in_entry, vars) + for field in 'Depends', 'Provides', 'Suggests', 'Recommends', 'Conflicts': + value = entry.get(field, PackageRelation()) + t = vars.get(field.lower(), []) + value.extend(t) + t = relations.get(field.lower(), []) + value.extend(t) + if value: + entry[field] = value + return entry + + def process_real_tree(self, in_entry, vars): + entry = self.process_package(in_entry, vars) + for i in (('Depends', 'Provides')): + value = PackageRelation() + value.extend(entry.get(i, [])) + if i == 'Depends': + v = self.changelog[0].version + value.append("linux-patch-debian-%s (= %s)" % (v.linux_version, v.complete)) + value.append(' | '.join(["linux-source-%s (= %s)" % (v.linux_version, v.complete) for v in self.versions])) + elif i == 'Provides': + value.extend(["linux-tree-%s" % v.complete.replace('~', '-') for v in self.versions]) + entry[i] = value + return entry + + def write(self, packages, makefile): + self.write_config() + super(Gencontrol, self).write(packages, makefile) + + def write_config(self): + f = file("debian/config.defines.dump", 'w') + self.config.dump(f) + f.close() + +if __name__ == '__main__': + Gencontrol()() diff --git a/debian/bin/genorig.py b/debian/bin/genorig.py new file mode 100755 index 000000000..d0d598274 --- /dev/null +++ b/debian/bin/genorig.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python + +import sys +sys.path.append("debian/lib/python") + +import os, os.path, re, shutil +from debian_linux.debian import Changelog, VersionLinux +from debian_linux.patches import PatchSeries + +class Main(object): + def __init__(self, input_tar, input_patch, override_version): + self.log = sys.stdout.write + + self.input_tar = input_tar + self.input_patch = input_patch + + changelog = Changelog(version = VersionLinux)[0] + source = changelog.source + version = changelog.version + + if override_version: + version = VersionLinux('%s-undef' % override_version) + + self.version_dfsg = version.linux_dfsg + if self.version_dfsg is None: + self.version_dfsg = '0' + + self.log('Using source name %s, version %s, dfsg %s\n' % (source, version.upstream, self.version_dfsg)) + + self.orig = '%s-%s' % (source, version.upstream) + self.orig_tar = '%s_%s.orig.tar.gz' % (source, version.upstream) + + def __call__(self): + import tempfile + self.dir = tempfile.mkdtemp(prefix = 'genorig', dir = 'debian') + try: + self.upstream_extract() + self.upstream_patch() + self.debian_patch() + self.tar() + finally: + shutil.rmtree(self.dir) + + def upstream_extract(self): + self.log("Extracting tarball %s\n" % self.input_tar) + match = re.match(r'(^|.*/)(?Plinux-\d+\.\d+\.\d+(-\S+)?)\.tar(\.(?P(bz2|gz)))?$', self.input_tar) + if not match: + raise RuntimeError("Can't identify name of tarball") + cmdline = ['tar -xf', self.input_tar, '-C', self.dir] + if match.group('extension') == 'bz2': + cmdline.append('-j') + elif match.group('extension') == 'gz': + cmdline.append('-z') + if os.spawnv(os.P_WAIT, '/bin/sh', ['sh', '-c', ' '.join(cmdline)]): + raise RuntimeError("Can't extract tarball") + os.rename(os.path.join(self.dir, match.group('dir')), os.path.join(self.dir, self.orig)) + + def upstream_patch(self): + if self.input_patch is None: + return + self.log("Patching source with %s\n" % self.input_patch) + match = re.match(r'(^|.*/)patch-\d+\.\d+\.\d+(-\S+?)?(\.(?P(bz2|gz)))?$', self.input_patch) + if not match: + raise RuntimeError("Can't identify name of patch") + cmdline = [] + if match.group('extension') == 'bz2': + cmdline.append('bzcat') + elif match.group('extension') == 'gz': + cmdline.append('zcat') + else: + cmdline.append('cat') + cmdline.append(self.input_patch) + cmdline.append('| (cd %s; patch -p1 -f -s -t --no-backup-if-mismatch)' % os.path.join(self.dir, self.orig)) + if os.spawnv(os.P_WAIT, '/bin/sh', ['sh', '-c', ' '.join(cmdline)]): + raise RuntimeError("Can't patch source") + + def debian_patch(self): + name = "orig-" + self.version_dfsg + self.log("Patching source with debian patch (series %s)\n" % name) + fp = file("debian/patches/series/" + name) + series = PatchSeries(name, "debian/patches", fp) + series(dir = os.path.join(self.dir, self.orig)) + + def tar(self): + out = os.path.join("../orig", self.orig_tar) + try: + os.mkdir("../orig") + except OSError: pass + try: + os.stat(out) + raise RuntimeError("Destination already exists") + except OSError: pass + self.log("Generate tarball %s\n" % out) + cmdline = ['tar -czf', out, '-C', self.dir, self.orig] + try: + if os.spawnv(os.P_WAIT, '/bin/sh', ['sh', '-c', ' '.join(cmdline)]): + raise RuntimeError("Can't patch source") + os.chmod(out, 0644) + except: + try: + os.unlink(out) + except OSError: + pass + raise + +if __name__ == '__main__': + from optparse import OptionParser + parser = OptionParser(usage = "%prog [OPTION]... TAR [PATCH]") + parser.add_option("-V", "--override-version", dest = "override_version", help = "Override version", metavar = "VERSION") + options, args = parser.parse_args() + + input_tar = args[0] + input_patch = None + if len(args) > 1: + input_patch = args[1] + + Main(input_tar, input_patch, options.override_version)() diff --git a/debian/bin/kconfig.py b/debian/bin/kconfig.py new file mode 100755 index 000000000..a8fee4946 --- /dev/null +++ b/debian/bin/kconfig.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import optparse, os.path, sys +from debian_linux.kconfig import * + +def merge(output, *config): + kconfig = KconfigFile() + for c in config: + kconfig.read(file(c)) + file(output, "w").write(str(kconfig)) + +if __name__ == '__main__': + sys.exit(merge(*sys.argv[1:])) diff --git a/debian/lib/python/debian_linux/__init__.py b/debian/lib/python/debian_linux/__init__.py new file mode 100644 index 000000000..b785cebf7 --- /dev/null +++ b/debian/lib/python/debian_linux/__init__.py @@ -0,0 +1 @@ +# Module diff --git a/debian/lib/python/debian_linux/abi.py b/debian/lib/python/debian_linux/abi.py new file mode 100644 index 000000000..d04ad450d --- /dev/null +++ b/debian/lib/python/debian_linux/abi.py @@ -0,0 +1,51 @@ +class symbols(dict): + def __init__(self, filename = None): + self.modules = {} + if filename is not None: + self.read(file(filename)) + + def cmp(self, new): + symbols_ref = set(self.keys()) + symbols_new = set(new.keys()) + + symbols_add = {} + symbols_remove = {} + + symbols_change = {} + + for symbol in symbols_new - symbols_ref: + symbols_add[symbol] = new[symbol] + + for symbol in symbols_ref.intersection(symbols_new): + symbol_ref = self[symbol] + symbol_new = new[symbol] + + ent = {'ref': symbol_ref, 'new': symbol_new, 'changes': {}} + for i in ('module', 'version', 'export'): + if symbol_ref[i] != symbol_new[i]: + ent['changes'][i] = {'ref': symbol_ref, 'new': symbol_new} + if ent['changes']: + symbols_change[symbol] = ent + + for symbol in symbols_ref - symbols_new: + symbols_remove[symbol] = self[symbol] + + return symbols_add, symbols_change, symbols_remove + + def read(self, file): + for line in file.readlines(): + version, symbol, module, export = line.strip().split() + + if self.has_key(symbol): + pass + symbols = self.modules.get(module, set()) + symbols.add(symbol) + self.modules[module] = symbols + self[symbol] = {'symbol': symbol, 'module': module, 'version': version, 'export': export} + + def write(self, file): + symbols = self.items() + symbols.sort() + for symbol, info in symbols: + file.write("%(version)s %(symbol)s %(module)s %(export)s\n" % info) + diff --git a/debian/lib/python/debian_linux/config.py b/debian/lib/python/debian_linux/config.py new file mode 100644 index 000000000..00b2ce3aa --- /dev/null +++ b/debian/lib/python/debian_linux/config.py @@ -0,0 +1,238 @@ +import os, os.path, re, sys, textwrap + +__all__ = [ + 'ConfigCoreDump', + 'ConfigCoreHierarchy', + 'ConfigParser', +] + +class SchemaItemBoolean(object): + def __call__(self, i): + i = i.strip().lower() + if i in ("true", "1"): + return True + if i in ("false", "0"): + return False + raise Error + +class SchemaItemList(object): + def __init__(self, type = "\s+"): + self.type = type + + def __call__(self, i): + i = i.strip() + if not i: + return [] + return [j.strip() for j in re.split(self.type, i)] + +class ConfigCore(dict): + def merge(self, section, arch = None, featureset = None, flavour = None): + ret = {} + ret.update(self.get((section,), {})) + if featureset: + ret.update(self.get((section, None, featureset), {})) + if arch: + ret.update(self.get((section, arch), {})) + if arch and featureset: + ret.update(self.get((section, arch, featureset), {})) + if arch and featureset and flavour: + ret.update(self.get((section, arch, None, flavour), {})) + ret.update(self.get((section, arch, featureset, flavour), {})) + return ret + + def dump(self, fp): + sections = self.keys() + sections.sort() + for section in sections: + fp.write('[%r]\n' % (section,)) + items = self[section] + items_keys = items.keys() + items_keys.sort() + for item in items: + fp.write('%s: %r\n' % (item, items[item])) + fp.write('\n') + +class ConfigCoreDump(ConfigCore): + def __init__(self, config = None, fp = None): + super(ConfigCoreDump, self).__init__(self) + if config is not None: + self.update(config) + if fp is not None: + from ConfigParser import RawConfigParser + config = RawConfigParser() + config.readfp(fp) + for section in config.sections(): + section_real = eval(section) + data = {} + for key, value in config.items(section): + value_real = eval(value) + data[key] = value_real + self[section_real] = data + +class ConfigCoreHierarchy(ConfigCore): + config_name = "defines" + + schemas = { + 'base': { + 'arches': SchemaItemList(), + 'enabled': SchemaItemBoolean(), + 'featuresets': SchemaItemList(), + 'flavours': SchemaItemList(), + 'modules': SchemaItemBoolean(), + }, + 'image': { + 'configs': SchemaItemList(), + 'initramfs': SchemaItemBoolean(), + 'initramfs-generators': SchemaItemList(), + }, + 'relations': { + }, + 'xen': { + 'dom0-support': SchemaItemBoolean(), + 'versions': SchemaItemList(), + } + } + + def __init__(self, dirs = []): + super(ConfigCoreHierarchy, self).__init__() + self._dirs = dirs + self._read_base() + + def _read_arch(self, arch): + config = ConfigParser(self.schemas) + config.read(self.get_files("%s/%s" % (arch, self.config_name))) + + featuresets = config['base',].get('featuresets', []) + flavours = config['base',].get('flavours', []) + + for section in iter(config): + if section[0] in featuresets: + real = (section[-1], arch, section[0]) + elif len(section) > 1: + real = (section[-1], arch, None) + section[:-1] + else: + real = (section[-1], arch) + section[:-1] + s = self.get(real, {}) + s.update(config[section]) + self[tuple(real)] = s + + for featureset in featuresets: + self._read_arch_featureset(arch, featureset) + + if flavours: + base = self['base', arch] + featuresets.insert(0, 'none') + base['featuresets'] = featuresets + del base['flavours'] + self['base', arch] = base + self['base', arch, 'none'] = {'flavours': flavours, 'implicit-flavour': True} + + def _read_arch_featureset(self, arch, featureset): + config = ConfigParser(self.schemas) + config.read(self.get_files("%s/%s/%s" % (arch, featureset, self.config_name))) + + flavours = config['base',].get('flavours', []) + + for section in iter(config): + real = (section[-1], arch, featureset) + section[:-1] + s = self.get(real, {}) + s.update(config[section]) + self[tuple(real)] = s + + def _read_base(self): + config = ConfigParser(self.schemas) + config.read(self.get_files(self.config_name)) + + arches = config['base',]['arches'] + featuresets = config['base',].get('featuresets', []) + + for section in iter(config): + if section[0].startswith('featureset-'): + real = (section[-1], None, section[0].lstrip('featureset-')) + else: + real = (section[-1],) + section[1:] + self[real] = config[section] + + for arch in arches: + self._read_arch(arch) + for featureset in featuresets: + self._read_featureset(featureset) + + def _read_featureset(self, featureset): + config = ConfigParser(self.schemas) + config.read(self.get_files("featureset-%s/%s" % (featureset, self.config_name))) + + for section in iter(config): + real = (section[-1], None, featureset) + s = self.get(real, {}) + s.update(config[section]) + self[real] = s + + def get_files(self, name): + return [os.path.join(i, name) for i in self._dirs if i] + +class ConfigParser(object): + __slots__ = '_config', 'schemas' + + def __init__(self, schemas): + self.schemas = schemas + + from ConfigParser import RawConfigParser + self._config = config = RawConfigParser() + + def __getitem__(self, key): + return self._convert()[key] + + def __iter__(self): + return iter(self._convert()) + + def __str__(self): + return '<%s(%s)>' % (self.__class__.__name__, self._convert()) + + def _convert(self): + ret = {} + for section in self._config.sections(): + data = {} + for key, value in self._config.items(section): + data[key] = value + s1 = section.split('_') + if s1[-1] in self.schemas: + ret[tuple(s1)] = self.SectionSchema(data, self.schemas[s1[-1]]) + else: + ret[(section,)] = self.Section(data) + return ret + + def keys(self): + return self._convert().keys() + + def read(self, data): + return self._config.read(data) + + class Section(dict): + def __init__(self, data): + super(ConfigParser.Section, self).__init__(data) + + class SectionSchema(Section): + __slots__ = () + + def __init__(self, data, schema): + for key in data.keys(): + try: + data[key] = schema[key](data[key]) + except KeyError: pass + super(ConfigParser.SectionSchema, self).__init__(data) + +if __name__ == '__main__': + import sys + config = ConfigCoreHierarchy(['debian/config']) + sections = config.keys() + sections.sort() + for section in sections: + print "[%s]" % (section,) + items = config[section] + items_keys = items.keys() + items_keys.sort() + for item in items: + print "%s: %s" % (item, items[item]) + print + diff --git a/debian/lib/python/debian_linux/debian.py b/debian/lib/python/debian_linux/debian.py new file mode 100644 index 000000000..8783d090c --- /dev/null +++ b/debian/lib/python/debian_linux/debian.py @@ -0,0 +1,348 @@ +import itertools, os.path, re, utils + +class Changelog(list): + _rules = r""" +^ +(?P + \w[-+0-9a-z.]+ +) +\ +\( +(?P + [^\(\)\ \t]+ +) +\) +\s+ +(?P + [-+0-9a-zA-Z.]+ +) +\; +""" + _re = re.compile(_rules, re.X) + + class Entry(object): + __slot__ = 'distribution', 'source', 'version' + + def __init__(self, distribution, source, version): + self.distribution, self.source, self.version = distribution, source, version + + def __init__(self, dir = '', version = None): + if version is None: + version = Version + f = file(os.path.join(dir, "debian/changelog")) + while True: + line = f.readline() + if not line: + break + match = self._re.match(line) + if not match: + continue + try: + v = version(match.group('version')) + except Exception: + if not len(self): + raise + v = Version(match.group('version')) + self.append(self.Entry(match.group('distribution'), match.group('source'), v)) + +class Version(object): + _version_rules = ur""" +^ +(?: + (?P + \d+ + ) + : +)? +(?P + .+? +) +(?: + - + (?P[^-]+) +)? +$ +""" + _version_re = re.compile(_version_rules, re.X) + + def __init__(self, version): + match = self._version_re.match(version) + if match is None: + raise RuntimeError, "Invalid debian version" + self.epoch = None + if match.group("epoch") is not None: + self.epoch = int(match.group("epoch")) + self.upstream = match.group("upstream") + self.revision = match.group("revision") + + def __str__(self): + return self.complete + + @property + def complete(self): + if self.epoch is not None: + return "%d:%s" % (self.epoch, self.complete_noepoch) + return self.complete_noepoch + + @property + def complete_noepoch(self): + if self.revision is not None: + return "%s-%s" % (self.upstream, self.revision) + return self.upstream + + @property + def debian(self): + from warnings import warn + warn("debian argument was replaced by revision", DeprecationWarning, stacklevel = 2) + return self.revision + +class VersionLinux(Version): + _version_linux_rules = ur""" +^ +(?P + (?P\d+\.\d+) + \. + \d+ +) +(?: + ~ + (?P + .+? + ) +)? +(?: + \.dfsg\. + (?P + \d+ + ) +)? +- +(?:[^-]+) +$ +""" + _version_linux_re = re.compile(_version_linux_rules, re.X) + + def __init__(self, version): + super(VersionLinux, self).__init__(version) + match = self._version_linux_re.match(version) + if match is None: + raise RuntimeError, "Invalid debian linux version" + d = match.groupdict() + self.linux_major = d['major'] + self.linux_modifier = d['modifier'] + self.linux_version = d['version'] + if d['modifier'] is not None: + self.linux_upstream = '-'.join((d['version'], d['modifier'])) + else: + self.linux_upstream = d['version'] + self.linux_dfsg = d['dfsg'] + +class PackageFieldList(list): + def __init__(self, value = None): + self.extend(value) + + def __str__(self): + return ' '.join(self) + + def _extend(self, value): + if value is not None: + self.extend([j.strip() for j in re.split('\s', value.strip())]) + + def extend(self, value): + if isinstance(value, str): + self._extend(value) + else: + super(PackageFieldList, self).extend(value) + +class PackageDescription(object): + __slots__ = "short", "long" + + def __init__(self, value = None): + self.long = [] + if value is not None: + self.short, long = value.split("\n", 1) + self.append(long) + else: + self.short = None + + def __str__(self): + ret = self.short + '\n' + w = utils.TextWrapper(width = 74, fix_sentence_endings = True) + pars = [] + for i in self.long: + pars.append('\n '.join(w.wrap(i))) + return self.short + '\n ' + '\n .\n '.join(pars) + + def append(self, str): + str = str.strip() + if str: + self.long.extend(str.split("\n.\n")) + +class PackageRelation(list): + def __init__(self, value = None): + if value is not None: + self.extend(value) + + def __str__(self): + return ', '.join([str(i) for i in self]) + + def _match(self, value): + for i in self: + if i._match(value): + return i + return None + + def append(self, value): + if isinstance(value, basestring): + value = PackageRelationGroup(value) + elif not isinstance(value, PackageRelationGroup): + raise ValueError, "got %s" % type(value) + j = self._match(value) + if j: + j._updateArches(value) + else: + super(PackageRelation, self).append(value) + + def extend(self, value): + if isinstance(value, basestring): + value = [j.strip() for j in re.split(',', value.strip())] + elif not isinstance(value, (list, tuple)): + raise ValueError, "got %s" % type(value) + for i in value: + self.append(i) + +class PackageRelationGroup(list): + def __init__(self, value = None): + if value is not None: + self.extend(value) + + def __str__(self): + return ' | '.join([str(i) for i in self]) + + def _match(self, value): + for i, j in itertools.izip(self, value): + if i.name != j.name or i.version != j.version: + return None + return self + + def _updateArches(self, value): + for i, j in itertools.izip(self, value): + if i.arches: + for arch in j.arches: + if arch not in i.arches: + i.arches.append(arch) + + def append(self, value): + if isinstance(value, basestring): + value = PackageRelationEntry(value) + elif not isinstance(value, PackageRelationEntry): + raise ValueError + super(PackageRelationGroup, self).append(value) + + def extend(self, value): + if isinstance(value, basestring): + value = [j.strip() for j in re.split('\|', value.strip())] + elif not isinstance(value, (list, tuple)): + raise ValueError + for i in value: + self.append(i) + +class PackageRelationEntry(object): + __slots__ = "name", "operator", "version", "arches" + + _re = re.compile(r'^(\S+)(?: \((<<|<=|=|!=|>=|>>)\s*([^)]+)\))?(?: \[([^]]+)\])?$') + + class _operator(object): + OP_LT = 1; OP_LE = 2; OP_EQ = 3; OP_NE = 4; OP_GE = 5; OP_GT = 6 + operators = { '<<': OP_LT, '<=': OP_LE, '=': OP_EQ, '!=': OP_NE, '>=': OP_GE, '>>': OP_GT } + operators_neg = { OP_LT: OP_GE, OP_LE: OP_GT, OP_EQ: OP_NE, OP_NE: OP_EQ, OP_GE: OP_LT, OP_GT: OP_LE } + operators_text = dict([(b, a) for a, b in operators.iteritems()]) + + __slots__ = '_op', + + def __init__(self, value): + self._op = self.operators[value] + + def __neg__(self): + return self.__class__(self.operators_text[self.operators_neg[self._op]]) + + def __str__(self): + return self.operators_text[self._op] + + def __init__(self, value = None): + if isinstance(value, basestring): + self.parse(value) + else: + raise ValueError + + def __str__(self): + ret = [self.name] + if self.operator is not None and self.version is not None: + ret.extend([' (', str(self.operator), ' ', self.version, ')']) + if self.arches: + ret.extend([' [', ' '.join(self.arches), ']']) + return ''.join(ret) + + def parse(self, value): + match = self._re.match(value) + if match is None: + raise RuntimeError, "Can't parse dependency %s" % value + match = match.groups() + self.name = match[0] + if match[1] is not None: + self.operator = self._operator(match[1]) + else: + self.operator = None + self.version = match[2] + if match[3] is not None: + self.arches = re.split('\s+', match[3]) + else: + self.arches = [] + +class Package(dict): + _fields = utils.SortedDict(( + ('Package', str), + ('Source', str), + ('Architecture', PackageFieldList), + ('Section', str), + ('Priority', str), + ('Maintainer', str), + ('Uploaders', str), + ('Standards-Version', str), + ('Build-Depends', PackageRelation), + ('Build-Depends-Indep', PackageRelation), + ('Provides', PackageRelation), + ('Pre-Depends', PackageRelation), + ('Depends', PackageRelation), + ('Recommends', PackageRelation), + ('Suggests', PackageRelation), + ('Replaces', PackageRelation), + ('Conflicts', PackageRelation), + ('Description', PackageDescription), + )) + + def __setitem__(self, key, value): + try: + cls = self._fields[key] + if not isinstance(value, cls): + value = cls(value) + except KeyError: pass + super(Package, self).__setitem__(key, value) + + def iterkeys(self): + keys = set(self.keys()) + for i in self._fields.iterkeys(): + if self.has_key(i): + keys.remove(i) + yield i + for i in keys: + yield i + + def iteritems(self): + for i in self.iterkeys(): + yield (i, self[i]) + + def itervalues(self): + for i in self.iterkeys(): + yield self[i] + diff --git a/debian/lib/python/debian_linux/gencontrol.py b/debian/lib/python/debian_linux/gencontrol.py new file mode 100644 index 000000000..be394308e --- /dev/null +++ b/debian/lib/python/debian_linux/gencontrol.py @@ -0,0 +1,310 @@ +from debian import * +from utils import SortedDict + +class PackagesList(SortedDict): + def append(self, package): + self[package['Package']] = package + + def extend(self, packages): + for package in packages: + self[package['Package']] = package + +class Makefile(object): + def __init__(self): + self.rules = {} + self.add('.NOTPARALLEL') + + def add(self, name, deps = None, cmds = None): + if name in self.rules: + self.rules[name].add(deps, cmds) + else: + self.rules[name] = self.Rule(name, deps, cmds) + if deps is not None: + for i in deps: + if i not in self.rules: + self.rules[i] = self.Rule(i) + + def write(self, out): + r = self.rules.keys() + r.sort() + for i in r: + self.rules[i].write(out) + + class Rule(object): + def __init__(self, name, deps = None, cmds = None): + self.name = name + self.deps, self.cmds = set(), [] + self.add(deps, cmds) + + def add(self, deps = None, cmds = None): + if deps is not None: + self.deps.update(deps) + if cmds is not None: + self.cmds.append(cmds) + + def write(self, out): + deps_string = '' + if self.deps: + deps = list(self.deps) + deps.sort() + deps_string = ' ' + ' '.join(deps) + + if self.cmds: + if deps_string: + out.write('%s::%s\n' % (self.name, deps_string)) + for c in self.cmds: + out.write('%s::\n' % self.name) + for i in c: + out.write('\t%s\n' % i) + else: + out.write('%s:%s\n' % (self.name, deps_string)) + +class MakeFlags(dict): + def __repr__(self): + repr = super(flags, self).__repr__() + return "%s(%s)" % (self.__class__.__name__, repr) + + def __str__(self): + return ' '.join(["%s='%s'" % i for i in self.iteritems()]) + + def copy(self): + return self.__class__(super(MakeFlags, self).copy()) + +class Gencontrol(object): + makefile_targets = ('binary-arch', 'build', 'setup', 'source') + + def __init__(self, config, templates, version = Version): + self.config, self.templates = config, templates + self.changelog = Changelog(version = version) + + def __call__(self): + packages = PackagesList() + makefile = Makefile() + + self.do_source(packages) + self.do_main(packages, makefile) + self.do_extra(packages, makefile) + + self.write(packages, makefile) + + def do_source(self, packages): + source = self.templates["control.source"][0] + source['Source'] = self.changelog[0].source + packages['source'] = self.process_package(source, self.vars) + + def do_main(self, packages, makefile): + config_entry = self.config['base',] + vars = self.vars.copy() + vars.update(config_entry) + + makeflags = MakeFlags() + extra = {} + + self.do_main_setup(vars, makeflags, extra) + self.do_main_packages(packages, extra) + self.do_main_makefile(makefile, makeflags, extra) + + for arch in iter(self.config['base',]['arches']): + self.do_arch(packages, makefile, arch, vars.copy(), makeflags.copy(), extra) + + def do_main_setup(self, vars, makeflags, extra): + pass + + def do_main_makefile(self, makefile, makeflags, extra): + makefile.add('binary-indep', cmds = ["$(MAKE) -f debian/rules.real binary-indep %s" % makeflags]) + + def do_main_packages(self, packages, extra): + pass + + def do_extra(self, packages, makefile): + templates_extra = self.templates.get("control.extra", None) + if templates_extra is None: + return + + packages.extend(self.process_packages(templates_extra, {})) + extra_arches = {} + for package in templates_extra: + arches = package['Architecture'] + for arch in arches: + i = extra_arches.get(arch, []) + i.append(package) + extra_arches[arch] = i + archs = extra_arches.keys() + archs.sort() + for arch in archs: + cmds = [] + for i in extra_arches[arch]: + tmp = [] + if i.has_key('X-Version-Overwrite-Epoch'): + tmp.append("-v1:%s" % self.version['source']) + cmds.append("$(MAKE) -f debian/rules.real install-dummy DH_OPTIONS='-p%s' GENCONTROL_ARGS='%s'" % (i['Package'], ' '.join(tmp))) + makefile.add('binary-arch_%s' % arch ['binary-arch_%s_extra' % arch]) + makefile.add("binary-arch_%s_extra" % arch, cmds = cmds) + + def do_arch(self, packages, makefile, arch, vars, makeflags, extra): + config_base = self.config['base', arch] + vars.update(config_base) + vars['arch'] = arch + + makeflags['ARCH'] = arch + + self.do_arch_setup(vars, makeflags, arch, extra) + self.do_arch_makefile(makefile, arch, makeflags, extra) + self.do_arch_packages(packages, makefile, arch, vars, makeflags, extra) + self.do_arch_recurse(packages, makefile, arch, vars, makeflags, extra) + + def do_arch_setup(self, vars, makeflags, arch, extra): + pass + + def do_arch_makefile(self, makefile, arch, makeflags, extra): + for i in self.makefile_targets: + target1 = i + target2 = '_'.join((target1, arch)) + target3 = '_'.join((target2, 'real')) + makefile.add(target1, [target2]) + makefile.add(target2, [target3]) + + def do_arch_packages(self, packages, makefile, arch, vars, makeflags, extra): + pass + + def do_arch_recurse(self, packages, makefile, arch, vars, makeflags, extra): + for featureset in self.config['base', arch]['featuresets']: + self.do_featureset(packages, makefile, arch, featureset, vars.copy(), makeflags.copy(), extra) + + def do_featureset(self, packages, makefile, arch, featureset, vars, makeflags, extra): + config_base = self.config.merge('base', arch, featureset) + vars.update(config_base) + + if not config_base.get('enabled', True): + return + + makeflags['FEATURESET'] = featureset + + vars['localversion'] = '' + if featureset != 'none': + vars['localversion'] = '-' + featureset + + self.do_featureset_setup(vars, makeflags, arch, featureset, extra) + self.do_featureset_makefile(makefile, arch, featureset, makeflags, extra) + self.do_featureset_packages(packages, makefile, arch, featureset, vars, makeflags, extra) + self.do_featureset_recurse(packages, makefile, arch, featureset, vars, makeflags, extra) + + def do_featureset_setup(self, vars, makeflags, arch, featureset, extra): + pass + + def do_featureset_makefile(self, makefile, arch, featureset, makeflags, extra): + for i in self.makefile_targets: + target1 = '_'.join((i, arch)) + target2 = '_'.join((target1, featureset)) + target3 = '_'.join((target2, 'real')) + makefile.add(target1, [target2]) + makefile.add(target2, [target3]) + + def do_featureset_packages(self, packages, makefile, arch, featureset, vars, makeflags, extra): + pass + + def do_featureset_recurse(self, packages, makefile, arch, featureset, vars, makeflags, extra): + for flavour in self.config['base', arch, featureset]['flavours']: + self.do_flavour(packages, makefile, arch, featureset, flavour, vars.copy(), makeflags.copy(), extra) + + def do_flavour(self, packages, makefile, arch, featureset, flavour, vars, makeflags, extra): + config_base = self.config.merge('base', arch, featureset, flavour) + vars.update(config_base) + + if not vars.has_key('longclass'): + vars['longclass'] = vars['class'] + + makeflags['FLAVOUR'] = flavour + vars['localversion'] += '-' + flavour + + self.do_flavour_setup(vars, makeflags, arch, featureset, flavour, extra) + self.do_flavour_makefile(makefile, arch, featureset, flavour, makeflags, extra) + self.do_flavour_packages(packages, makefile, arch, featureset, flavour, vars, makeflags, extra) + + def do_flavour_setup(self, vars, makeflags, arch, featureset, flavour, extra): + for i in ( + ('kernel-arch', 'KERNEL_ARCH'), + ('localversion', 'LOCALVERSION'), + ): + if vars.has_key(i[0]): + makeflags[i[1]] = vars[i[0]] + + def do_flavour_makefile(self, makefile, arch, featureset, flavour, makeflags, extra): + for i in self.makefile_targets: + target1 = '_'.join((i, arch, featureset)) + target2 = '_'.join((target1, flavour)) + target3 = '_'.join((target2, 'real')) + makefile.add(target1, [target2]) + makefile.add(target2, [target3]) + + def do_flavour_packages(self, packages, makefile, arch, featureset, flavour, vars, makeflags, extra): + pass + + def process_relation(self, key, e, in_e, vars): + import copy + dep = copy.deepcopy(in_e[key]) + for groups in dep: + for item in groups: + item.name = self.substitute(item.name, vars) + e[key] = dep + + def process_description(self, e, in_e, vars): + in_desc = in_e['Description'] + desc = in_desc.__class__() + desc.short = self.substitute(in_desc.short, vars) + for i in in_desc.long: + desc.append(self.substitute(i, vars)) + e['Description'] = desc + + def process_package(self, in_entry, vars): + e = Package() + for key, value in in_entry.iteritems(): + if isinstance(value, PackageRelation): + self.process_relation(key, e, in_entry, vars) + elif key == 'Description': + self.process_description(e, in_entry, vars) + elif key[:2] == 'X-': + pass + else: + e[key] = self.substitute(value, vars) + return e + + def process_packages(self, in_entries, vars): + entries = [] + for i in in_entries: + entries.append(self.process_package(i, vars)) + return entries + + def substitute(self, s, vars): + if isinstance(s, (list, tuple)): + for i in xrange(len(s)): + s[i] = self.substitute(s[i], vars) + return s + def subst(match): + return vars[match.group(1)] + return re.sub(r'@([-_a-z]+)@', subst, s) + + def write(self, packages, makefile): + self.write_control(packages.itervalues()) + self.write_makefile(makefile) + + def write_config(self): + f = file("debian/config.dump", 'w') + self.config.write(f) + f.close() + + def write_control(self, list): + self.write_rfc822(file("debian/control", 'w'), list) + + def write_makefile(self, makefile): + f = file("debian/rules.gen", 'w') + makefile.write(f) + f.close() + + def write_rfc822(self, f, list): + for entry in list: + for key, value in entry.iteritems(): + f.write("%s: %s\n" % (key, value)) + f.write('\n') + + diff --git a/debian/lib/python/debian_linux/kconfig.py b/debian/lib/python/debian_linux/kconfig.py new file mode 100644 index 000000000..86542e56f --- /dev/null +++ b/debian/lib/python/debian_linux/kconfig.py @@ -0,0 +1,70 @@ +__all__ = ( + "KconfigFile", +) + +class EntryString(object): + __slots__ = "name", "value" + + def __init__(self, name, value): + self.name = name + self.value = value + + def __str__(self): + return "CONFIG_%s=%s" % (self.name, self.value) + +class EntryTristate(object): + __slots__ = "name", "value" + + VALUE_NO = 0 + VALUE_YES = 1 + VALUE_MOD = 2 + + def __init__(self, name, value = None): + self.name = name + if value == 'n' or value is None: + self.value = self.VALUE_NO + elif value == 'y': + self.value = self.VALUE_YES + elif value == 'm': + self.value = self.VALUE_MOD + + def __str__(self): + conf = "CONFIG_%s" % self.name + if self.value == self.VALUE_NO: + return "# %s is not set" % conf + elif self.value == self.VALUE_YES: + return "%s=y" % conf + elif self.value == self.VALUE_MOD: + return "%s=m" % conf + +class KconfigFile(dict): + def __str__(self): + ret = [] + for i in self.str_iter(): + ret.append(i) + return '\n'.join(ret) + '\n' + + def read(self, f): + for line in iter(f.readlines()): + line = line.strip() + if line.startswith("CONFIG_"): + i = line.find('=') + option = line[7:i] + value = line[i+1:] + if value in ('y', 'm'): + entry = EntryTristate(option, value) + else: + entry = EntryString(option, value) + self[option] = entry + elif line.startswith("# CONFIG_"): + option = line[9:-11] + self[option] = EntryTristate(option) + elif line.startswith("#") or not line: + pass + else: + raise RuntimeError, "Can't recognize %s" % line + + def str_iter(self): + for key, value in self.iteritems(): + yield str(value) + diff --git a/debian/lib/python/debian_linux/patches.py b/debian/lib/python/debian_linux/patches.py new file mode 100644 index 000000000..9981274d2 --- /dev/null +++ b/debian/lib/python/debian_linux/patches.py @@ -0,0 +1,206 @@ +import os, shutil + +class Operation(object): + def __init__(self, name, data): + self.name, self.data = name, data + + def __call__(self, dir = '.', reverse = False): + try: + if not reverse: + self.do(dir) + else: + self.do_reverse(dir) + self._log(True) + except: + self._log(False) + raise + + def _log(self, result): + if result: + s = "OK" + else: + s = "FAIL" + print """ (%s) %-4s %s""" % (self.operation, s, self.name) + + def do(self, dir): + raise NotImplementedError + + def do_reverse(self, dir): + raise NotImplementedError + +class OperationPatch(Operation): + def __init__(self, name, fp, data): + super(OperationPatch, self).__init__(name, data) + self.fp = fp + + def _call(self, dir, extraargs): + cmdline = "cd %s; patch -p1 -f -s -t --no-backup-if-mismatch %s" % (dir, extraargs) + f = os.popen(cmdline, 'wb') + shutil.copyfileobj(self.fp, f) + if f.close(): + raise RuntimeError("Patch failed") + + def patch_push(self, dir): + self._call(dir, '--fuzz=1') + + def patch_pop(self, dir): + self._call(dir, '-R') + +class OperationPatchPush(OperationPatch): + operation = '+' + + do = OperationPatch.patch_push + do_reverse = OperationPatch.patch_pop + +class OperationPatchPop(OperationPatch): + operation = '-' + + do = OperationPatch.patch_pop + do_reverse = OperationPatch.patch_push + +class SubOperation(Operation): + def _log(self, result): + if result: + s = "OK" + else: + s = "FAIL" + print """ %-10s %-4s %s""" % ('(%s)' % self.operation, s, self.name) + +class SubOperationFilesRemove(SubOperation): + operation = "remove" + + def do(self, dir): + dir = os.path.join(dir, self.name) + if os.path.isdir(dir): + shutil.rmtree(dir) + else: + os.unlink(dir) + +class SubOperationFilesUnifdef(SubOperation): + operation = "unifdef" + + def do(self, dir): + filename = os.path.join(dir, self.name) + cmdline = "unifdef %s %s" % (filename, ' '.join(self.data)) + f = os.popen(cmdline, 'rb') + data = f.read() + ret = f.close() + if ret is None: + raise RuntimeError("unifdef of %s removed nothing" % self.name) + elif ret != 256: + raise RuntimeError("unifdef failed") + f1 = file(filename, 'wb') + f1.write(data) + f1.close() + +class OperationFiles(Operation): + operation = 'X' + + suboperations = { + 'remove': SubOperationFilesRemove, + 'rm': SubOperationFilesRemove, + 'unifdef': SubOperationFilesUnifdef, + } + + def __init__(self, name, fp, data): + super(OperationFiles, self).__init__(name, data) + + ops = [] + + for line in fp: + line = line.strip() + if not line or line[0] == '#': + continue + + items = line.split() + operation, filename = items[:2] + data = items[2:] + + if operation not in self.suboperations: + raise RuntimeError('Undefined operation "%s" in series %s' % (operation, name)) + + ops.append(self.suboperations[operation](filename, data)) + + self.ops = ops + + def do(self, dir): + for i in self.ops: + i(dir = dir) + +class PatchSeries(list): + operations = { + '+': OperationPatchPush, + '-': OperationPatchPop, + 'X': OperationFiles, + } + + def __init__(self, name, root, fp): + self.name, self.root = name, root + + from gzip import GzipFile + from bz2 import BZ2File + + for line in fp: + line = line.strip() + + if not len(line) or line[0] == '#': + continue + + items = line.split(' ') + operation, filename = items[:2] + data = items[2:] + + if operation in self.operations: + f = os.path.join(self.root, filename) + for suffix, cls in (('', file), ('.bz2', BZ2File), ('.gz', GzipFile)): + f1 = f + suffix + if os.path.exists(f1): + fp = cls(f1) + break + else: + raise RuntimeError("Can't find patch %s for series %s" % (filename, self.name)) + else: + raise RuntimeError('Undefined operation "%s" in series %s' % (operation, name)) + + self.append(self.operations[operation](filename, fp, data)) + + def __call__(self, cond = bool, dir = '.', reverse = False): + if not reverse: + l = self + else: + l = self[::-1] + for i in l: + if cond(i): + i(dir = dir, reverse = reverse) + + def __repr__(self): + return '<%s object for %s>' % (self.__class__.__name__, self.name) + +class PatchSeriesList(list): + def __call__(self, cond = bool, reverse = False): + if not reverse: + l = self + else: + l = self[::-1] + for i in self: + if reverse: + print "--> Try to unapply %s." % i.name + else: + print "--> Try to apply %s." % i.name + i(cond = cond, reverse = reverse) + if reverse: + print "--> %s fully unapplied." % i.name + else: + print "--> %s fully applied." % i.name + + @classmethod + def read(cls, home, files): + ret = cls() + for i in files: + try: + fp = file(os.path.join(home, 'series', i)) + ret.append(PatchSeries(i, home, fp)) + except IOError: + pass + return ret + diff --git a/debian/lib/python/debian_linux/utils.py b/debian/lib/python/debian_linux/utils.py new file mode 100644 index 000000000..b0ff1a6e2 --- /dev/null +++ b/debian/lib/python/debian_linux/utils.py @@ -0,0 +1,108 @@ +import debian, re, os, textwrap + +_marker = object + +class SortedDict(dict): + __slots__ = '_list', + + def __init__(self, entries = None): + super(SortedDict, self).__init__() + self._list = [] + if entries is not None: + for key, value in entries: + self[key] = value + + def __delitem__(self, key): + super(SortedDict, self).__delitem__(key) + self._list.remove(key) + + def __setitem__(self, key, value): + super(SortedDict, self).__setitem__(key, value) + if key not in self._list: + self._list.append(key) + + def iterkeys(self): + for i in iter(self._list): + yield i + + def iteritems(self): + for i in iter(self._list): + yield (i, self[i]) + + def itervalues(self): + for i in iter(self._list): + yield self[i] + +class Templates(dict): + def __init__(self, dirs = ["debian/templates"]): + self.dirs = dirs + + def __getitem__(self, key): + return self.get(key) + + def __setitem__(self, key, value): + raise NotImplemented() + + def _read(self, name): + prefix, id = name.split('.', 1) + + for dir in self.dirs: + filename = "%s/%s.in" % (dir, name) + if os.path.exists(filename): + f = file(filename) + if prefix == 'control': + return self._read_control(f) + return f.read() + + def _read_control(self, f): + entries = [] + + while True: + e = debian.Package() + last = None + lines = [] + while True: + line = f.readline() + if not line: + break + line = line.strip('\n') + if not line: + break + if line[0] in ' \t': + if not last: + raise ValueError('Continuation line seen before first header') + lines.append(line.lstrip()) + continue + if last: + e[last] = '\n'.join(lines) + i = line.find(':') + if i < 0: + raise ValueError("Not a header, not a continuation: ``%s''" % line) + last = line[:i] + lines = [line[i+1:].lstrip()] + if last: + e[last] = '\n'.join(lines) + if not e: + break + + entries.append(e) + + return entries + + def get(self, key, default = _marker): + ret = super(Templates, self).get(key, _marker) + if ret is not _marker: + return ret + value = self._read(key) + if value is None: + if default is _marker: + raise KeyError(key) + return default + super(Templates, self).__setitem__(key, value) + return value + +class TextWrapper(textwrap.TextWrapper): + wordsep_re = re.compile( + r'(\s+|' # any whitespace + r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash +