diff --git a/openerp-server b/openerp-server index 1486c493ff5..3586b11c7d2 100755 --- a/openerp-server +++ b/openerp-server @@ -39,10 +39,11 @@ import signal import sys import threading import traceback +import time -import openerp.release as release -__author__ = release.author -__version__ = release.version +import openerp +__author__ = openerp.release.author +__version__ = openerp.release.version if os.name == 'posix': import pwd @@ -53,80 +54,58 @@ if os.name == 'posix': sys.exit(1) #----------------------------------------------------------------------- -# import the tools module so that the commandline parameters are parsed +# parse the command line #----------------------------------------------------------------------- -import openerp.tools as tools -tools.config.parse_config(sys.argv[1:]) +openerp.tools.config.parse_config(sys.argv[1:]) +config = openerp.tools.config #---------------------------------------------------------- # get logger #---------------------------------------------------------- -import openerp.netsvc as netsvc -netsvc.init_logger() +openerp.netsvc.init_logger() logger = logging.getLogger('server') -logger.info("OpenERP version - %s", release.version) -for name, value in [('addons_path', tools.config['addons_path']), - ('database hostname', tools.config['db_host'] or 'localhost'), - ('database port', tools.config['db_port'] or '5432'), - ('database user', tools.config['db_user'])]: +logger.info("OpenERP version - %s", __version__) +for name, value in [('addons_path', config['addons_path']), + ('database hostname', config['db_host'] or 'localhost'), + ('database port', config['db_port'] or '5432'), + ('database user', config['db_user'])]: logger.info("%s - %s", name, value) # Don't allow if the connection to PostgreSQL done by postgres user -if tools.config['db_user'] == 'postgres': +if config['db_user'] == 'postgres': logger.error("Connecting to the database as 'postgres' user is forbidden, as it present major security issues. Shutting down.") sys.exit(1) -import time - #---------------------------------------------------------- # init net service #---------------------------------------------------------- logger.info('initialising distributed objects services') -#--------------------------------------------------------------- -# connect to the database and initialize it with base if needed -#--------------------------------------------------------------- -import openerp.pooler as pooler - -#---------------------------------------------------------- -# import basic modules -#---------------------------------------------------------- -import openerp.osv as osv -import openerp.workflow as workflow -import openerp.report as report -import openerp.service as service - -#---------------------------------------------------------- -# import addons -#---------------------------------------------------------- - -import openerp.addons as addons - #---------------------------------------------------------- # Load and update databases if requested #---------------------------------------------------------- -import openerp.service.http_server as service_http_server +if not ( config["stop_after_init"] or \ + config["translate_in"] or \ + config["translate_out"] ): + openerp.osv.osv.start_object_proxy() + openerp.service.web_services.start_web_services() + http_server = openerp.service.http_server + netrpc_server = openerp.service.netrpc_server + http_server.init_servers() + http_server.init_xmlrpc() + http_server.init_static_http() + netrpc_server.init_servers() -if not ( tools.config["stop_after_init"] or \ - tools.config["translate_in"] or \ - tools.config["translate_out"] ): - service_http_server.init_servers() - service_http_server.init_xmlrpc() - service_http_server.init_static_http() - - import openerp.service.netrpc_server as service_netrpc_server - service_netrpc_server.init_servers() - -if tools.config['db_name']: - for dbname in tools.config['db_name'].split(','): - db,pool = pooler.get_db_and_pool(dbname, update_module=tools.config['init'] or tools.config['update'], pooljobs=False) +if config['db_name']: + for dbname in config['db_name'].split(','): + db, pool = openerp.pooler.get_db_and_pool(dbname, update_module=config['init'] or config['update'], pooljobs=False) cr = db.cursor() - if tools.config["test_file"]: - logger.info('loading test file %s', tools.config["test_file"]) - tools.convert_yaml_import(cr, 'base', file(tools.config["test_file"]), {}, 'test', True) + if config["test_file"]: + logger.info('loading test file %s', config["test_file"]) + openerp.tools.convert_yaml_import(cr, 'base', file(config["test_file"]), {}, 'test', True) cr.rollback() pool.get('ir.cron')._poolJobs(db.dbname) @@ -136,35 +115,33 @@ if tools.config['db_name']: #---------------------------------------------------------- # translation stuff #---------------------------------------------------------- -if tools.config["translate_out"]: - import csv - - if tools.config["language"]: - msg = "language %s" % (tools.config["language"],) +if config["translate_out"]: + if config["language"]: + msg = "language %s" % (config["language"],) else: msg = "new language" - logger.info('writing translation file for %s to %s', msg, tools.config["translate_out"]) + logger.info('writing translation file for %s to %s', msg, config["translate_out"]) - fileformat = os.path.splitext(tools.config["translate_out"])[-1][1:].lower() - buf = file(tools.config["translate_out"], "w") - dbname = tools.config['db_name'] - cr = pooler.get_db(dbname).cursor() - tools.trans_export(tools.config["language"], tools.config["translate_modules"] or ["all"], buf, fileformat, cr) + fileformat = os.path.splitext(config["translate_out"])[-1][1:].lower() + buf = file(config["translate_out"], "w") + dbname = config['db_name'] + cr = openerp.pooler.get_db(dbname).cursor() + openerp.tools.trans_export(config["language"], config["translate_modules"] or ["all"], buf, fileformat, cr) cr.close() buf.close() logger.info('translation file written successfully') sys.exit(0) -if tools.config["translate_in"]: - context = {'overwrite': tools.config["overwrite_existing_translations"]} - dbname = tools.config['db_name'] - cr = pooler.get_db(dbname).cursor() - tools.trans_load(cr, - tools.config["translate_in"], - tools.config["language"], +if config["translate_in"]: + context = {'overwrite': config["overwrite_existing_translations"]} + dbname = config['db_name'] + cr = openerp.pooler.get_db(dbname).cursor() + openerp.tools.trans_load(cr, + config["translate_in"], + config["language"], context=context) - tools.trans_update_res_ids(cr) + openerp.tools.trans_update_res_ids(cr) cr.commit() cr.close() sys.exit(0) @@ -172,9 +149,10 @@ if tools.config["translate_in"]: #---------------------------------------------------------------------------------- # if we don't want the server to continue to run after initialization, we quit here #---------------------------------------------------------------------------------- -if tools.config["stop_after_init"]: +if config["stop_after_init"]: sys.exit(0) +openerp.netsvc.start_agent() #---------------------------------------------------------- # Launch Servers @@ -186,15 +164,16 @@ SIGNALS = dict( [(getattr(signal, sign), sign) for sign in LST_SIGNALS] ) -netsvc.quit_signals_received = 0 +quit_signals_received = 0 def handler(signum, frame): """ :param signum: the signal number :param frame: the interrupted stack frame or None """ - netsvc.quit_signals_received += 1 - if netsvc.quit_signals_received > 1: + global quit_signals_received + quit_signals_received += 1 + if quit_signals_received > 1: sys.stderr.write("Forced shutdown.\n") os._exit(0) @@ -219,10 +198,10 @@ if os.name == 'posix': signal.signal(signal.SIGQUIT, dumpstacks) def quit(): - netsvc.Agent.quit() - netsvc.Server.quitAll() - if tools.config['pidfile']: - os.unlink(tools.config['pidfile']) + openerp.netsvc.Agent.quit() + openerp.netsvc.Server.quitAll() + if config['pidfile']: + os.unlink(config['pidfile']) logger = logging.getLogger('shutdown') logger.info("Initiating OpenERP Server shutdown") logger.info("Hit CTRL-C again or send a second signal to immediately terminate the server...") @@ -240,17 +219,17 @@ def quit(): time.sleep(0.05) sys.exit(0) -if tools.config['pidfile']: - fd = open(tools.config['pidfile'], 'w') +if config['pidfile']: + fd = open(config['pidfile'], 'w') pidtext = "%d" % (os.getpid()) fd.write(pidtext) fd.close() -netsvc.Server.startAll() +openerp.netsvc.Server.startAll() logger.info('OpenERP server is running, waiting for connections...') -while netsvc.quit_signals_received == 0: +while quit_signals_received == 0: time.sleep(60) quit() diff --git a/openerp/__init__.py b/openerp/__init__.py index 0e098a163c8..b7b5fc1b533 100644 --- a/openerp/__init__.py +++ b/openerp/__init__.py @@ -19,8 +19,14 @@ # ############################################################################## +""" OpenERP core library. + +""" + import addons +import conf import loglevels +import modules import netsvc import osv import pooler diff --git a/openerp/addons/__init__.py b/openerp/addons/__init__.py index f91d66eb7a7..88a0d113151 100644 --- a/openerp/addons/__init__.py +++ b/openerp/addons/__init__.py @@ -20,960 +20,16 @@ # ############################################################################## -import os, sys, imp -from os.path import join as opj -import itertools -import zipimport +""" Addons module. -import openerp.osv as osv -import openerp.tools as tools -import openerp.tools.osutil as osutil -from openerp.tools.safe_eval import safe_eval as eval -import openerp.pooler as pooler -from openerp.tools.translate import _ +This module only serves to contain OpenERP addons. For the code to +manage those addons, see openerp.modules. This module conveniently +reexports some symbols from openerp.modules. Importing them from here +is deprecated. -import openerp.netsvc as netsvc - -import zipfile -import openerp.release as release - -import re -import base64 -from zipfile import PyZipFile, ZIP_DEFLATED -from cStringIO import StringIO - -import logging - -logger = netsvc.Logger() - -_ad = os.path.dirname(__file__) # default addons path (base) -ad_paths = [] - -# Modules already loaded -loaded = [] - -def initialize_sys_path(): - global ad_paths - - if ad_paths: - return - - ad_paths = map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(',')) - - sys.path.insert(1, _ad) - - ad_cnt=1 - for adp in ad_paths: - if adp != _ad: - sys.path.insert(ad_cnt, adp) - ad_cnt+=1 - - ad_paths.append(_ad) # for get_module_path - -class Graph(dict): - - def addNode(self, name, deps): - max_depth, father = 0, None - for n in [Node(x, self) for x in deps]: - if n.depth >= max_depth: - father = n - max_depth = n.depth - if father: - father.addChild(name) - else: - Node(name, self) - - def update_from_db(self, cr): - if not len(self): - return - # update the graph with values from the database (if exist) - ## First, we set the default values for each package in graph - additional_data = dict.fromkeys(self.keys(), {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None}) - ## Then we get the values from the database - cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version' - ' FROM ir_module_module' - ' WHERE name IN %s',(tuple(additional_data),) - ) - - ## and we update the default values with values from the database - additional_data.update(dict([(x.pop('name'), x) for x in cr.dictfetchall()])) - - for package in self.values(): - for k, v in additional_data[package.name].items(): - setattr(package, k, v) - - def __iter__(self): - level = 0 - done = set(self.keys()) - while done: - level_modules = [(name, module) for name, module in self.items() if module.depth==level] - for name, module in level_modules: - done.remove(name) - yield module - level += 1 - -class Singleton(object): - def __new__(cls, name, graph): - if name in graph: - inst = graph[name] - else: - inst = object.__new__(cls) - inst.name = name - graph[name] = inst - return inst - - -class Node(Singleton): - - def __init__(self, name, graph): - self.graph = graph - if not hasattr(self, 'children'): - self.children = [] - if not hasattr(self, 'depth'): - self.depth = 0 - - def addChild(self, name): - node = Node(name, self.graph) - node.depth = self.depth + 1 - if node not in self.children: - self.children.append(node) - for attr in ('init', 'update', 'demo'): - if hasattr(self, attr): - setattr(node, attr, True) - self.children.sort(lambda x, y: cmp(x.name, y.name)) - - def __setattr__(self, name, value): - super(Singleton, self).__setattr__(name, value) - if name in ('init', 'update', 'demo'): - tools.config[name][self.name] = 1 - for child in self.children: - setattr(child, name, value) - if name == 'depth': - for child in self.children: - setattr(child, name, value + 1) - - def __iter__(self): - return itertools.chain(iter(self.children), *map(iter, self.children)) - - def __str__(self): - return self._pprint() - - def _pprint(self, depth=0): - s = '%s\n' % self.name - for c in self.children: - s += '%s`-> %s' % (' ' * depth, c._pprint(depth+1)) - return s - - -def get_module_path(module, downloaded=False): - """Return the path of the given module.""" - initialize_sys_path() - for adp in ad_paths: - if os.path.exists(opj(adp, module)) or os.path.exists(opj(adp, '%s.zip' % module)): - return opj(adp, module) - - if downloaded: - return opj(_ad, module) - logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: module not found' % (module,)) - return False - - -def get_module_filetree(module, dir='.'): - path = get_module_path(module) - if not path: - return False - - dir = os.path.normpath(dir) - if dir == '.': - dir = '' - if dir.startswith('..') or (dir and dir[0] == '/'): - raise Exception('Cannot access file outside the module') - - if not os.path.isdir(path): - # zipmodule - zip = zipfile.ZipFile(path + ".zip") - files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()] - else: - files = osutil.listdir(path, True) - - tree = {} - for f in files: - if not f.startswith(dir): - continue - - if dir: - f = f[len(dir)+int(not dir.endswith('/')):] - lst = f.split(os.sep) - current = tree - while len(lst) != 1: - current = current.setdefault(lst.pop(0), {}) - current[lst.pop(0)] = None - - return tree - -def zip_directory(directory, b64enc=True, src=True): - """Compress a directory - - @param directory: The directory to compress - @param base64enc: if True the function will encode the zip file with base64 - @param src: Integrate the source files - - @return: a string containing the zip file - """ - - RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I) - - def _zippy(archive, path, src=True): - path = os.path.abspath(path) - base = os.path.basename(path) - for f in osutil.listdir(path, True): - bf = os.path.basename(f) - if not RE_exclude.search(bf) and (src or bf in ('__openerp__.py', '__terp__.py') or not bf.endswith('.py')): - archive.write(os.path.join(path, f), os.path.join(base, f)) - - archname = StringIO() - archive = PyZipFile(archname, "w", ZIP_DEFLATED) - - # for Python 2.5, ZipFile.write() still expects 8-bit strings (2.6 converts to utf-8) - directory = tools.ustr(directory).encode('utf-8') - - archive.writepy(directory) - _zippy(archive, directory, src=src) - archive.close() - archive_data = archname.getvalue() - archname.close() - - if b64enc: - return base64.encodestring(archive_data) - - return archive_data - -def get_module_as_zip(modulename, b64enc=True, src=True): - """Generate a module as zip file with the source or not and can do a base64 encoding - - @param modulename: The module name - @param b64enc: if True the function will encode the zip file with base64 - @param src: Integrate the source files - - @return: a stream to store in a file-like object - """ - - ap = get_module_path(str(modulename)) - if not ap: - raise Exception('Unable to find path for module %s' % modulename) - - ap = ap.encode('utf8') - if os.path.isfile(ap + '.zip'): - val = file(ap + '.zip', 'rb').read() - if b64enc: - val = base64.encodestring(val) - else: - val = zip_directory(ap, b64enc, src) - - return val - - -def get_module_resource(module, *args): - """Return the full path of a resource of the given module. - - @param module: the module - @param args: the resource path components - - @return: absolute path to the resource - """ - a = get_module_path(module) - if not a: return False - resource_path = opj(a, *args) - if zipfile.is_zipfile( a +'.zip') : - zip = zipfile.ZipFile( a + ".zip") - files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()] - resource_path = '/'.join(args) - if resource_path in files: - return opj(a, resource_path) - elif os.path.exists(resource_path): - return resource_path - return False - -def get_modules(): - """Returns the list of module names - """ - def listdir(dir): - def clean(name): - name = os.path.basename(name) - if name[-4:] == '.zip': - name = name[:-4] - return name - - def is_really_module(name): - name = opj(dir, name) - return os.path.isdir(name) or zipfile.is_zipfile(name) - return map(clean, filter(is_really_module, os.listdir(dir))) - - plist = [] - initialize_sys_path() - for ad in ad_paths: - plist.extend(listdir(ad)) - return list(set(plist)) - -def load_information_from_description_file(module): - """ - :param module: The name of the module (sale, purchase, ...) - """ - - for filename in ['__openerp__.py', '__terp__.py']: - description_file = get_module_resource(module, filename) - if description_file : - desc_f = tools.file_open(description_file) - try: - return eval(desc_f.read()) - finally: - desc_f.close() - - #TODO: refactor the logger in this file to follow the logging guidelines - # for 6.0 - logging.getLogger('addons').debug('The module %s does not contain a description file:'\ - '__openerp__.py or __terp__.py (deprecated)', module) - return {} - -def get_modules_with_version(): - modules = get_modules() - res = {} - for module in modules: - try: - info = load_information_from_description_file(module) - res[module] = "%s.%s" % (release.major_version, info['version']) - except Exception, e: - continue - return res - -def create_graph(cr, module_list, force=None): - graph = Graph() - upgrade_graph(graph, cr, module_list, force) - return graph - -def upgrade_graph(graph, cr, module_list, force=None): - if force is None: - force = [] - packages = [] - len_graph = len(graph) - for module in module_list: - mod_path = get_module_path(module) - terp_file = get_module_resource(module, '__openerp__.py') - if not terp_file: - terp_file = get_module_resource(module, '__terp__.py') - if not mod_path or not terp_file: - logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not found, skipped' % (module)) - continue - - if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'): - terp_f = tools.file_open(terp_file) - try: - info = eval(terp_f.read()) - except Exception: - logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: eval file %s' % (module, terp_file)) - raise - finally: - terp_f.close() - if info.get('installable', True): - packages.append((module, info.get('depends', []), info)) - else: - logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not installable, skipped' % (module)) - - dependencies = dict([(p, deps) for p, deps, data in packages]) - current, later = set([p for p, dep, data in packages]), set() - - while packages and current > later: - package, deps, data = packages[0] - - # if all dependencies of 'package' are already in the graph, add 'package' in the graph - if reduce(lambda x, y: x and y in graph, deps, True): - if not package in current: - packages.pop(0) - continue - later.clear() - current.remove(package) - graph.addNode(package, deps) - node = Node(package, graph) - node.data = data - for kind in ('init', 'demo', 'update'): - if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force: - setattr(node, kind, True) - else: - later.add(package) - packages.append((package, deps, data)) - packages.pop(0) - - graph.update_from_db(cr) - - for package in later: - unmet_deps = filter(lambda p: p not in graph, dependencies[package]) - logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: Unmet dependencies: %s' % (package, ', '.join(unmet_deps))) - - result = len(graph) - len_graph - if result != len(module_list): - logger.notifyChannel('init', netsvc.LOG_WARNING, 'Not all modules have loaded.') - return result - - -def init_module_objects(cr, module_name, obj_list): - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: creating or updating database tables' % module_name) - todo = [] - for obj in obj_list: - try: - result = obj._auto_init(cr, {'module': module_name}) - except Exception, e: - raise - if result: - todo += result - if hasattr(obj, 'init'): - obj.init(cr) - cr.commit() - todo.sort() - for t in todo: - t[1](cr, *t[2]) - cr.commit() - - -def register_class(m): - """ - Register module named m, if not already registered - """ - - def log(e): - mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or '' - msg = "Couldn't load %smodule %s" % (mt, m) - logger.notifyChannel('init', netsvc.LOG_CRITICAL, msg) - logger.notifyChannel('init', netsvc.LOG_CRITICAL, e) - - global loaded - if m in loaded: - return - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: registering objects' % m) - mod_path = get_module_path(m) - - initialize_sys_path() - try: - zip_mod_path = mod_path + '.zip' - if not os.path.isfile(zip_mod_path): - fm = imp.find_module(m, ad_paths) - try: - imp.load_module(m, *fm) - finally: - if fm[0]: - fm[0].close() - else: - zimp = zipimport.zipimporter(zip_mod_path) - zimp.load_module(m) - except Exception, e: - log(e) - raise - else: - loaded.append(m) - - -class MigrationManager(object): - """ - This class manage the migration of modules - Migrations files must be python files containing a "migrate(cr, installed_version)" function. - Theses files must respect a directory tree structure: A 'migrations' folder which containt a - folder by version. Version can be 'module' version or 'server.module' version (in this case, - the files will only be processed by this version of the server). Python file names must start - by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation - Example: - - - `-- migrations - |-- 1.0 - | |-- pre-update_table_x.py - | |-- pre-update_table_y.py - | |-- post-clean-data.py - | `-- README.txt # not processed - |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server - | |-- pre-delete_table_z.py - | `-- post-clean-data.py - `-- foo.py # not processed - - This similar structure is generated by the maintenance module with the migrations files get by - the maintenance contract - - """ - def __init__(self, cr, graph): - self.cr = cr - self.graph = graph - self.migrations = {} - self._get_files() - - def _get_files(self): - - """ - import addons.base.maintenance.utils as maintenance_utils - maintenance_utils.update_migrations_files(self.cr) - #""" - - for pkg in self.graph: - self.migrations[pkg.name] = {} - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): - continue - - self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {} - self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {} - - def migrate_module(self, pkg, stage): - assert stage in ('pre', 'post') - stageformat = {'pre': '[>%s]', - 'post': '[%s>]', - } - - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): - return - - def convert_version(version): - if version.startswith(release.major_version) and version != release.major_version: - return version # the version number already containt the server version - return "%s.%s" % (release.major_version, version) - - def _get_migration_versions(pkg): - def __get_dir(tree): - return [d for d in tree if tree[d] is not None] - - versions = list(set( - __get_dir(self.migrations[pkg.name]['module']) + - __get_dir(self.migrations[pkg.name]['maintenance']) - )) - versions.sort(key=lambda k: parse_version(convert_version(k))) - return versions - - def _get_migration_files(pkg, version, stage): - """ return a list of tuple (module, file) - """ - m = self.migrations[pkg.name] - lst = [] - - mapping = {'module': opj(pkg.name, 'migrations'), - 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name), - } - - for x in mapping.keys(): - if version in m[x]: - for f in m[x][version]: - if m[x][version][f] is not None: - continue - if not f.startswith(stage + '-'): - continue - lst.append(opj(mapping[x], version, f)) - lst.sort() - return lst - - def mergedict(a, b): - a = a.copy() - a.update(b) - return a - - from openerp.tools.parse_version import parse_version - - parsed_installed_version = parse_version(pkg.installed_version or '') - current_version = parse_version(convert_version(pkg.data.get('version', '0'))) - - versions = _get_migration_versions(pkg) - - for version in versions: - if parsed_installed_version < parse_version(convert_version(version)) <= current_version: - - strfmt = {'addon': pkg.name, - 'stage': stage, - 'version': stageformat[stage] % version, - } - - for pyfile in _get_migration_files(pkg, version, stage): - name, ext = os.path.splitext(os.path.basename(pyfile)) - if ext.lower() != '.py': - continue - mod = fp = fp2 = None - try: - fp = tools.file_open(pyfile) - - # imp.load_source need a real file object, so we create - # one from the file-like object we get from file_open - fp2 = os.tmpfile() - fp2.write(fp.read()) - fp2.seek(0) - try: - mod = imp.load_source(name, pyfile, fp2) - logger.notifyChannel('migration', netsvc.LOG_INFO, 'module %(addon)s: Running migration %(version)s %(name)s' % mergedict({'name': mod.__name__}, strfmt)) - mod.migrate(self.cr, pkg.installed_version) - except ImportError: - logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % mergedict({'file': pyfile}, strfmt)) - raise - except AttributeError: - logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) - except: - raise - finally: - if fp: - fp.close() - if fp2: - fp2.close() - if mod: - del mod - -log = logging.getLogger('init') - -def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, **kwargs): - """Migrates+Updates or Installs all module nodes from ``graph`` - :param graph: graph of module nodes to load - :param status: status dictionary for keeping track of progress - :param perform_checks: whether module descriptors should be checked for validity (prints warnings - for same cases, and even raise osv_except if certificate is invalid) - :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped - :return: list of modules that were installed or updated - """ - def process_sql_file(cr, fp): - queries = fp.read().split(';') - for query in queries: - new_query = ' '.join(query.split()) - if new_query: - cr.execute(new_query) - - def load_init_update_xml(cr, m, idref, mode, kind): - for filename in package.data.get('%s_xml' % kind, []): - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, filename)) - _, ext = os.path.splitext(filename) - fp = tools.file_open(opj(m, filename)) - try: - if ext == '.csv': - noupdate = (kind == 'init') - tools.convert_csv_import(cr, m, os.path.basename(filename), fp.read(), idref, mode=mode, noupdate=noupdate) - elif ext == '.sql': - process_sql_file(cr, fp) - elif ext == '.yml': - tools.convert_yaml_import(cr, m, fp, idref, mode=mode, **kwargs) - else: - tools.convert_xml_import(cr, m, fp, idref, mode=mode, **kwargs) - finally: - fp.close() - - def load_demo_xml(cr, m, idref, mode): - for xml in package.data.get('demo_xml', []): - name, ext = os.path.splitext(xml) - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, xml)) - fp = tools.file_open(opj(m, xml)) - try: - if ext == '.csv': - tools.convert_csv_import(cr, m, os.path.basename(xml), fp.read(), idref, mode=mode, noupdate=True) - elif ext == '.yml': - tools.convert_yaml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) - else: - tools.convert_xml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) - finally: - fp.close() - - def load_data(cr, module_name, id_map, mode): - _load_data(cr, module_name, id_map, mode, 'data') - - def load_demo(cr, module_name, id_map, mode): - _load_data(cr, module_name, id_map, mode, 'demo') - - def load_test(cr, module_name, id_map, mode): - cr.commit() - if not tools.config.options['test_disable']: - try: - _load_data(cr, module_name, id_map, mode, 'test') - except Exception, e: - logging.getLogger('test').exception('Tests failed to execute in module %s', module_name) - finally: - if tools.config.options['test_commit']: - cr.commit() - else: - cr.rollback() - - def _load_data(cr, module_name, id_map, mode, kind): - for filename in package.data.get(kind, []): - noupdate = (kind == 'demo') - _, ext = os.path.splitext(filename) - log.info("module %s: loading %s", module_name, filename) - pathname = os.path.join(module_name, filename) - file = tools.file_open(pathname) - try: - if ext == '.sql': - process_sql_file(cr, file) - elif ext == '.csv': - noupdate = (kind == 'init') - tools.convert_csv_import(cr, module_name, pathname, file.read(), id_map, mode, noupdate) - elif ext == '.yml': - tools.convert_yaml_import(cr, module_name, file, id_map, mode, noupdate) - else: - tools.convert_xml_import(cr, module_name, file, id_map, mode, noupdate) - finally: - file.close() - - # **kwargs is passed directly to convert_xml_import - if not status: - status = {} - - status = status.copy() - processed_modules = [] - statusi = 0 - pool = pooler.get_pool(cr.dbname) - migrations = MigrationManager(cr, graph) - modobj = None - logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) - - for package in graph: - if skip_modules and package.name in skip_modules: - continue - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name) - migrations.migrate_module(package, 'pre') - register_class(package.name) - modules = pool.instanciate(package.name, cr) - if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): - init_module_objects(cr, package.name, modules) - cr.commit() - - for package in graph: - status['progress'] = (float(statusi)+0.1) / len(graph) - m = package.name - mid = package.id - - if skip_modules and m in skip_modules: - continue - - if modobj is None: - modobj = pool.get('ir.module.module') - - if modobj and perform_checks: - modobj.check(cr, 1, [mid]) - - idref = {} - status['progress'] = (float(statusi)+0.4) / len(graph) - - mode = 'update' - if hasattr(package, 'init') or package.state == 'to install': - mode = 'init' - - if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): - for kind in ('init', 'update'): - if package.state=='to upgrade': - # upgrading the module information - modobj.write(cr, 1, [mid], modobj.get_values_from_terp(package.data)) - load_init_update_xml(cr, m, idref, mode, kind) - load_data(cr, m, idref, mode) - if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): - status['progress'] = (float(statusi)+0.75) / len(graph) - load_demo_xml(cr, m, idref, mode) - load_demo(cr, m, idref, mode) - cr.execute('update ir_module_module set demo=%s where id=%s', (True, mid)) - - # launch tests only in demo mode, as most tests will depend - # on demo data. Other tests can be added into the regular - # 'data' section, but should probably not alter the data, - # as there is no rollback. - load_test(cr, m, idref, mode) - - processed_modules.append(package.name) - - migrations.migrate_module(package, 'post') - - if modobj: - ver = release.major_version + '.' + package.data.get('version', '1.0') - # Set new modules and dependencies - modobj.write(cr, 1, [mid], {'state': 'installed', 'latest_version': ver}) - cr.commit() - # Update translations for all installed languages - modobj.update_translations(cr, 1, [mid], None, {'overwrite': tools.config['overwrite_existing_translations']}) - cr.commit() - - package.state = 'installed' - for kind in ('init', 'demo', 'update'): - if hasattr(package, kind): - delattr(package, kind) - - statusi += 1 - - cr.commit() - - return processed_modules - -def _check_module_names(cr, module_names): - mod_names = set(module_names) - if 'base' in mod_names: - # ignore dummy 'all' module - if 'all' in mod_names: - mod_names.remove('all') - if mod_names: - cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),)) - if cr.dictfetchone()['count'] != len(mod_names): - # find out what module name(s) are incorrect: - cr.execute("SELECT name FROM ir_module_module") - incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) - logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) - -def load_modules(db, force_demo=False, status=None, update_module=False): - - initialize_sys_path() - - # Backward compatibility: addons don't have to import openerp.xxx, they still can import xxx - for k, v in list(sys.modules.items()): - if k.startswith('openerp.') and sys.modules.get(k[8:]) is None: - sys.modules[k[8:]] = v - - if not status: - status = {} - cr = db.cursor() - if cr: - cr.execute("SELECT relname FROM pg_class WHERE relkind='r' AND relname='ir_module_module'") - if len(cr.fetchall())==0: - logger.notifyChannel("init", netsvc.LOG_INFO, "init db") - tools.init_db(cr) - tools.config["init"]["all"] = 1 - tools.config['update']['all'] = 1 - if not tools.config['without_demo']: - tools.config["demo"]['all'] = 1 - force = [] - if force_demo: - force.append('demo') - - # This is a brand new pool, just created in pooler.get_db_and_pool() - pool = pooler.get_pool(cr.dbname) - - try: - processed_modules = [] - report = tools.assertion_report() - # NOTE: Try to also load the modules that have been marked as uninstallable previously... - STATES_TO_LOAD = ['installed', 'to upgrade', 'uninstallable'] - if 'base' in tools.config['update'] or 'all' in tools.config['update']: - cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) - - # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) - graph = create_graph(cr, ['base'], force) - if not graph: - logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)') - raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)')) - processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)) - - if tools.config['load_language']: - for lang in tools.config['load_language'].split(','): - tools.load_language(cr, lang) - - # STEP 2: Mark other modules to be loaded/updated - if update_module: - modobj = pool.get('ir.module.module') - logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list') - if ('base' in tools.config['init']) or ('base' in tools.config['update']): - modobj.update_list(cr, 1) - - _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys())) - - mods = [k for k in tools.config['init'] if tools.config['init'][k]] - if mods: - ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)]) - if ids: - modobj.button_install(cr, 1, ids) - - mods = [k for k in tools.config['update'] if tools.config['update'][k]] - if mods: - ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)]) - if ids: - modobj.button_upgrade(cr, 1, ids) - - cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) - - STATES_TO_LOAD += ['to install'] - - - # STEP 3: Load marked modules (skipping base which was done in STEP 1) - loop_guardrail = 0 - while True: - loop_guardrail += 1 - if loop_guardrail > 100: - raise ValueError('Possible recursive module tree detected, aborting.') - cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(STATES_TO_LOAD),)) - - module_list = [name for (name,) in cr.fetchall() if name not in graph] - if not module_list: - break - - new_modules_in_graph = upgrade_graph(graph, cr, module_list, force) - if new_modules_in_graph == 0: - # nothing to load - break - - logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list))) - processed_modules.extend(load_module_graph(cr, graph, status, report=report, skip_modules=processed_modules)) - - # load custom models - cr.execute('select model from ir_model where state=%s', ('manual',)) - for model in cr.dictfetchall(): - pool.get('ir.model').instanciate(cr, 1, model['model'], {}) - - # STEP 4: Finish and cleanup - if processed_modules: - cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") - for (model, name) in cr.fetchall(): - model_obj = pool.get(model) - if model_obj and not isinstance(model_obj, osv.osv.osv_memory): - logger.notifyChannel('init', netsvc.LOG_WARNING, 'object %s (%s) has no access rules!' % (model, name)) - - # Temporary warning while we remove access rights on osv_memory objects, as they have - # been replaced by owner-only access rights - cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""") - for (model, name) in cr.fetchall(): - model_obj = pool.get(model) - if isinstance(model_obj, osv.osv.osv_memory): - logger.notifyChannel('init', netsvc.LOG_WARNING, 'In-memory object %s (%s) should not have explicit access rules!' % (model, name)) - - cr.execute("SELECT model from ir_model") - for (model,) in cr.fetchall(): - obj = pool.get(model) - if obj: - obj._check_removed_columns(cr, log=True) - else: - logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is referenced but not present in the orm pool!" % model) - - # Cleanup orphan records - pool.get('ir.model.data')._process_end(cr, 1, processed_modules) - - if report.get_report(): - logger.notifyChannel('init', netsvc.LOG_INFO, report) - - for kind in ('init', 'demo', 'update'): - tools.config[kind] = {} - - cr.commit() - if update_module: - cr.execute("select id,name from ir_module_module where state=%s", ('to remove',)) - for mod_id, mod_name in cr.fetchall(): - cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,)) - for rmod, rid in cr.fetchall(): - uid = 1 - rmod_module= pool.get(rmod) - if rmod_module: - rmod_module.unlink(cr, uid, [rid]) - else: - logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid)) - cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,)) - cr.commit() - # - # TODO: remove menu without actions of children - # - while True: - cr.execute('''delete from - ir_ui_menu - where - (id not IN (select parent_id from ir_ui_menu where parent_id is not null)) - and - (id not IN (select res_id from ir_values where model='ir.ui.menu')) - and - (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''') - cr.commit() - if not cr.rowcount: - break - else: - logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,)) - - cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',)) - cr.commit() - finally: - cr.close() +""" +# get_module_path is used only by base_module_quality +from openerp.modules import get_module_resource, get_module_path # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/addons/base/ir/ir_ui_menu.py b/openerp/addons/base/ir/ir_ui_menu.py index 99a13b46cff..d67f0c247d1 100644 --- a/openerp/addons/base/ir/ir_ui_menu.py +++ b/openerp/addons/base/ir/ir_ui_menu.py @@ -24,7 +24,7 @@ import base64 import re import tools -import addons +import openerp.modules from osv import fields, osv from tools.translate import _ @@ -229,7 +229,7 @@ class ir_ui_menu(osv.osv): if not path: return False path_info = path.split(',') - icon_path = addons.get_module_resource(path_info[0],path_info[1]) + icon_path = openerp.modules.get_module_resource(path_info[0],path_info[1]) icon_image = False if icon_path: try: diff --git a/openerp/addons/base/module/module.py b/openerp/addons/base/module/module.py index 5458771b49d..51fdb224918 100644 --- a/openerp/addons/base/module/module.py +++ b/openerp/addons/base/module/module.py @@ -30,7 +30,7 @@ import urllib import zipfile import zipimport -import addons +import openerp.modules as addons import pooler import release import tools @@ -79,8 +79,7 @@ class module(osv.osv): info = {} try: info = addons.load_information_from_description_file(name) - if 'version' in info: - info['version'] = release.major_version + '.' + info['version'] + info['version'] = release.major_version + '.' + info['version'] except Exception: cls.__logger.debug('Error when trying to fetch informations for ' 'module %s', name, exc_info=True) diff --git a/openerp/conf/__init__.py b/openerp/conf/__init__.py new file mode 100644 index 00000000000..7de8070f9e2 --- /dev/null +++ b/openerp/conf/__init__.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Library-wide configuration variables. + +For now, configuration code is in openerp.tools.config. It is in mainly +unprocessed form, e.g. addons_path is a string with commas-separated +paths. The aim is to have code related to configuration (command line +parsing, configuration file loading and saving, ...) in this module +and provide real Python variables, e.g. addons_paths is really a list +of paths. + +""" + +import deprecation + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/conf/deprecation.py b/openerp/conf/deprecation.py new file mode 100644 index 00000000000..53be31b46f9 --- /dev/null +++ b/openerp/conf/deprecation.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Regroup variables for deprecated features. + +To keep the OpenERP server backward compatible with older modules, some +additional code is needed throughout the core library. This module keeps +track of those specific measures by providing variables that can be unset +by the user to check if her code is future proof. + +""" + +# If True, the Python modules inside the openerp namespace are made available +# without the 'openerp.' prefix. E.g. openerp.osv.osv and osv.osv refer to the +# same module. +# Introduced around 2011.02. +open_openerp_namespace = True + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/db/__init__.py b/openerp/db/__init__.py new file mode 100644 index 00000000000..5eb06af1ad1 --- /dev/null +++ b/openerp/db/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Lower-level database access. + +This module provides access to the underlying database without going +through the ORM. The goal is to gather sql_db.py and other various db +code. + +""" + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/modules/__init__.py b/openerp/modules/__init__.py new file mode 100644 index 00000000000..171d8418269 --- /dev/null +++ b/openerp/modules/__init__.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010-2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Modules (also called addons) management. + +""" + +import openerp.modules.db +import openerp.modules.graph +import openerp.modules.loading +import openerp.modules.migration +import openerp.modules.module + +# TODO temporarily expose those things +from openerp.modules.module import \ + get_modules, get_modules_with_version, \ + load_information_from_description_file, \ + get_module_resource, zip_directory, \ + get_module_path, initialize_sys_path, \ + register_module_classes, init_module_models + +from openerp.modules.loading import load_modules + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/modules/db.py b/openerp/modules/db.py new file mode 100644 index 00000000000..25ce7283c97 --- /dev/null +++ b/openerp/modules/db.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +import openerp.modules + +def is_initialized(cr): + """ Check if a database has been initialized for the ORM. + + The database can be initialized with the 'initialize' function below. + + """ + cr.execute("SELECT relname FROM pg_class WHERE relkind='r' AND relname='ir_module_module'") + return len(cr.fetchall()) > 0 + +def initialize(cr): + """ Initialize a database with for the ORM. + + This executes base/base.sql, creates the ir_module_categories (taken + from each module descriptor file), and creates the ir_module_module + and ir_model_data entries. + + """ + f = openerp.modules.get_module_resource('base', 'base.sql') + base_sql_file = openerp.tools.misc.file_open(f) + try: + cr.execute(base_sql_file.read()) + cr.commit() + finally: + base_sql_file.close() + + for i in openerp.modules.get_modules(): + mod_path = openerp.modules.get_module_path(i) + if not mod_path: + continue + + # This will raise an exception if no/unreadable descriptor file. + info = openerp.modules.load_information_from_description_file(i) + + if not info: + continue + categories = info['category'].split('/') + category_id = create_categories(cr, categories) + + if info['installable']: + if info['active']: + state = 'to install' + else: + state = 'uninstalled' + else: + state = 'uninstallable' + + cr.execute('INSERT INTO ir_module_module \ + (author, website, name, shortdesc, description, \ + category_id, state, certificate, web, license) \ + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id', ( + info['author'], + info['website'], i, info['name'], + info['description'], category_id, state, info['certificate'], + info['web'], + info['license'])) + id = cr.fetchone()[0] + cr.execute('INSERT INTO ir_model_data \ + (name,model,module, res_id, noupdate) VALUES (%s,%s,%s,%s,%s)', ( + 'module_meta_information', 'ir.module.module', i, id, True)) + dependencies = info['depends'] + for d in dependencies: + cr.execute('INSERT INTO ir_module_module_dependency \ + (module_id,name) VALUES (%s, %s)', (id, d)) + cr.commit() + +def create_categories(cr, categories): + """ Create the ir_module_category entries for some categories. + + categories is a list of strings forming a single category with its + parent categories, like ['Grand Parent', 'Parent', 'Child']. + + Return the database id of the (last) category. + + """ + p_id = None + while categories: + if p_id is not None: + cr.execute('SELECT id \ + FROM ir_module_category \ + WHERE name=%s AND parent_id=%s', (categories[0], p_id)) + else: + cr.execute('SELECT id \ + FROM ir_module_category \ + WHERE name=%s AND parent_id IS NULL', (categories[0],)) + c_id = cr.fetchone() + if not c_id: + cr.execute('INSERT INTO ir_module_category \ + (name, parent_id) \ + VALUES (%s, %s) RETURNING id', (categories[0], p_id)) + c_id = cr.fetchone()[0] + else: + c_id = c_id[0] + p_id = c_id + categories = categories[1:] + return p_id + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/modules/graph.py b/openerp/modules/graph.py new file mode 100644 index 00000000000..078d23c235b --- /dev/null +++ b/openerp/modules/graph.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010-2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Modules dependency graph. """ + +import os, sys, imp +from os.path import join as opj +import itertools +import zipimport + +import openerp + +import openerp.osv as osv +import openerp.tools as tools +import openerp.tools.osutil as osutil +from openerp.tools.safe_eval import safe_eval as eval +import openerp.pooler as pooler +from openerp.tools.translate import _ + +import openerp.netsvc as netsvc + +import zipfile +import openerp.release as release + +import re +import base64 +from zipfile import PyZipFile, ZIP_DEFLATED +from cStringIO import StringIO + +import logging + +logger = netsvc.Logger() + + +class Graph(dict): + """ Modules dependency graph. + + The graph is a mapping from module name to Nodes. + + """ + + def add_node(self, name, deps): + max_depth, father = 0, None + for n in [Node(x, self) for x in deps]: + if n.depth >= max_depth: + father = n + max_depth = n.depth + if father: + return father.add_child(name) + else: + return Node(name, self) + + def update_from_db(self, cr): + if not len(self): + return + # update the graph with values from the database (if exist) + ## First, we set the default values for each package in graph + additional_data = dict.fromkeys(self.keys(), {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None}) + ## Then we get the values from the database + cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version' + ' FROM ir_module_module' + ' WHERE name IN %s',(tuple(additional_data),) + ) + + ## and we update the default values with values from the database + additional_data.update(dict([(x.pop('name'), x) for x in cr.dictfetchall()])) + + for package in self.values(): + for k, v in additional_data[package.name].items(): + setattr(package, k, v) + + def add_module(self, cr, module, force=None): + self.add_modules(cr, [module], force) + + def add_modules(self, cr, module_list, force=None): + if force is None: + force = [] + packages = [] + len_graph = len(self) + for module in module_list: + # This will raise an exception if no/unreadable descriptor file. + # NOTE The call to load_information_from_description_file is already + # done by db.initialize, so it is possible to not do it again here. + info = openerp.modules.module.load_information_from_description_file(module) + if info['installable']: + packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version + else: + logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not installable, skipped' % (module)) + + dependencies = dict([(p, info['depends']) for p, info in packages]) + current, later = set([p for p, info in packages]), set() + + while packages and current > later: + package, info = packages[0] + deps = info['depends'] + + # if all dependencies of 'package' are already in the graph, add 'package' in the graph + if reduce(lambda x, y: x and y in self, deps, True): + if not package in current: + packages.pop(0) + continue + later.clear() + current.remove(package) + node = self.add_node(package, deps) + node.data = info + for kind in ('init', 'demo', 'update'): + if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force: + setattr(node, kind, True) + else: + later.add(package) + packages.append((package, info)) + packages.pop(0) + + self.update_from_db(cr) + + for package in later: + unmet_deps = filter(lambda p: p not in self, dependencies[package]) + logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: Unmet dependencies: %s' % (package, ', '.join(unmet_deps))) + + result = len(self) - len_graph + if result != len(module_list): + logger.notifyChannel('init', netsvc.LOG_WARNING, 'Not all modules have loaded.') + return result + + + def __iter__(self): + level = 0 + done = set(self.keys()) + while done: + level_modules = [(name, module) for name, module in self.items() if module.depth==level] + for name, module in level_modules: + done.remove(name) + yield module + level += 1 + + +class Singleton(object): + def __new__(cls, name, graph): + if name in graph: + inst = graph[name] + else: + inst = object.__new__(cls) + inst.name = name + graph[name] = inst + return inst + + +class Node(Singleton): + """ One module in the modules dependency graph. + + Node acts as a per-module singleton. + + """ + + def __init__(self, name, graph): + self.graph = graph + if not hasattr(self, 'children'): + self.children = [] + if not hasattr(self, 'depth'): + self.depth = 0 + + def add_child(self, name): + node = Node(name, self.graph) + node.depth = self.depth + 1 + if node not in self.children: + self.children.append(node) + for attr in ('init', 'update', 'demo'): + if hasattr(self, attr): + setattr(node, attr, True) + self.children.sort(lambda x, y: cmp(x.name, y.name)) + return node + + def __setattr__(self, name, value): + super(Singleton, self).__setattr__(name, value) + if name in ('init', 'update', 'demo'): + tools.config[name][self.name] = 1 + for child in self.children: + setattr(child, name, value) + if name == 'depth': + for child in self.children: + setattr(child, name, value + 1) + + def __iter__(self): + return itertools.chain(iter(self.children), *map(iter, self.children)) + + def __str__(self): + return self._pprint() + + def _pprint(self, depth=0): + s = '%s\n' % self.name + for c in self.children: + s += '%s`-> %s' % (' ' * depth, c._pprint(depth+1)) + return s + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py new file mode 100644 index 00000000000..17ae2dce3ef --- /dev/null +++ b/openerp/modules/loading.py @@ -0,0 +1,421 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010-2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Modules (also called addons) management. + +""" + +import os, sys, imp +from os.path import join as opj +import itertools +import zipimport + +import openerp + +import openerp.osv as osv +import openerp.tools as tools +import openerp.tools.osutil as osutil +from openerp.tools.safe_eval import safe_eval as eval +import openerp.pooler as pooler +from openerp.tools.translate import _ + +import openerp.netsvc as netsvc + +import zipfile +import openerp.release as release + +import re +import base64 +from zipfile import PyZipFile, ZIP_DEFLATED +from cStringIO import StringIO + +import logging + +import openerp.modules.db +import openerp.modules.graph +import openerp.modules.migration + +from openerp.modules.module import \ + get_modules, get_modules_with_version, \ + load_information_from_description_file, \ + get_module_resource, zip_directory, \ + get_module_path, initialize_sys_path, \ + register_module_classes, init_module_models + +logger = netsvc.Logger() + + +def open_openerp_namespace(): + # See comment for open_openerp_namespace. + if openerp.conf.deprecation.open_openerp_namespace: + for k, v in list(sys.modules.items()): + if k.startswith('openerp.') and sys.modules.get(k[8:]) is None: + sys.modules[k[8:]] = v + + +def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): + """Migrates+Updates or Installs all module nodes from ``graph`` + :param graph: graph of module nodes to load + :param status: status dictionary for keeping track of progress + :param perform_checks: whether module descriptors should be checked for validity (prints warnings + for same cases, and even raise osv_except if certificate is invalid) + :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped + :return: list of modules that were installed or updated + """ + def process_sql_file(cr, fp): + queries = fp.read().split(';') + for query in queries: + new_query = ' '.join(query.split()) + if new_query: + cr.execute(new_query) + + def load_init_xml(cr, m, idref, mode): + _load_data(cr, m, idref, mode, 'init_xml') + + def load_update_xml(cr, m, idref, mode): + _load_data(cr, m, idref, mode, 'update_xml') + + def load_demo_xml(cr, m, idref, mode): + _load_data(cr, m, idref, mode, 'demo_xml') + + def load_data(cr, module_name, idref, mode): + _load_data(cr, module_name, idref, mode, 'data') + + def load_demo(cr, module_name, idref, mode): + _load_data(cr, module_name, idref, mode, 'demo') + + def load_test(cr, module_name, idref, mode): + cr.commit() + if not tools.config.options['test_disable']: + try: + _load_data(cr, module_name, idref, mode, 'test') + except Exception, e: + logging.getLogger('test').exception('Tests failed to execute in module %s', module_name) + finally: + if tools.config.options['test_commit']: + cr.commit() + else: + cr.rollback() + + def _load_data(cr, module_name, idref, mode, kind): + """ + + kind: data, demo, test, init_xml, update_xml, demo_xml. + + noupdate is False, unless it is demo data or it is csv data in + init mode. + + """ + for filename in package.data[kind]: + log = logging.getLogger('init') + log.info("module %s: loading %s", module_name, filename) + _, ext = os.path.splitext(filename) + pathname = os.path.join(module_name, filename) + fp = tools.file_open(pathname) + noupdate = False + if kind in ('demo', 'demo_xml'): + noupdate = True + try: + if ext == '.csv': + if kind in ('init', 'init_xml'): + noupdate = True + tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate) + elif ext == '.sql': + process_sql_file(cr, fp) + elif ext == '.yml': + tools.convert_yaml_import(cr, module_name, fp, idref, mode, noupdate) + else: + tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report) + finally: + fp.close() + + if status is None: + status = {} + + processed_modules = [] + statusi = 0 + pool = pooler.get_pool(cr.dbname) + migrations = openerp.modules.migration.MigrationManager(cr, graph) + logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) + + # register, instanciate and initialize models for each modules + for package in graph: + if skip_modules and package.name in skip_modules: + continue + logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name) + migrations.migrate_module(package, 'pre') + register_module_classes(package.name) + models = pool.instanciate(package.name, cr) + if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): + init_module_models(cr, package.name, models) + cr.commit() + + # load data for each modules + modobj = pool.get('ir.module.module') + for package in graph: + status['progress'] = (float(statusi)+0.1) / len(graph) + m = package.name + mid = package.id + + if skip_modules and m in skip_modules: + continue + + if perform_checks: + modobj.check(cr, 1, [mid]) + + idref = {} + status['progress'] = (float(statusi)+0.4) / len(graph) + + mode = 'update' + if hasattr(package, 'init') or package.state == 'to install': + mode = 'init' + + if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): + if package.state=='to upgrade': + # upgrading the module information + modobj.write(cr, 1, [mid], modobj.get_values_from_terp(package.data)) + load_init_xml(cr, m, idref, mode) + load_update_xml(cr, m, idref, mode) + load_data(cr, m, idref, mode) + if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): + status['progress'] = (float(statusi)+0.75) / len(graph) + load_demo_xml(cr, m, idref, mode) + load_demo(cr, m, idref, mode) + cr.execute('update ir_module_module set demo=%s where id=%s', (True, mid)) + + # launch tests only in demo mode, as most tests will depend + # on demo data. Other tests can be added into the regular + # 'data' section, but should probably not alter the data, + # as there is no rollback. + load_test(cr, m, idref, mode) + + processed_modules.append(package.name) + + migrations.migrate_module(package, 'post') + + ver = release.major_version + '.' + package.data['version'] + # Set new modules and dependencies + modobj.write(cr, 1, [mid], {'state': 'installed', 'latest_version': ver}) + cr.commit() + # Update translations for all installed languages + modobj.update_translations(cr, 1, [mid], None) + cr.commit() + + package.state = 'installed' + for kind in ('init', 'demo', 'update'): + if hasattr(package, kind): + delattr(package, kind) + + statusi += 1 + + cr.commit() + + return processed_modules + +def _check_module_names(cr, module_names): + mod_names = set(module_names) + if 'base' in mod_names: + # ignore dummy 'all' module + if 'all' in mod_names: + mod_names.remove('all') + if mod_names: + cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),)) + if cr.dictfetchone()['count'] != len(mod_names): + # find out what module name(s) are incorrect: + cr.execute("SELECT name FROM ir_module_module") + incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) + logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) + +def load_modules(db, force_demo=False, status=None, update_module=False): + # TODO status['progress'] reporting is broken: used twice (and reset each + # time to zero) in load_module_graph, not fine-grained enough. + # It should be a method exposed by the pool. + + initialize_sys_path() + + open_openerp_namespace() + + force = [] + if force_demo: + force.append('demo') + + cr = db.cursor() + try: + if not openerp.modules.db.is_initialized(cr): + logger.notifyChannel("init", netsvc.LOG_INFO, "init db") + openerp.modules.db.initialize(cr) + tools.config["init"]["all"] = 1 + tools.config['update']['all'] = 1 + if not tools.config['without_demo']: + tools.config["demo"]['all'] = 1 + + # This is a brand new pool, just created in pooler.get_db_and_pool() + pool = pooler.get_pool(cr.dbname) + + processed_modules = [] + report = tools.assertion_report() + # NOTE: Try to also load the modules that have been marked as uninstallable previously... + STATES_TO_LOAD = ['installed', 'to upgrade', 'uninstallable'] + if 'base' in tools.config['update'] or 'all' in tools.config['update']: + cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) + + # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) + graph = openerp.modules.graph.Graph() + graph.add_module(cr, 'base', force) + if not graph: + logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)') + raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)')) + processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)) + + if tools.config['load_language']: + for lang in tools.config['load_language'].split(','): + tools.load_language(cr, lang) + + # STEP 2: Mark other modules to be loaded/updated + if update_module: + modobj = pool.get('ir.module.module') + if ('base' in tools.config['init']) or ('base' in tools.config['update']): + logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list') + modobj.update_list(cr, 1) + + _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys())) + + mods = [k for k in tools.config['init'] if tools.config['init'][k]] + if mods: + ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)]) + if ids: + modobj.button_install(cr, 1, ids) + + mods = [k for k in tools.config['update'] if tools.config['update'][k]] + if mods: + ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)]) + if ids: + modobj.button_upgrade(cr, 1, ids) + + cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) + + STATES_TO_LOAD += ['to install'] + + + # STEP 3: Load marked modules (skipping base which was done in STEP 1) + loop_guardrail = 0 + while True: + loop_guardrail += 1 + if loop_guardrail > 100: + raise ValueError('Possible recursive module tree detected, aborting.') + cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(STATES_TO_LOAD),)) + + module_list = [name for (name,) in cr.fetchall() if name not in graph] + if not module_list: + break + + new_modules_in_graph = graph.add_modules(cr, module_list, force) + if new_modules_in_graph == 0: + # nothing to load + break + + logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list))) + processed_modules.extend(load_module_graph(cr, graph, status, report=report, skip_modules=processed_modules)) + + # load custom models + cr.execute('select model from ir_model where state=%s', ('manual',)) + for model in cr.dictfetchall(): + pool.get('ir.model').instanciate(cr, 1, model['model'], {}) + + # STEP 4: Finish and cleanup + if processed_modules: + cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") + for (model, name) in cr.fetchall(): + model_obj = pool.get(model) + if model_obj and not isinstance(model_obj, osv.osv.osv_memory): + logger.notifyChannel('init', netsvc.LOG_WARNING, 'object %s (%s) has no access rules!' % (model, name)) + + # Temporary warning while we remove access rights on osv_memory objects, as they have + # been replaced by owner-only access rights + cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""") + for (model, name) in cr.fetchall(): + model_obj = pool.get(model) + if isinstance(model_obj, osv.osv.osv_memory): + logger.notifyChannel('init', netsvc.LOG_WARNING, 'In-memory object %s (%s) should not have explicit access rules!' % (model, name)) + + cr.execute("SELECT model from ir_model") + for (model,) in cr.fetchall(): + obj = pool.get(model) + if obj: + obj._check_removed_columns(cr, log=True) + else: + logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is referenced but not present in the orm pool!" % model) + + # Cleanup orphan records + pool.get('ir.model.data')._process_end(cr, 1, processed_modules) + + if report.get_report(): + logger.notifyChannel('init', netsvc.LOG_INFO, report) + + for kind in ('init', 'demo', 'update'): + tools.config[kind] = {} + + cr.commit() + if update_module: + # Remove records referenced from ir_model_data for modules to be + # removed (and removed the references from ir_model_data). + cr.execute("select id,name from ir_module_module where state=%s", ('to remove',)) + for mod_id, mod_name in cr.fetchall(): + cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,)) + for rmod, rid in cr.fetchall(): + uid = 1 + rmod_module= pool.get(rmod) + if rmod_module: + # TODO group by module so that we can delete multiple ids in a call + rmod_module.unlink(cr, uid, [rid]) + else: + logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid)) + cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,)) + cr.commit() + + # Remove menu items that are not referenced by any of other + # (child) menu item, ir_values, or ir_model_data. + # This code could be a method of ir_ui_menu. + # TODO: remove menu without actions of children + while True: + cr.execute('''delete from + ir_ui_menu + where + (id not IN (select parent_id from ir_ui_menu where parent_id is not null)) + and + (id not IN (select res_id from ir_values where model='ir.ui.menu')) + and + (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''') + cr.commit() + if not cr.rowcount: + break + else: + logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,)) + + # Pretend that modules to be removed are actually uninstalled. + cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',)) + cr.commit() + finally: + cr.close() + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/modules/migration.py b/openerp/modules/migration.py new file mode 100644 index 00000000000..d26f5927141 --- /dev/null +++ b/openerp/modules/migration.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010-2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Modules migration handling. """ + +import os, sys, imp +from os.path import join as opj +import itertools +import zipimport + +import openerp + +import openerp.osv as osv +import openerp.tools as tools +import openerp.tools.osutil as osutil +from openerp.tools.safe_eval import safe_eval as eval +import openerp.pooler as pooler +from openerp.tools.translate import _ + +import openerp.netsvc as netsvc + +import zipfile +import openerp.release as release + +import re +import base64 +from zipfile import PyZipFile, ZIP_DEFLATED +from cStringIO import StringIO + +import logging + +import openerp.modules.db +import openerp.modules.graph + +logger = netsvc.Logger() + + +class MigrationManager(object): + """ + This class manage the migration of modules + Migrations files must be python files containing a "migrate(cr, installed_version)" function. + Theses files must respect a directory tree structure: A 'migrations' folder which containt a + folder by version. Version can be 'module' version or 'server.module' version (in this case, + the files will only be processed by this version of the server). Python file names must start + by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation + Example: + + + `-- migrations + |-- 1.0 + | |-- pre-update_table_x.py + | |-- pre-update_table_y.py + | |-- post-clean-data.py + | `-- README.txt # not processed + |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server + | |-- pre-delete_table_z.py + | `-- post-clean-data.py + `-- foo.py # not processed + + This similar structure is generated by the maintenance module with the migrations files get by + the maintenance contract + + """ + def __init__(self, cr, graph): + self.cr = cr + self.graph = graph + self.migrations = {} + self._get_files() + + def _get_files(self): + + """ + import addons.base.maintenance.utils as maintenance_utils + maintenance_utils.update_migrations_files(self.cr) + #""" + + for pkg in self.graph: + self.migrations[pkg.name] = {} + if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): + continue + + get_module_filetree = openerp.modules.module.get_module_filetree + self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {} + self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {} + + def migrate_module(self, pkg, stage): + assert stage in ('pre', 'post') + stageformat = {'pre': '[>%s]', + 'post': '[%s>]', + } + + if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): + return + + def convert_version(version): + if version.startswith(release.major_version) and version != release.major_version: + return version # the version number already containt the server version + return "%s.%s" % (release.major_version, version) + + def _get_migration_versions(pkg): + def __get_dir(tree): + return [d for d in tree if tree[d] is not None] + + versions = list(set( + __get_dir(self.migrations[pkg.name]['module']) + + __get_dir(self.migrations[pkg.name]['maintenance']) + )) + versions.sort(key=lambda k: parse_version(convert_version(k))) + return versions + + def _get_migration_files(pkg, version, stage): + """ return a list of tuple (module, file) + """ + m = self.migrations[pkg.name] + lst = [] + + mapping = {'module': opj(pkg.name, 'migrations'), + 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name), + } + + for x in mapping.keys(): + if version in m[x]: + for f in m[x][version]: + if m[x][version][f] is not None: + continue + if not f.startswith(stage + '-'): + continue + lst.append(opj(mapping[x], version, f)) + lst.sort() + return lst + + def mergedict(a, b): + a = a.copy() + a.update(b) + return a + + from openerp.tools.parse_version import parse_version + + parsed_installed_version = parse_version(pkg.installed_version or '') + current_version = parse_version(convert_version(pkg.data['version'])) + + versions = _get_migration_versions(pkg) + + for version in versions: + if parsed_installed_version < parse_version(convert_version(version)) <= current_version: + + strfmt = {'addon': pkg.name, + 'stage': stage, + 'version': stageformat[stage] % version, + } + + for pyfile in _get_migration_files(pkg, version, stage): + name, ext = os.path.splitext(os.path.basename(pyfile)) + if ext.lower() != '.py': + continue + mod = fp = fp2 = None + try: + fp = tools.file_open(pyfile) + + # imp.load_source need a real file object, so we create + # one from the file-like object we get from file_open + fp2 = os.tmpfile() + fp2.write(fp.read()) + fp2.seek(0) + try: + mod = imp.load_source(name, pyfile, fp2) + logger.notifyChannel('migration', netsvc.LOG_INFO, 'module %(addon)s: Running migration %(version)s %(name)s' % mergedict({'name': mod.__name__}, strfmt)) + mod.migrate(self.cr, pkg.installed_version) + except ImportError: + logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % mergedict({'file': pyfile}, strfmt)) + raise + except AttributeError: + logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) + except: + raise + finally: + if fp: + fp.close() + if fp2: + fp2.close() + if mod: + del mod + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/modules/module.py b/openerp/modules/module.py new file mode 100644 index 00000000000..bf53ca6d87c --- /dev/null +++ b/openerp/modules/module.py @@ -0,0 +1,379 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010-2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +import os, sys, imp +from os.path import join as opj +import itertools +import zipimport + +import openerp + +import openerp.osv as osv +import openerp.tools as tools +import openerp.tools.osutil as osutil +from openerp.tools.safe_eval import safe_eval as eval +import openerp.pooler as pooler +from openerp.tools.translate import _ + +import openerp.netsvc as netsvc + +import zipfile +import openerp.release as release + +import re +import base64 +from zipfile import PyZipFile, ZIP_DEFLATED +from cStringIO import StringIO + +import logging + +import openerp.modules.db +import openerp.modules.graph + +_ad = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'addons') # default addons path (base) +ad_paths = [] + +# Modules already loaded +loaded = [] + +logger = netsvc.Logger() + +def initialize_sys_path(): + global ad_paths + + if ad_paths: + return + + ad_paths = map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(',')) + + sys.path.insert(1, _ad) + + ad_cnt=1 + for adp in ad_paths: + if adp != _ad: + sys.path.insert(ad_cnt, adp) + ad_cnt+=1 + + ad_paths.append(_ad) # for get_module_path + + +def get_module_path(module, downloaded=False): + """Return the path of the given module. + + Search the addons paths and return the first path where the given + module is found. If downloaded is True, return the default addons + path if nothing else is found. + + """ + initialize_sys_path() + for adp in ad_paths: + if os.path.exists(opj(adp, module)) or os.path.exists(opj(adp, '%s.zip' % module)): + return opj(adp, module) + + if downloaded: + return opj(_ad, module) + logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: module not found' % (module,)) + return False + + +def get_module_filetree(module, dir='.'): + path = get_module_path(module) + if not path: + return False + + dir = os.path.normpath(dir) + if dir == '.': + dir = '' + if dir.startswith('..') or (dir and dir[0] == '/'): + raise Exception('Cannot access file outside the module') + + if not os.path.isdir(path): + # zipmodule + zip = zipfile.ZipFile(path + ".zip") + files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()] + else: + files = osutil.listdir(path, True) + + tree = {} + for f in files: + if not f.startswith(dir): + continue + + if dir: + f = f[len(dir)+int(not dir.endswith('/')):] + lst = f.split(os.sep) + current = tree + while len(lst) != 1: + current = current.setdefault(lst.pop(0), {}) + current[lst.pop(0)] = None + + return tree + +def zip_directory(directory, b64enc=True, src=True): + """Compress a directory + + @param directory: The directory to compress + @param base64enc: if True the function will encode the zip file with base64 + @param src: Integrate the source files + + @return: a string containing the zip file + """ + + RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I) + + def _zippy(archive, path, src=True): + path = os.path.abspath(path) + base = os.path.basename(path) + for f in osutil.listdir(path, True): + bf = os.path.basename(f) + if not RE_exclude.search(bf) and (src or bf in ('__openerp__.py', '__terp__.py') or not bf.endswith('.py')): + archive.write(os.path.join(path, f), os.path.join(base, f)) + + archname = StringIO() + archive = PyZipFile(archname, "w", ZIP_DEFLATED) + + # for Python 2.5, ZipFile.write() still expects 8-bit strings (2.6 converts to utf-8) + directory = tools.ustr(directory).encode('utf-8') + + archive.writepy(directory) + _zippy(archive, directory, src=src) + archive.close() + archive_data = archname.getvalue() + archname.close() + + if b64enc: + return base64.encodestring(archive_data) + + return archive_data + +def get_module_as_zip(modulename, b64enc=True, src=True): + """Generate a module as zip file with the source or not and can do a base64 encoding + + @param modulename: The module name + @param b64enc: if True the function will encode the zip file with base64 + @param src: Integrate the source files + + @return: a stream to store in a file-like object + """ + + ap = get_module_path(str(modulename)) + if not ap: + raise Exception('Unable to find path for module %s' % modulename) + + ap = ap.encode('utf8') + if os.path.isfile(ap + '.zip'): + val = file(ap + '.zip', 'rb').read() + if b64enc: + val = base64.encodestring(val) + else: + val = zip_directory(ap, b64enc, src) + + return val + + +def get_module_resource(module, *args): + """Return the full path of a resource of the given module. + + @param module: the module + @param args: the resource path components + + @return: absolute path to the resource + + TODO name it get_resource_path + TODO make it available inside on osv object (self.get_resource_path) + """ + a = get_module_path(module) + if not a: return False + resource_path = opj(a, *args) + if zipfile.is_zipfile( a +'.zip') : + zip = zipfile.ZipFile( a + ".zip") + files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()] + resource_path = '/'.join(args) + if resource_path in files: + return opj(a, resource_path) + elif os.path.exists(resource_path): + return resource_path + return False + + +def load_information_from_description_file(module): + """ + :param module: The name of the module (sale, purchase, ...) + """ + + terp_file = get_module_resource(module, '__openerp__.py') + if not terp_file: + terp_file = get_module_resource(module, '__terp__.py') + mod_path = get_module_path(module) + if terp_file: + info = {} + if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'): + terp_f = tools.file_open(terp_file) + try: + info = eval(terp_f.read()) + except Exception: + logger.notifyChannel('modules', netsvc.LOG_ERROR, + 'module %s: exception while evaluating file %s' % + (module, terp_file)) + raise + finally: + terp_f.close() + # TODO the version should probably be mandatory + info.setdefault('version', '0') + info.setdefault('category', 'Uncategorized') + info.setdefault('depends', []) + info.setdefault('author', '') + info.setdefault('website', '') + info.setdefault('name', False) + info.setdefault('description', '') + info['certificate'] = info.get('certificate') or None + info['web'] = info.get('web') or False + info['license'] = info.get('license') or 'AGPL-3' + info.setdefault('installable', True) + info.setdefault('active', False) + for kind in ['data', 'demo', 'test', + 'init_xml', 'update_xml', 'demo_xml']: + info.setdefault(kind, []) + return info + + #TODO: refactor the logger in this file to follow the logging guidelines + # for 6.0 + logging.getLogger('modules').debug('module %s: no descriptor file' + ' found: __openerp__.py or __terp__.py (deprecated)', module) + return {} + + +def init_module_models(cr, module_name, obj_list): + """ Initialize a list of models. + + Call _auto_init and init on each model to create or update the + database tables supporting the models. + + TODO better explanation of _auto_init and init. + + """ + + logger.notifyChannel('init', netsvc.LOG_INFO, + 'module %s: creating or updating database tables' % module_name) + # TODO _auto_init doesn't seem to return anything + # so this todo list would be useless. + todo = [] + for obj in obj_list: + try: + # TODO the module in the context doesn't seem usefull: + # it is available (at least) in the class' _module attribute. + # (So module_name would be useless too.) + result = obj._auto_init(cr, {'module': module_name}) + except Exception, e: + raise + if result: + todo += result + if hasattr(obj, 'init'): + obj.init(cr) + cr.commit() + todo.sort() + for t in todo: + t[1](cr, *t[2]) + cr.commit() + + +def load_module(module_name): + """ Load a Python module found on the addons paths.""" + fm = imp.find_module(module_name, ad_paths) + try: + imp.load_module(module_name, *fm) + finally: + if fm[0]: + fm[0].close() + + +def register_module_classes(m): + """ Register module named m, if not already registered. + + This will load the module and register all of its models. (Actually, the + explicit constructor call of each of the models inside the module will + register them.) + + """ + + def log(e): + mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or '' + msg = "Couldn't load %smodule %s" % (mt, m) + logger.notifyChannel('init', netsvc.LOG_CRITICAL, msg) + logger.notifyChannel('init', netsvc.LOG_CRITICAL, e) + + global loaded + if m in loaded: + return + logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: registering objects' % m) + mod_path = get_module_path(m) + + initialize_sys_path() + try: + zip_mod_path = mod_path + '.zip' + if not os.path.isfile(zip_mod_path): + load_module(m) + else: + zimp = zipimport.zipimporter(zip_mod_path) + zimp.load_module(m) + except Exception, e: + log(e) + raise + else: + loaded.append(m) + + +def get_modules(): + """Returns the list of module names + """ + def listdir(dir): + def clean(name): + name = os.path.basename(name) + if name[-4:] == '.zip': + name = name[:-4] + return name + + def is_really_module(name): + name = opj(dir, name) + return os.path.isdir(name) or zipfile.is_zipfile(name) + return map(clean, filter(is_really_module, os.listdir(dir))) + + plist = [] + initialize_sys_path() + for ad in ad_paths: + plist.extend(listdir(ad)) + return list(set(plist)) + + +def get_modules_with_version(): + modules = get_modules() + res = {} + for module in modules: + try: + info = load_information_from_description_file(module) + res[module] = "%s.%s" % (release.major_version, info['version']) + except Exception, e: + continue + return res + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py new file mode 100644 index 00000000000..3b11219d5a5 --- /dev/null +++ b/openerp/modules/registry.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Models registries. + +""" + +import openerp.sql_db + + +class BoundRegistry(object): + """ Model registry/database connection pair.""" + def __init__(self, db, registry): + self.db = db + self.registry = registry + + +class RegistryManager(object): + """ Model registries manager. + + The manager is responsible for creation and deletion of bound model + registries (essentially database connection/model registry pairs). + + """ + + # TODO maybe should receive the addons paths + def __init__(self): + # Mapping between db name and bound model registry. + # Accessed through the methods below. + self.bound_registries = {} + + + def get(self, db_name, force_demo=False, status=None, update_module=False, + pooljobs=True): + """ Return a bound registry for a given database name.""" + + if db_name in self.bound_registries: + bound_registry = self.bound_registries[db_name] + else: + bound_registry = self.new(db_name, force_demo, status, + update_module, pooljobs) + return bound_registry + + + def new(self, db_name, force_demo=False, status=None, + update_module=False, pooljobs=True): + """ Create and return a new bound registry for a given database name. + + The (possibly) previous bound registry for that database name is + discarded. + + """ + + import openerp.modules + import openerp.osv.osv as osv_osv + db = openerp.sql_db.db_connect(db_name) + pool = osv_osv.osv_pool() + + # Initializing a registry will call general code which will in turn + # call registries.get (this object) to obtain the registry being + # initialized. Make it available in the bound_registries dictionary + # then remove it if an exception is raised. + self.delete(db_name) + bound_registry = BoundRegistry(db, pool) + self.bound_registries[db_name] = bound_registry + try: + # This should be a method on BoundRegistry + openerp.modules.load_modules(db, force_demo, status, update_module) + except Exception: + del self.bound_registries[db_name] + raise + + cr = db.cursor() + try: + pool.init_set(cr, False) + pool.get('ir.actions.report.xml').register_all(cr) + cr.commit() + finally: + cr.close() + + if pooljobs: + pool.get('ir.cron').restart(db.dbname) + + return bound_registry + + + def delete(self, db_name): + if db_name in self.bound_registries: + del self.bound_registries[db_name] + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/netsvc.py b/openerp/netsvc.py index 9d3e338a9e1..9ce7f4a35d2 100644 --- a/openerp/netsvc.py +++ b/openerp/netsvc.py @@ -213,6 +213,19 @@ def init_logger(): logger.addHandler(handler) logger.setLevel(int(tools.config['log_level'] or '0')) +# A alternative logging scheme for automated runs of the +# server intended to test it. +def init_alternative_logger(): + class H(logging.Handler): + def emit(self, record): + if record.levelno > 20: + print record.levelno, record.pathname, record.msg + handler = H() + logger = logging.getLogger() + logger.handlers = [] + logger.addHandler(handler) + logger.setLevel(logging.ERROR) + class Agent(object): """Singleton that keeps track of cancellable tasks to run at a given timestamp. @@ -277,12 +290,13 @@ class Agent(object): time.sleep(1) time.sleep(60) -agent_runner = threading.Thread(target=Agent.runner, name="netsvc.Agent.runner") -# the agent runner is a typical daemon thread, that will never quit and must be -# terminated when the main process exits - with no consequence (the processing -# threads it spawns are not marked daemon) -agent_runner.setDaemon(True) -agent_runner.start() +def start_agent(): + agent_runner = threading.Thread(target=Agent.runner, name="netsvc.Agent.runner") + # the agent runner is a typical daemon thread, that will never quit and must be + # terminated when the main process exits - with no consequence (the processing + # threads it spawns are not marked daemon) + agent_runner.setDaemon(True) + agent_runner.start() import traceback diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index ca3c6e27dbd..2482b5e44b5 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -63,6 +63,36 @@ from openerp.tools.safe_eval import safe_eval as eval from openerp.tools import SKIPPED_ELEMENT_TYPES regex_order = re.compile('^(([a-z0-9_]+|"[a-z0-9_]+")( *desc| *asc)?( *, *|))+$', re.I) +regex_object_name = re.compile(r'^[a-z0-9_.]+$') + +def check_object_name(name): + """ Check if the given name is a valid openerp object name. + + The _name attribute in osv and osv_memory object is subject to + some restrictions. This function returns True or False whether + the given name is allowed or not. + + TODO: this is an approximation. The goal in this approximation + is to disallow uppercase characters (in some places, we quote + table/column names and in other not, which leads to this kind + of errors: + + psycopg2.ProgrammingError: relation "xxx" does not exist). + + The same restriction should apply to both osv and osv_memory + objects for consistency. + + """ + if regex_object_name.match(name) is None: + return False + return True + +def raise_on_invalid_object_name(name): + if not check_object_name(name): + msg = "The _name attribute %s is not valid." % name + logger = netsvc.Logger() + logger.notifyChannel('orm', netsvc.LOG_ERROR, msg) + raise except_orm('ValueError', msg) POSTGRES_CONFDELTYPES = { 'RESTRICT': 'r', @@ -509,6 +539,7 @@ class orm_template(object): cr.commit() def _auto_init(self, cr, context=None): + raise_on_invalid_object_name(self._name) self._field_create(cr, context=context) def __init__(self, cr): @@ -2367,6 +2398,7 @@ class orm(orm_template): self._table, column['attname']) def _auto_init(self, cr, context=None): + raise_on_invalid_object_name(self._name) if context is None: context = {} store_compute = False diff --git a/openerp/osv/osv.py b/openerp/osv/osv.py index be3b227d9f9..3a5fdd05a14 100644 --- a/openerp/osv/osv.py +++ b/openerp/osv/osv.py @@ -23,18 +23,20 @@ # OSV: Objects Services # +import sys +import inspect import orm import openerp.netsvc as netsvc import openerp.pooler as pooler +import openerp.sql_db as sql_db import copy import logging from psycopg2 import IntegrityError, errorcodes from openerp.tools.func import wraps from openerp.tools.translate import translate -module_list = [] +# Mapping between openerp module names and their osv classes. module_class_list = {} -class_pool = {} class except_osv(Exception): def __init__(self, name, value, exc_type='warning'): @@ -92,7 +94,7 @@ class object_proxy(netsvc.Service): ids = args[3] else: ids = [] - cr = pooler.get_db_only(dbname).cursor() + cr = sql_db.db_connect(db_name).cursor() return src(obj, cr, uid, ids, context=(ctx or {})) except Exception: pass @@ -103,7 +105,7 @@ class object_proxy(netsvc.Service): # be returned, it is the best we have. try: - cr = pooler.get_db_only(dbname).cursor() + cr = sql_db.db_connect(db_name).cursor() res = translate(cr, name=False, source_type=ttype, lang=lang, source=src) if res: @@ -202,19 +204,22 @@ class object_proxy(netsvc.Service): cr.close() return res -object_proxy() class osv_pool(object): + """ Model registry for a particular database. + + The registry is essentially a mapping between model names and model + instances. There is one registry instance per database. + + """ + def __init__(self): self._ready = False - self.obj_pool = {} - self.module_object_list = {} - self.created = [] + self.obj_pool = {} # model name/model instance mapping self._sql_error = {} self._store_function = {} self._init = True self._init_parent = {} - self.logger = logging.getLogger("pool") def init_set(self, cr, mode): different = mode != self._init @@ -229,46 +234,74 @@ class osv_pool(object): self._ready = True return different - def obj_list(self): + """ Return the list of model names in this registry.""" return self.obj_pool.keys() - # adds a new object instance to the object pool. - # if it already existed, the instance is replaced - def add(self, name, obj_inst): - if name in self.obj_pool: - del self.obj_pool[name] - self.obj_pool[name] = obj_inst - module = obj_inst.__class__.__module__.split('.')[0] - self.module_object_list.setdefault(module, []).append(obj_inst) + def add(self, model_name, model): + """ Add or replace a model in the registry.""" + self.obj_pool[model_name] = model - # Return None if object does not exist def get(self, name): - obj = self.obj_pool.get(name, None) - return obj + """ Return a model for a given name or None if it doesn't exist.""" + return self.obj_pool.get(name) #TODO: pass a list of modules to load def instanciate(self, module, cr): + """ Instanciate all the classes of a given module for a particular db.""" + res = [] - class_list = module_class_list.get(module, []) - for klass in class_list: + + # instanciate classes registered through their constructor + for klass in module_class_list.get(module, []): res.append(klass.createInstance(self, module, cr)) + return res class osv_base(object): + """ Base class for openerp models. + + OpenERP models are created by inheriting from this class (although + not directly; more specifically by inheriting from osv or + osv_memory). The constructor is called once, usually directly + after the class definition, e.g.: + + class user(osv): + ... + user() + + The system will later instanciate the class once per database (on + which the class' module is installed). + + """ + def __init__(self, pool, cr): + """ Initialize a model and make it part of the given registry.""" pool.add(self._name, self) self.pool = pool super(osv_base, self).__init__(cr) def __new__(cls): + """ Register this model. + + This doesn't create an instance but simply register the model + as being part of the module where it is defined. + + TODO make it possible to not even have to call the constructor + to be registered. + + """ + + # Set the module name (e.g. base, sale, accounting, ...) on the class. module = cls.__module__.split('.')[0] if not hasattr(cls, '_module'): cls._module = module + + # Remember which models to instanciate for this module. module_class_list.setdefault(cls._module, []).append(cls) - class_pool[cls._name] = cls - if module not in module_list: - module_list.append(cls._module) + + # Since we don't return an instance here, the __init__ + # method won't be called. return None class osv_memory(osv_base, orm.orm_memory): @@ -365,5 +398,8 @@ class osv(osv_base, orm.orm): return obj createInstance = classmethod(createInstance) +def start_object_proxy(): + object_proxy() + # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/pooler.py b/openerp/pooler.py index 3b9875c18fc..aace66a5c84 100644 --- a/openerp/pooler.py +++ b/openerp/pooler.py @@ -19,64 +19,50 @@ # ############################################################################## -pool_dic = {} +""" Functions kept for backward compatibility. + + They are simple wrappers around a global RegistryManager methods. + +""" + +from openerp.modules.registry import RegistryManager + +_Registries = None + + +def ensure_registries(): + global _Registries + if _Registries is None: + _Registries = RegistryManager() + def get_db_and_pool(db_name, force_demo=False, status=None, update_module=False, pooljobs=True): - if not status: - status={} + """Create and return a database connection and a newly initialized registry.""" + ensure_registries() + bound_registry = _Registries.get(db_name, force_demo, status, update_module, pooljobs) + return bound_registry.db, bound_registry.registry - db = get_db_only(db_name) - if db_name in pool_dic: - pool = pool_dic[db_name] - else: - import openerp.addons as addons - import openerp.osv.osv as osv_osv - pool = osv_osv.osv_pool() - pool_dic[db_name] = pool - - try: - addons.load_modules(db, force_demo, status, update_module) - except Exception: - del pool_dic[db_name] - raise - - cr = db.cursor() - try: - pool.init_set(cr, False) - pool.get('ir.actions.report.xml').register_all(cr) - cr.commit() - finally: - cr.close() - - if pooljobs: - pool.get('ir.cron').restart(db.dbname) - return db, pool +def delete_pool(db_name): + """Delete an existing registry.""" + ensure_registries() + _Registries.delete(db_name) def restart_pool(db_name, force_demo=False, status=None, update_module=False): - if db_name in pool_dic: - del pool_dic[db_name] - return get_db_and_pool(db_name, force_demo, status, update_module=update_module) - - -def get_db_only(db_name): - # ATTENTION: - # do not put this import outside this function - # sql_db must not be loaded before the logger is initialized. - # sql_db import psycopg2.tool which create a default logger if there is not. - # this resulting of having the logs outputed twice... - import openerp.sql_db as sql_db - db = sql_db.db_connect(db_name) - return db + """Delete an existing registry and return a database connection and a newly initialized registry.""" + ensure_registries() + bound_registry = _Registries.new(db_name, force_demo, status, update_module, True) + return bound_registry.db, bound_registry.registry def get_db(db_name): + """Return a database connection. The corresponding registry is initialized.""" return get_db_and_pool(db_name)[0] def get_pool(db_name, force_demo=False, status=None, update_module=False): - pool = get_db_and_pool(db_name, force_demo, status, update_module)[1] - return pool + """Return a model registry.""" + return get_db_and_pool(db_name, force_demo, status, update_module)[1] # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/report/interface.py b/openerp/report/interface.py index 5bb1224a549..178c015ba34 100644 --- a/openerp/report/interface.py +++ b/openerp/report/interface.py @@ -27,7 +27,7 @@ import openerp.netsvc as netsvc import openerp.pooler as pooler import openerp.tools as tools -import openerp.addons as addons +import openerp.modules import print_xml import render import urllib @@ -119,7 +119,7 @@ class report_rml(report_int): pos_xml = i.end() doc = print_xml.document(cr, uid, {}, {}) - tmpl_path = addons.get_module_resource('base', 'report', 'corporate_defaults.xml') + tmpl_path = openerp.modules.get_module_resource('base', 'report', 'corporate_defaults.xml') doc.parse(tmpl_path, [uid], 'res.users', context) corporate_header = doc.xml_get() doc.close() diff --git a/openerp/report/render/__init__.py b/openerp/report/render/__init__.py index 1d0a4464e0b..1becd5ee514 100644 --- a/openerp/report/render/__init__.py +++ b/openerp/report/render/__init__.py @@ -27,7 +27,7 @@ try: import Image except ImportError: import logging - logging.getLogger('init').warning('Python Imaging not installed, you can use only .JPG pictures !') + logging.warning('Python Imaging not installed, you can use only .JPG pictures !') # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/service/__init__.py b/openerp/service/__init__.py index 529acbec628..a5f28e95678 100644 --- a/openerp/service/__init__.py +++ b/openerp/service/__init__.py @@ -19,6 +19,8 @@ # ############################################################################## +import http_server +import netrpc_server import web_services diff --git a/openerp/service/web_services.py b/openerp/service/web_services.py index 04bf78fce10..10ab738c15b 100644 --- a/openerp/service/web_services.py +++ b/openerp/service/web_services.py @@ -28,12 +28,12 @@ import time import sys import platform from openerp.tools.translate import _ -import openerp.addons as addons import openerp.netsvc as netsvc import openerp.pooler as pooler import openerp.release as release import openerp.sql_db as sql_db import openerp.tools as tools +import openerp.modules import locale import logging from cStringIO import StringIO @@ -90,12 +90,7 @@ class db(netsvc.ExportService): cr = None try: serv.actions[id]['progress'] = 0 - cr = sql_db.db_connect(db_name).cursor() - tools.init_db(cr) tools.config['lang'] = lang - cr.commit() - cr.close() - cr = None pool = pooler.restart_pool(db_name, demo, serv.actions[id], update_module=True)[1] @@ -138,7 +133,7 @@ class db(netsvc.ExportService): def exp_get_progress(self, id): if self.actions[id]['thread'].isAlive(): -# return addons.init_progress[db_name] +# return openerp.modules.init_progress[db_name] return (min(self.actions[id].get('progress', 0),0.95), []) else: clean = self.actions[id]['clean'] @@ -153,6 +148,7 @@ class db(netsvc.ExportService): def exp_drop(self, db_name): sql_db.close_db(db_name) + openerp.netsvc.Agent.cancel(db_name) logger = netsvc.Logger() db = sql_db.db_connect('template1') @@ -255,6 +251,7 @@ class db(netsvc.ExportService): def exp_rename(self, old_name, new_name): sql_db.close_db(old_name) + openerp.netsvc.Agent.cancel(db_name) logger = netsvc.Logger() db = sql_db.db_connect('template1') @@ -345,7 +342,6 @@ class db(netsvc.ExportService): l.notifyChannel('web-services', netsvc.LOG_ERROR, tb_s) raise return True -db() class _ObjectService(netsvc.ExportService): "A common base class for those who have fn(db, uid, password,...) " @@ -428,7 +424,7 @@ GNU Public Licence. if not rc.id: raise tm.RemoteContractException('This contract does not exist or is not active') - return rc.get_available_updates(rc.id, addons.get_modules_with_version()) + return rc.get_available_updates(rc.id, openerp.modules.get_modules_with_version()) except tm.RemoteContractException, e: self.abortResponse(1, 'Migration Error', 'warning', str(e)) @@ -446,7 +442,7 @@ GNU Public Licence. l.notifyChannel('migration', netsvc.LOG_INFO, 'starting migration with contract %s' % (rc.name,)) - zips = rc.retrieve_updates(rc.id, addons.get_modules_with_version()) + zips = rc.retrieve_updates(rc.id, openerp.modules.get_modules_with_version()) from shutil import rmtree, copytree, copy @@ -458,7 +454,7 @@ GNU Public Licence. for module in zips: l.notifyChannel('migration', netsvc.LOG_INFO, 'upgrade module %s' % (module,)) - mp = addons.get_module_path(module) + mp = openerp.modules.get_module_path(module) if mp: if os.path.isdir(mp): copytree(mp, os.path.join(backup_directory, module)) @@ -559,7 +555,6 @@ GNU Public Licence. logger.warning("Counters of SQL will not be reliable unless DEBUG_SQL is set at the server's config.") return sql_db.sql_counter -common() class objects_proxy(netsvc.ExportService): def __init__(self, name="object"): @@ -583,8 +578,6 @@ class objects_proxy(netsvc.ExportService): def new_dispatch(self,method,auth,params): pass -objects_proxy() - # # Wizard ID: 1 @@ -645,7 +638,6 @@ class wizard(netsvc.ExportService): raise Exception, 'AccessDenied' else: raise Exception, 'WizardNotFound' -wizard() # # TODO: set a maximum report number per user to avoid DOS attacks @@ -759,7 +751,13 @@ class report_spool(netsvc.ExportService): else: raise Exception, 'ReportNotFound' -report_spool() + +def start_web_services(): + db() + common() + objects_proxy() + wizard() + report_spool() # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/sql_db.py b/openerp/sql_db.py index 8afd684d3f1..e759a13e1d3 100644 --- a/openerp/sql_db.py +++ b/openerp/sql_db.py @@ -54,7 +54,6 @@ psycopg2.extensions.register_type(psycopg2.extensions.new_type((700, 701, 1700,) import tools from tools.func import wraps, frame_codeinfo -from netsvc import Agent from datetime import datetime as mdt from datetime import timedelta import threading @@ -68,17 +67,22 @@ sql_counter = 0 class Cursor(object): IN_MAX = 1000 # decent limit on size of IN queries - guideline = Oracle limit - __logger = logging.getLogger('db.cursor') + __logger = None def check(f): @wraps(f) def wrapper(self, *args, **kwargs): if self.__closed: - raise psycopg2.OperationalError('Unable to use the cursor after having closed it') + msg = 'Unable to use a closed cursor.' + if self.__closer: + msg += ' It was closed at %s, line %s' % self.__closer + raise psycopg2.OperationalError(msg) return f(self, *args, **kwargs) return wrapper def __init__(self, pool, dbname, serialized=False): + if self.__class__.__logger is None: + self.__class__.__logger = logging.getLogger('db.cursor') self.sql_from_log = {} self.sql_into_log = {} @@ -100,6 +104,7 @@ class Cursor(object): self.__caller = frame_codeinfo(currentframe(),2) else: self.__caller = False + self.__closer = False def __del__(self): if not self.__closed: @@ -198,6 +203,8 @@ class Cursor(object): if not self._obj: return + if self.sql_log: + self.__closer = frame_codeinfo(currentframe(),3) self.print_log() if not self._serialized: @@ -375,15 +382,18 @@ def dsn_are_equals(first, second): return key(first) == key(second) -_Pool = ConnectionPool(int(tools.config['db_maxconn'])) +_Pool = None def db_connect(db_name): + global _Pool + if _Pool is None: + _Pool = ConnectionPool(int(tools.config['db_maxconn'])) currentThread().dbname = db_name return Connection(_Pool, db_name) def close_db(db_name): + """ You might want to call openerp.netsvc.Agent.cancel(db_name) along this function.""" _Pool.close_all(dsn(db_name)) - Agent.cancel(db_name) tools.cache.clean_caches_for_db(db_name) ct = currentThread() if hasattr(ct, 'dbname'): diff --git a/openerp/tools/config.py b/openerp/tools/config.py index 3119e511269..8fb1ecd04f4 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -39,67 +39,21 @@ def check_ssl(): class configmanager(object): def __init__(self, fname=None): + # Options not exposed on the command line. Command line options will be added + # from optparse's parser. self.options = { - 'email_from':False, - 'xmlrpc_interface': '', # this will bind the server to all interfaces - 'xmlrpc_port': 8069, - 'netrpc_interface': '', - 'netrpc_port': 8070, - 'xmlrpcs_interface': '', # this will bind the server to all interfaces - 'xmlrpcs_port': 8071, - 'db_host': False, - 'db_port': False, - 'db_name': False, - 'db_user': False, - 'db_password': False, - 'db_maxconn': 64, - 'reportgz': False, - 'netrpc': True, - 'xmlrpc': True, - 'xmlrpcs': True, - 'translate_in': None, - 'translate_out': None, - 'overwrite_existing_translations': False, - 'load_language': None, - 'language': None, - 'pg_path': None, 'admin_passwd': 'admin', 'csv_internal_sep': ',', - 'addons_path': None, - 'root_path': None, - 'debug_mode': False, - 'import_partial': "", - 'pidfile': None, - 'logfile': None, - 'logrotate': True, - 'smtp_server': 'localhost', - 'smtp_user': False, - 'smtp_port':25, - 'smtp_ssl':False, - 'smtp_password': False, - 'stop_after_init': False, # this will stop the server after initialization - 'syslog' : False, - 'log_level': logging.INFO, - 'assert_exit_level': logging.ERROR, # level above which a failed assert will be raised - 'cache_timeout': 100000, 'login_message': False, - 'list_db' : True, - 'timezone' : False, # to override the default TZ - 'test_file' : False, - 'test_report_directory' : False, - 'test_disable' : False, - 'test_commit' : False, - 'static_http_enable': False, - 'static_http_document_root': None, - 'static_http_url_prefix': None, - 'secure_cert_file': 'server.cert', - 'secure_pkey_file': 'server.pkey', 'publisher_warranty_url': 'http://services.openerp.com/publisher-warranty/', - 'osv_memory_count_limit': None, # number of records in each osv_memory virtual table - 'osv_memory_age_limit': 1, # hours + 'reportgz': False, + 'root_path': None, } - self.blacklist_for_save = set(["publisher_warranty_url", "load_language", "root_path"]) + # Not exposed in the configuration file. + self.blacklist_for_save = set( + ['publisher_warranty_url', 'load_language', 'root_path', + 'init', 'save', 'config', 'update']) self.misc = {} self.config_file = fname @@ -122,15 +76,18 @@ class configmanager(object): group.add_option("--without-demo", dest="without_demo", help="disable loading demo data for modules to be installed (comma-separated, use \"all\" for all modules). Requires -d and -i. Default is %default", default=False) - group.add_option("-P", "--import-partial", dest="import_partial", - help="Use this for big data importation, if it crashes you will be able to continue at the current state. Provide a filename to store intermediate importation states.", default=False) + group.add_option("-P", "--import-partial", dest="import_partial", default='', + help="Use this for big data importation, if it crashes you will be able to continue at the current state. Provide a filename to store intermediate importation states.") group.add_option("--pidfile", dest="pidfile", help="file where the server pid will be stored") parser.add_option_group(group) group = optparse.OptionGroup(parser, "XML-RPC Configuration") - group.add_option("--xmlrpc-interface", dest="xmlrpc_interface", help="specify the TCP IP address for the XML-RPC protocol") - group.add_option("--xmlrpc-port", dest="xmlrpc_port", help="specify the TCP port for the XML-RPC protocol", type="int") - group.add_option("--no-xmlrpc", dest="xmlrpc", action="store_false", help="disable the XML-RPC protocol") + group.add_option("--xmlrpc-interface", dest="xmlrpc_interface", default='', + help="Specify the TCP IP address for the XML-RPC protocol. The empty string binds to all interfaces.") + group.add_option("--xmlrpc-port", dest="xmlrpc_port", default=8069, + help="specify the TCP port for the XML-RPC protocol", type="int") + group.add_option("--no-xmlrpc", dest="xmlrpc", action="store_false", default=True, + help="disable the XML-RPC protocol") parser.add_option_group(group) title = "XML-RPC Secure Configuration" @@ -138,18 +95,26 @@ class configmanager(object): title += " (disabled as ssl is unavailable)" group = optparse.OptionGroup(parser, title) - group.add_option("--xmlrpcs-interface", dest="xmlrpcs_interface", help="specify the TCP IP address for the XML-RPC Secure protocol") - group.add_option("--xmlrpcs-port", dest="xmlrpcs_port", help="specify the TCP port for the XML-RPC Secure protocol", type="int") - group.add_option("--no-xmlrpcs", dest="xmlrpcs", action="store_false", help="disable the XML-RPC Secure protocol") - group.add_option("--cert-file", dest="secure_cert_file", help="specify the certificate file for the SSL connection") - group.add_option("--pkey-file", dest="secure_pkey_file", help="specify the private key file for the SSL connection") + group.add_option("--xmlrpcs-interface", dest="xmlrpcs_interface", default='', + help="Specify the TCP IP address for the XML-RPC Secure protocol. The empty string binds to all interfaces.") + group.add_option("--xmlrpcs-port", dest="xmlrpcs_port", default=8071, + help="specify the TCP port for the XML-RPC Secure protocol", type="int") + group.add_option("--no-xmlrpcs", dest="xmlrpcs", action="store_false", default=True, + help="disable the XML-RPC Secure protocol") + group.add_option("--cert-file", dest="secure_cert_file", default='server.cert', + help="specify the certificate file for the SSL connection") + group.add_option("--pkey-file", dest="secure_pkey_file", default='server.pkey', + help="specify the private key file for the SSL connection") parser.add_option_group(group) # NET-RPC group = optparse.OptionGroup(parser, "NET-RPC Configuration") - group.add_option("--netrpc-interface", dest="netrpc_interface", help="specify the TCP IP address for the NETRPC protocol") - group.add_option("--netrpc-port", dest="netrpc_port", help="specify the TCP port for the NETRPC protocol", type="int") - group.add_option("--no-netrpc", dest="netrpc", action="store_false", help="disable the NETRPC protocol") + group.add_option("--netrpc-interface", dest="netrpc_interface", default='', + help="specify the TCP IP address for the NETRPC protocol") + group.add_option("--netrpc-port", dest="netrpc_port", default=8070, + help="specify the TCP port for the NETRPC protocol", type="int") + group.add_option("--no-netrpc", dest="netrpc", action="store_false", default=True, + help="disable the NETRPC protocol") parser.add_option_group(group) # Static HTTP @@ -161,45 +126,60 @@ class configmanager(object): # Testing Group group = optparse.OptionGroup(parser, "Testing Configuration") - group.add_option("--test-file", dest="test_file", help="Launch a YML test file.") - group.add_option("--test-report-directory", dest="test_report_directory", help="If set, will save sample of all reports in this directory.") + group.add_option("--test-file", dest="test_file", default=False, + help="Launch a YML test file.") + group.add_option("--test-report-directory", dest="test_report_directory", default=False, + help="If set, will save sample of all reports in this directory.") group.add_option("--test-disable", action="store_true", dest="test_disable", default=False, help="Disable loading test files.") group.add_option("--test-commit", action="store_true", dest="test_commit", default=False, help="Commit database changes performed by tests.") group.add_option("--assert-exit-level", dest='assert_exit_level', type="choice", choices=self._LOGLEVELS.keys(), - help="specify the level at which a failed assertion will stop the server. Accepted values: %s" % (self._LOGLEVELS.keys(),)) + default='error', + help="specify the level at which a failed assertion will stop the server. Accepted values: %s" % (self._LOGLEVELS.keys(),)) parser.add_option_group(group) # Logging Group group = optparse.OptionGroup(parser, "Logging Configuration") group.add_option("--logfile", dest="logfile", help="file where the server log will be stored") - group.add_option("--no-logrotate", dest="logrotate", action="store_false", + group.add_option("--no-logrotate", dest="logrotate", action="store_false", default=True, help="do not rotate the logfile") group.add_option("--syslog", action="store_true", dest="syslog", default=False, help="Send the log to the syslog server") group.add_option('--log-level', dest='log_level', type='choice', choices=self._LOGLEVELS.keys(), + default='info', help='specify the level of the logging. Accepted values: ' + str(self._LOGLEVELS.keys())) parser.add_option_group(group) # SMTP Group group = optparse.OptionGroup(parser, "SMTP Configuration") - group.add_option('--email-from', dest='email_from', help='specify the SMTP email address for sending email') - group.add_option('--smtp', dest='smtp_server', help='specify the SMTP server for sending email') - group.add_option('--smtp-port', dest='smtp_port', help='specify the SMTP port', type="int") - group.add_option('--smtp-ssl', dest='smtp_ssl', action='store_true', help='specify the SMTP server support SSL or not') - group.add_option('--smtp-user', dest='smtp_user', help='specify the SMTP username for sending email') - group.add_option('--smtp-password', dest='smtp_password', help='specify the SMTP password for sending email') + group.add_option('--email-from', dest='email_from', default=False, + help='specify the SMTP email address for sending email') + group.add_option('--smtp', dest='smtp_server', default='localhost', + help='specify the SMTP server for sending email') + group.add_option('--smtp-port', dest='smtp_port', default=25, + help='specify the SMTP port', type="int") + group.add_option('--smtp-ssl', dest='smtp_ssl', action='store_true', default=False, + help='specify the SMTP server support SSL or not') + group.add_option('--smtp-user', dest='smtp_user', default=False, + help='specify the SMTP username for sending email') + group.add_option('--smtp-password', dest='smtp_password', default=False, + help='specify the SMTP password for sending email') parser.add_option_group(group) group = optparse.OptionGroup(parser, "Database related options") - group.add_option("-d", "--database", dest="db_name", help="specify the database name") - group.add_option("-r", "--db_user", dest="db_user", help="specify the database user name") - group.add_option("-w", "--db_password", dest="db_password", help="specify the database password") + group.add_option("-d", "--database", dest="db_name", default=False, + help="specify the database name") + group.add_option("-r", "--db_user", dest="db_user", default=False, + help="specify the database user name") + group.add_option("-w", "--db_password", dest="db_password", default=False, + help="specify the database password") group.add_option("--pg_path", dest="pg_path", help="specify the pg executable path") - group.add_option("--db_host", dest="db_host", help="specify the database host") - group.add_option("--db_port", dest="db_port", help="specify the database port", type="int") - group.add_option("--db_maxconn", dest="db_maxconn", type='int', + group.add_option("--db_host", dest="db_host", default=False, + help="specify the database host") + group.add_option("--db_port", dest="db_port", default=False, + help="specify the database port", type="int") + group.add_option("--db_maxconn", dest="db_maxconn", type='int', default=64, help="specify the the maximum number of physical connections to posgresql") parser.add_option_group(group) @@ -211,6 +191,7 @@ class configmanager(object): group.add_option('--load-language', dest="load_language", help="specifies the languages for the translations you want to be loaded") group.add_option('-l', "--language", dest="language", + default=None, help="specify the language of the translation file. Use it with --i18n-export or --i18n-import") group.add_option("--i18n-export", dest="translate_out", help="export all sentences to be translated to a CSV file, a PO file or a TGZ archive and exit") @@ -226,17 +207,19 @@ class configmanager(object): parser.add_option_group(group) security = optparse.OptionGroup(parser, 'Security-related options') - security.add_option('--no-database-list', action="store_false", dest='list_db', help="disable the ability to return the list of databases") + security.add_option('--no-database-list', action="store_false", dest='list_db', default=True, + help="disable the ability to return the list of databases") parser.add_option_group(security) # Advanced options group = optparse.OptionGroup(parser, "Advanced options") - group.add_option("--cache-timeout", dest="cache_timeout", + group.add_option("--cache-timeout", dest="cache_timeout", default=100000, help="set the timeout for the cache system", type="int") group.add_option('--debug', dest='debug_mode', action='store_true', default=False, help='enable debug mode') group.add_option("--stop-after-init", action="store_true", dest="stop_after_init", default=False, - help="stop the server after it initializes") - group.add_option("-t", "--timezone", dest="timezone", help="specify reference timezone for the server (e.g. Europe/Brussels") + help="stop the server after its initialization") + group.add_option("-t", "--timezone", dest="timezone", default=False, + help="specify reference timezone for the server (e.g. Europe/Brussels") group.add_option("--osv-memory-count-limit", dest="osv_memory_count_limit", default=False, help="Force a limit on the maximum number of records kept in the virtual " "osv_memory tables. The default is False, which means no count-based limit.", @@ -248,6 +231,14 @@ class configmanager(object): type="float") parser.add_option_group(group) + # Copy all optparse options into self.options. + for group in parser.option_groups: + for option in group.option_list: + if option.default == ('NO', 'DEFAULT'): + self.options[option.dest] = None + else: + self.options[option.dest] = option.default + self.parse_config() def parse_config(self, args=[]): diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py index e1a6a717e87..5aa01d4f267 100644 --- a/openerp/tools/misc.py +++ b/openerp/tools/misc.py @@ -69,76 +69,6 @@ _logger = logging.getLogger('tools') # We include the *Base ones just in case, currently they seem to be subclasses of the _* ones. SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase) -# initialize a database with base/base.sql -def init_db(cr): - import openerp.addons as addons - f = addons.get_module_resource('base', 'base.sql') - base_sql_file = file_open(f) - try: - cr.execute(base_sql_file.read()) - cr.commit() - finally: - base_sql_file.close() - - for i in addons.get_modules(): - mod_path = addons.get_module_path(i) - if not mod_path: - continue - - info = addons.load_information_from_description_file(i) - - if not info: - continue - categs = info.get('category', 'Uncategorized').split('/') - p_id = None - while categs: - if p_id is not None: - cr.execute('SELECT id \ - FROM ir_module_category \ - WHERE name=%s AND parent_id=%s', (categs[0], p_id)) - else: - cr.execute('SELECT id \ - FROM ir_module_category \ - WHERE name=%s AND parent_id IS NULL', (categs[0],)) - c_id = cr.fetchone() - if not c_id: - cr.execute('INSERT INTO ir_module_category \ - (name, parent_id) \ - VALUES (%s, %s) RETURNING id', (categs[0], p_id)) - c_id = cr.fetchone()[0] - else: - c_id = c_id[0] - p_id = c_id - categs = categs[1:] - - active = info.get('active', False) - installable = info.get('installable', True) - if installable: - if active: - state = 'to install' - else: - state = 'uninstalled' - else: - state = 'uninstallable' - cr.execute('INSERT INTO ir_module_module \ - (author, website, name, shortdesc, description, \ - category_id, state, certificate, web, license) \ - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id', ( - info.get('author', ''), - info.get('website', ''), i, info.get('name', False), - info.get('description', ''), p_id, state, info.get('certificate') or None, - info.get('web') or False, - info.get('license') or 'AGPL-3')) - id = cr.fetchone()[0] - cr.execute('INSERT INTO ir_model_data \ - (name,model,module, res_id, noupdate) VALUES (%s,%s,%s,%s,%s)', ( - 'module_meta_information', 'ir.module.module', i, id, True)) - dependencies = info.get('depends', []) - for d in dependencies: - cr.execute('INSERT INTO ir_module_module_dependency \ - (module_id,name) VALUES (%s, %s)', (id, d)) - cr.commit() - def find_in_path(name): try: return which(name) @@ -204,8 +134,8 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False): @return: fileobject if pathinfo is False else (fileobject, filepath) """ - import openerp.addons as addons - adps = addons.ad_paths + import openerp.modules as addons + adps = addons.module.ad_paths rtp = os.path.normcase(os.path.abspath(config['root_path'])) if name.replace(os.path.sep, '/').startswith('addons/'): @@ -717,7 +647,28 @@ def is_hashable(h): except TypeError: return False -class cache(object): +class dummy_cache(object): + """ Cache decorator replacement to actually do no caching. + + This can be useful to benchmark and/or track memory leak. + + """ + + def __init__(self, timeout=None, skiparg=2, multi=None, size=8192): + pass + + def clear(self, dbname, *args, **kwargs): + pass + + @classmethod + def clean_caches_for_db(cls, dbname): + pass + + def __call__(self, fn): + fn.clear_cache = self.clear + return fn + +class real_cache(object): """ Use it as a decorator of the function you plan to cache Timeout: 0 = no timeout, otherwise in seconds @@ -842,6 +793,9 @@ class cache(object): cached_result.clear_cache = self.clear return cached_result +# TODO make it an option +cache = real_cache + def to_xml(s): return s.replace('&','&').replace('<','<').replace('>','>') diff --git a/openerp/tools/translate.py b/openerp/tools/translate.py index 9723135ff49..5761ba87cc0 100644 --- a/openerp/tools/translate.py +++ b/openerp/tools/translate.py @@ -27,6 +27,7 @@ import itertools import locale import os import openerp.pooler as pooler +import openerp.sql_db as sql_db import re import logging import tarfile @@ -160,7 +161,7 @@ class GettextAlias(object): # find current DB based on thread/worker db name (see netsvc) db_name = getattr(threading.currentThread(), 'dbname', None) if db_name: - return pooler.get_db_only(db_name) + return sql_db.db_connect(db_name) def _get_cr(self, frame): is_new_cr = False diff --git a/openerp/tools/yaml_import.py b/openerp/tools/yaml_import.py index 06d9526d77e..2b6c0815e0d 100644 --- a/openerp/tools/yaml_import.py +++ b/openerp/tools/yaml_import.py @@ -792,7 +792,7 @@ class YamlInterpreter(object): is_preceded_by_comment = False return is_preceded_by_comment -def yaml_import(cr, module, yamlfile, idref=None, mode='init', noupdate=False, report=None): +def yaml_import(cr, module, yamlfile, idref=None, mode='init', noupdate=False): if idref is None: idref = {} yaml_string = yamlfile.read() diff --git a/tools/module_graph.py b/tools/module_graph.py index 05bb058a447..b29bd316e2f 100755 --- a/tools/module_graph.py +++ b/tools/module_graph.py @@ -27,6 +27,7 @@ import optparse import sys import glob +# TODO use the same function provided in openerp.modules def load_information_from_description_file(module): """ :param module: The name of the module (sale, purchase, ...)