From 2af260f978976b8934e66393c09b9203ce123cf8 Mon Sep 17 00:00:00 2001 From: "nch@tinyerp.com" <> Date: Fri, 18 Jun 2010 12:42:04 +0530 Subject: [PATCH] [REVERT]:Integration script,Makefile changes need to create diff repository as it is affecting test.openobject.com:8010 bzr revid: nch@tinyerp.com-20100618071204-44ftuhvtun3yerzt --- Makefile | 32 +- bin/addons/base_quality_interrogation.py | 474 +++++++++++------------ 2 files changed, 261 insertions(+), 245 deletions(-) diff --git a/Makefile b/Makefile index e527a5e3ba4..782a3d51a3f 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,43 @@ # -*- makefile -*- addons-path := bin/addons/ +extra-addons:= '' root-path := bin/ port := 8069 net_port := 8070 +module := base +database := terp +language := fr_FR +i18n-import := bin/addons/base/i18n/fr_FR.po interrogation_file := bin/addons/base_quality_interrogation.py login := admin password := admin -openerp-test: - python $(interrogation_file) openerp-test --root-path=$(root-path) --addons-path=$(addons-path) --net_port=$(net_port) --port=$(port) --login=$(login) --password=$(password) +start: + python $(interrogation_file) start-server --root-path=$(root-path) --addons-path=$(addons-path) --port=$(port) --net_port=$(net_port) + +create-db: + python $(interrogation_file) create-db --database=$(database) --root-path=$(root-path) --addons-path=$(addons-path) --port=$(port) --net_port=$(net_port) --login=$(login) --password=$(password) + +drop-db: + python $(interrogation_file) drop-db --database=$(database) --root-path=$(root-path) --addons-path=$(addons-path) --port=$(port) --net_port=$(net_port) + +install-module: + python $(interrogation_file) install-module --modules=$(module) --database=$(database) --root-path=$(root-path) --addons-path=$(addons-path) --extra-addons=$(extra-addons) --net_port=$(net_port) --port=$(port) --login=$(login) --password=$(password) + +upgrade-module: + python $(interrogation_file) upgrade-module --modules=$(module) --database=$(database) --root-path=$(root-path) --addons-path=$(addons-path) --port=$(port) --net_port=$(net_port) --login=$(login) --password=$(password) + + +install-translation: + python $(interrogation_file) install-translation --database=$(database) --translate-in=$(i18n-import) --net_port=$(net_port) --port=$(port) --login=$(login) --password=$(password) --root-path=$(root-path) --addons-path=$(addons-path) + version: python bin/openerp-server.py --version + +check-quality: + python $(interrogation_file) check-quality --database=$(database) --modules=$(module) --net_port=$(net_port) --port=$(port) --login=$(login) --password=$(password) --addons-path=$(addons-path) --root-path=$(root-path) + + + diff --git a/bin/addons/base_quality_interrogation.py b/bin/addons/base_quality_interrogation.py index 74a3a11fe02..4e9c3321dfa 100755 --- a/bin/addons/base_quality_interrogation.py +++ b/bin/addons/base_quality_interrogation.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ############################################################################## -# +# # OpenERP, Open Source Management Solution # Copyright (C) 2004-2009 Tiny SPRL (). # @@ -15,11 +15,12 @@ # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . +# along with this program. If not, see . # ############################################################################## import xmlrpclib +import ConfigParser import optparse import sys import thread @@ -47,118 +48,8 @@ def to_decode(s): except UnicodeError: return s -def clear_log_dir(path): - dirpath = os.path.realpath(path) - if os.path.isdir(path): - for file in os.listdir(dirpath): - sub_path = dirpath + os.sep + file - if os.path.isdir(sub_path): - clear_log_dir(sub_path) - else: - os.remove(sub_path) - os.rmdir(path) - return True - def start_server(root_path, port, netport, addons_path): - logfile = 'test_logs/logs.txt' - os.system('python2.5 %sopenerp-server.py --pidfile=openerp.pid --no-xmlrpcs --xmlrpc-port=%s --netrpc-port=%s --addons-path=%s --logfile=%s' %(root_path, str(port),str(netport),addons_path, logfile)) - - -def create_db(url, dbname, user='admin', pwd='admin', lang='en_US'): - conn = xmlrpclib.ServerProxy(url + '/xmlrpc/db') - obj_conn = xmlrpclib.ServerProxy(url + '/xmlrpc/object') - wiz_conn = xmlrpclib.ServerProxy(url + '/xmlrpc/wizard') - login_conn = xmlrpclib.ServerProxy(url + '/xmlrpc/common') - db_list = execute(conn, 'list') - if dbname in db_list: - execute(conn, 'drop', admin_passwd, dbname) - id = execute(conn,'create',admin_passwd, dbname, True, lang) - wait(id,url) - return True - -def login(url, dbname, user, pwd): - conn = xmlrpclib.ServerProxy(url + '/xmlrpc/common') - uid = execute(conn,'login',dbname, user, pwd) - return uid - -def install_module(url='', uid=1, pwd='admin', dbname='', module=''): - obj_conn = xmlrpclib.ServerProxy(url + '/xmlrpc/object') - wizard_conn = xmlrpclib.ServerProxy(url + '/xmlrpc/wizard') - module_ids = execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', - 'search', [('name','in',[module,'base_module_quality'])]) - for action in ('button_install','button_upgrade'): - execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', action, module_ids) - wiz_id = execute(wizard_conn, 'create', dbname, uid, pwd, 'module.upgrade.simple') - state = 'init' - datas = {} - while state!='end': - res = execute(wizard_conn, 'execute', dbname, uid, pwd, wiz_id, datas, state, {}) - if state == 'init': - state = 'start' - elif state == 'start': - state = 'end' - return True - -def check_quality(url, user, pwd, dbname, module, quality_logs): - uid = login(url, dbname, user, pwd) - quality_logs += 'quality-logs' - if uid: - conn = xmlrpclib.ServerProxy(url + '/xmlrpc/object') - final = {} - qualityresult = {} - test_detail = {} - quality_result = execute(conn,'execute', dbname, uid, pwd,'module.quality.check','check_quality',module) - detail_html = '' - html = '''''' - html +="

Module: %s

"%(quality_result['name']) - html += "

Final score: %s

"%(quality_result['final_score']) - html += "
" - html += "
    " - for x,y,detail in quality_result['check_detail_ids']: - test = detail.get('name') - msg = detail.get('message','') - score = round(float(detail.get('score',0)),2) - html += "
  • %s
    • "%(test.replace(' ','-'),test) - detail_html +='''

    %s (Score : %s)

    %s
    %s
    '''%(test.replace(' ', '-'), test, score, msg, detail.get('detail', '')) - test_detail[test] = (score,msg,detail.get('detail','')) - html += "
" - html += "%s"%(detail_html) - html += "
" - if not os.path.isdir(quality_logs): - os.mkdir(quality_logs) - fp = open('%s/%s.html'%(quality_logs,module),'wb') - fp.write(to_decode(html)) - fp.close() - else: - print 'Login Failed...' - return True - -def install_translation(url, uid, pwd, dbname, po_files): - conn = xmlrpclib.ServerProxy(url + '/xmlrpc/wizard') - wiz_id = execute(conn,'create', dbname, uid, pwd, 'module.lang.import') - for file in po_files: - lang,ext = os.path.splitext(file.split('/')[-1]) - if ext == '.po': - state = 'init' - datas = {'form':{}} - while state!='end': - res = execute(conn,'execute',dbname, uid, pwd, wiz_id, datas, state, {}) - if 'datas' in res: - datas['form'].update( res['datas'].get('form',{}) ) - if res['type']=='form': - for field in res['fields'].keys(): - datas['form'][field] = res['fields'][field].get('value', False) - state = res['state'][-1][0] - trans_obj = open(file) - datas['form'].update({ - 'name': lang, - 'code': lang, - 'data' : base64.encodestring(trans_obj.read()) - }) - trans_obj.close() - elif res['type']=='action': - state = res['state'] - + os.system('python2.5 %sopenerp-server.py --pidfile=openerp.pid --no-xmlrpcs --xmlrpc-port=%s --netrpc-port=%s --addons-path=%s' %(root_path, str(port),str(netport),addons_path)) def clean(): if os.path.isfile('openerp.pid'): ps = open('openerp.pid') @@ -188,181 +79,278 @@ def execute(connector, method, *args): wait_count = 0 return res -def wait(id, url=''): +def login(uri, dbname, user, pwd): + conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/common') + uid = execute(conn,'login',dbname, user, pwd) + return uid + +def import_translate(uri, user, pwd, dbname, translate_in): + uid = login(uri, dbname, user, pwd) + if uid: + conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/wizard') + wiz_id = execute(conn,'create',dbname, uid, pwd, 'module.lang.import') + for trans_in in translate_in: + lang,ext = os.path.splitext(trans_in.split('/')[-1]) + state = 'init' + datas = {'form':{}} + while state!='end': + res = execute(conn,'execute',dbname, uid, pwd, wiz_id, datas, state, {}) + if 'datas' in res: + datas['form'].update( res['datas'].get('form',{}) ) + if res['type']=='form': + for field in res['fields'].keys(): + datas['form'][field] = res['fields'][field].get('value', False) + state = res['state'][-1][0] + trans_obj = open(trans_in) + datas['form'].update({ + 'name': lang, + 'code': lang, + 'data' : base64.encodestring(trans_obj.read()) + }) + trans_obj.close() + elif res['type']=='action': + state = res['state'] + + +def check_quality(uri, user, pwd, dbname, modules, quality_logs): + uid = login(uri, dbname, user, pwd) + quality_logs += 'quality-logs' + if uid: + conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/object') + final = {} + for module in modules: + qualityresult = {} + test_detail = {} + quality_result = execute(conn,'execute', dbname, uid, pwd,'module.quality.check','check_quality',module) + detail_html = '' + html = '''''' + html +="

Module: %s

"%(quality_result['name']) + html += "

Final score: %s

"%(quality_result['final_score']) + html += "
" + html += "
    " + for x,y,detail in quality_result['check_detail_ids']: + test = detail.get('name') + msg = detail.get('message','') + score = round(float(detail.get('score',0)),2) + html += "
  • %s
    • "%(test.replace(' ','-'),test) + detail_html +='''

    %s (Score : %s)

    %s
    %s
    '''%(test.replace(' ', '-'), test, score, msg, detail.get('detail', '')) + test_detail[test] = (score,msg,detail.get('detail','')) + html += "
" + html += "%s"%(detail_html) + html += "
" + if not os.path.isdir(quality_logs): + os.mkdir(quality_logs) + fp = open('%s/%s.html'%(quality_logs,module),'wb') + fp.write(to_decode(html)) + fp.close() + #final[quality_result['name']] = (quality_result['final_score'],html,test_detail) + + #fp = open('quality_log.pck','wb') + #pck_obj = pickle.dump(final,fp) + #fp.close() + #print "LOG PATH%s"%(os.path.realpath('quality_log.pck')) + return True + else: + print 'Login Failed...' + clean() + sys.exit(1) + + + +def wait(id,url=''): progress=0.0 sock2 = xmlrpclib.ServerProxy(url+'/xmlrpc/db') while not progress==1.0: progress,users = execute(sock2,'get_progress',admin_passwd, id) return True -def store_logs(module, step, msg='' , want_html_log = False): - logfile = open('test_logs/logs.txt','r') - mod_log = open('test_logs/%s/%s.txt'%(module, step),'w') - mod_log.write(logfile.read() + '\n' + msg) - mod_log.close() - logfile.close() - - logfile = open('test_logs/logs.txt','w') - logfile.write('') - logfile.close() - if want_html_log: - store_html_logs(module) +def create_db(uri, dbname, user='admin', pwd='admin', lang='en_US'): + conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/db') + obj_conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/object') + wiz_conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/wizard') + login_conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/common') + db_list = execute(conn, 'list') + if dbname in db_list: + drop_db(uri, dbname) + id = execute(conn,'create',admin_passwd, dbname, True, lang) + wait(id,uri) + install_module(uri, dbname, ['base_module_quality'],user=user,pwd=pwd) return True -def store_html_logs(module): - html = '''''' - html +="

Module: %s

"%(module) - html += "
" - html += "
    " - detail_html = '' - path = os.path.realpath('test_logs') + os.sep + module - for log in os.listdir(path): - log_fp = open(path + os.sep + log, 'r') - step_name, ext = os.path.splitext(log) - data = '' - for line in log_fp.readlines(): - data += line + '
    ' - html += "
  • %s
    • "%(step_name, step_name) - detail_html += '''

    %s

    %s
    '''%(step_name, step_name, data) - log_fp.close() - html += "
" - html += "%s"%(detail_html) - html += "
" - - html_fp = open(path + os.sep + module +'.html','wb') - html_fp.write(to_decode(html)) - html_fp.close() +def drop_db(uri, dbname): + conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/db') + db_list = execute(conn,'list') + if dbname in db_list: + execute(conn, 'drop', admin_passwd, dbname) return True -def openerp_test(url, rootpath = '', addons='', user='admin', pwd='admin', quality_log='', port=8069, netport=8070): - modules = os.listdir(addons) - for module in modules: - print "Testing Module: ", module - if module == '.bzr': - continue - dbname = module - try: - os.mkdir('test_logs' + os.sep + module) - except OSError: - os.remove('test_logs' + os.sep + module) - os.mkdir('test_logs' + os.sep + module) - # 1: Create Database also drop the database if it exists !: - try: - create_db(url, dbname, user, pwd) - store_logs(module, 'createDB') - except xmlrpclib.Fault, e: - print e.faultString - store_logs(module, 'createDB', want_html_log=True) - continue - except Exception, e: - print e - store_logs(module, 'createDB', want_html_log=True) - continue +def make_links(uri, uid, dbname, source, destination, module, user, pwd): + if module in ('base','quality_integration_server'): + return True + if os.path.islink(destination + '/' + module): + os.unlink(destination + '/' + module) + for path in source: + if os.path.isdir(path + '/' + module): + os.symlink(path + '/' + module, destination + '/' + module) + obj_conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/object') + execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', 'update_list') + module_ids = execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', 'search', [('name','=',module)]) + if len(module_ids): + data = execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', 'read', module_ids[0],['name','dependencies_id']) + dep_datas = execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module.dependency', 'read', data['dependencies_id'],['name']) + for dep_data in dep_datas: + make_links(uri, uid, dbname, source, destination, dep_data['name'], user, pwd) + return False - uid = login(url, dbname, user, pwd) - # 2: Install the module - try: - install_module(url, uid, pwd, dbname, module) - store_logs(module, 'Install_module', ) - except xmlrpclib.Fault, e: - print e.faultString - store_logs(module, 'Install_module', want_html_log=True) - continue - except Exception, e: - print e - store_logs(module, 'Install_module', want_html_log=True) - continue +def install_module(uri, dbname, modules, addons='', extra_addons='', user='admin', pwd='admin'): + uid = login(uri, dbname, user, pwd) + if extra_addons: + extra_addons = extra_addons.split(',') + if uid: + if addons and extra_addons: + for module in modules: + make_links(uri, uid, dbname, extra_addons, addons, module, user, pwd) - # 3: Check Quality of the module - try: - check_quality(url, user, pwd, dbname, module, quality_log) - except xmlrpclib.Fault, e: - print e.faultString - store_logs(module, 'Check_quality', want_html_log=True) - continue - except Exception, e: - print e - store_logs(module, 'Check_quality', want_html_log=True) - continue - - #4:Install Translations for the module - try: - if os.path.isdir(addons+'/'+module+'/'+'i18n'): - po_files = os.listdir(addons+'/'+module+'/'+'i18n') - po_file_paths = [] - for po_file in po_files: - if module == 'base': - po_link = '%saddons/%s/i18n/%s'%(rootpath, module, po_file) - else: - po_link = '%s/%s/i18n/%s'%(addons, module, po_file) - po_file_paths.append(po_link) - install_translation(url, uid, pwd, dbname, po_file_paths) - store_logs(module, 'Install_translation') - else: - msg = "Translations Not Available for %s! "%(module) - store_logs(module, 'Install_translation', msg) - except xmlrpclib.Fault, e: - print e.faultString - store_logs(module, 'Install_translation', want_html_log=True) - continue - except Exception, e: - print e - store_logs(module, 'Install_translation', want_html_log=True) - continue - store_html_logs(module) - sys.exit(0) + obj_conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/object') + wizard_conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/wizard') + module_ids = execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', 'search', [('name','in',modules)]) + execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', 'button_install', module_ids) + wiz_id = execute(wizard_conn, 'create', dbname, uid, pwd, 'module.upgrade.simple') + state = 'init' + datas = {} + #while state!='menu': + while state!='end': + res = execute(wizard_conn, 'execute', dbname, uid, pwd, wiz_id, datas, state, {}) + if state == 'init': + state = 'start' + elif state == 'start': + state = 'end' return True +def upgrade_module(uri, dbname, modules, user='admin', pwd='admin'): + uid = login(uri, dbname, user, pwd) + if uid: + obj_conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/object') + wizard_conn = xmlrpclib.ServerProxy(uri + '/xmlrpc/wizard') + module_ids = execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', 'search', [('name','in',modules)]) + execute(obj_conn, 'execute', dbname, uid, pwd, 'ir.module.module', 'button_upgrade', module_ids) + wiz_id = execute(wizard_conn, 'create', dbname, uid, pwd, 'module.upgrade.simple') + state = 'init' + datas = {} + #while state!='menu': + while state!='end': + res = execute(wizard_conn, 'execute', dbname, uid, pwd, wiz_id, datas, state, {}) + if state == 'init': + state = 'start' + elif state == 'start': + state = 'end' + + return True + + + + + usage = """%prog command [options] -Basic Command: - This command will do the following tests on the modules: - openerp-test: - 1: Start Server - 2: Create new database - 3: Drop database if it exists - 4: Install module from addons one by one. - 5: Install translation file for each module. - 6: Calculate quality and dump quality result into quality_log.pck using pickle +Basic Commands: + start-server Start Server + create-db Create new database + drop-db Drop database + install-module Install module + upgrade-module Upgrade module + install-translation Install translation file + check-quality Calculate quality and dump quality result into quality_log.pck using pickle """ parser = optparse.OptionParser(usage) +parser.add_option("--modules", dest="modules", + help="specify modules to install or check quality") parser.add_option("--addons-path", dest="addons_path", help="specify the addons path") -parser.add_option("--root-path", dest="root_path", help="specify the root path") parser.add_option("--quality-logs", dest="quality_logs", help="specify the path of quality logs files which has to stores") +parser.add_option("--root-path", dest="root_path", help="specify the root path") parser.add_option("-p", "--port", dest="port", help="specify the TCP port", type="int") parser.add_option("--net_port", dest="netport",help="specify the TCP port for netrpc") +parser.add_option("-d", "--database", dest="db_name", help="specify the database name") parser.add_option("--login", dest="login", help="specify the User Login") parser.add_option("--password", dest="pwd", help="specify the User Password") +parser.add_option("--translate-in", dest="translate_in", + help="specify .po files to import translation terms") +parser.add_option("--extra-addons", dest="extra_addons", + help="specify extra_addons and trunkCommunity modules path ") (opt, args) = parser.parse_args() - if len(args) != 1: parser.error("incorrect number of arguments") command = args[0] -if not command == 'openerp-test': +if command not in ('start-server','create-db','drop-db','install-module','upgrade-module','check-quality','install-translation'): parser.error("incorrect command") +def die(cond, msg): + if cond: + print msg + sys.exit(1) + +die(opt.modules and (not opt.db_name), + "the modules option cannot be used without the database (-d) option") + +die(opt.translate_in and (not opt.db_name), + "the translate-in option cannot be used without the database (-d) option") + options = { 'addons-path' : opt.addons_path or 'addons', 'quality-logs' : opt.quality_logs or '', 'root-path' : opt.root_path or '', + 'translate-in': [], 'port' : opt.port or 8069, 'netport':opt.netport or 8070, + 'database': opt.db_name or 'terp', + 'modules' : opt.modules or [], 'login' : opt.login or 'admin', 'pwd' : opt.pwd or '', + 'extra-addons':opt.extra_addons or [] } -url = 'http://localhost:' + str(options['port']) +options['modules'] = opt.modules and map(lambda m: m.strip(), opt.modules.split(',')) or [] +# Hint:i18n-import=purchase:ar_AR.po+sale:fr_FR.po,nl_BE.po +if opt.translate_in: + translate = opt.translate_in + for module_name,po_files in map(lambda x:tuple(x.split(':')),translate.split('+')): + for po_file in po_files.split(','): + if module_name == 'base': + po_link = '%saddons/%s/i18n/%s'%(options['root-path'],module_name,po_file) + else: + po_link = '%s/%s/i18n/%s'%(options['addons-path'], module_name, po_file) + options['translate-in'].append(po_link) + +uri = 'http://localhost:' + str(options['port']) server_thread = threading.Thread(target=start_server, args=(options['root-path'], options['port'],options['netport'], options['addons-path'])) +try: + server_thread.start() + if command == 'create-db': + create_db(uri, options['database'], options['login'], options['pwd']) + if command == 'drop-db': + drop_db(uri, options['database']) + if command == 'install-module': + install_module(uri, options['database'], options['modules'],options['addons-path'],options['extra-addons'],options['login'], options['pwd']) + if command == 'upgrade-module': + upgrade_module(uri, options['database'], options['modules'], options['login'], options['pwd']) + if command == 'check-quality': + check_quality(uri, options['login'], options['pwd'], options['database'], options['modules'], options['quality-logs']) + if command == 'install-translation': + import_translate(uri, options['login'], options['pwd'], options['database'], options['translate-in']) + clean() + sys.exit(0) -clear_log_dir('test_logs') -os.mkdir('test_logs') -server_thread.start() +except xmlrpclib.Fault, e: + print e.faultString + clean() + sys.exit(1) +except Exception, e: + print e + clean() + sys.exit(1) -openerp_test(url, options['root-path'], options['addons-path'], - options['login'], options['pwd'], options['quality-logs'], options['port'],options['netport']) -clean() # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: