[FIX] ir.translation: export/import of QWeb terms

Fixes the translation term import/export logic to
support terms inside QWeb templates.

Refactored a bit the export code so the babel-based
QWeb terms extractor for ./static/src/*.xml files
uses the same logic as the regular extractor for
ir.ui.views with type QWeb.

Server-side QWeb rendering uses a mix of the native
 view inheritance mechanism and the template inclusion
(t-call) mechanism. During rendering the translations
are only applied at "template" level, *after* the
view inheritance has already been resolved.
As a result translations are local to a template,
not to the inherited view in which they are actually
written.
In terms of exporting PO[T] files, this is done by
resolving the "root" QWeb template a view belongs
to, and using it as the location of the translated term.

During import there is one extra quirk for QWeb
terms: they need to be linked to the `website` model
rather than the actual `ir.ui.view` model they
are really pointing to, so the rendering phase can
properly recognize them.

openerp/addons/base/ir/ir_translation.py

@@ -78,6 +78,11 @@ class ir_translation_import_cursor(object):
         """Feed a translation, as a dictionary, into the cursor
         """
         params = dict(trans_dict, state="translated" if trans_dict['value'] else "to_translate")
+
+        # ugly hack for QWeb views - pending refactoring of translations in master
+        if params['imd_model'] == 'website' and params['type'] == 'view':
+            params['imd_model'] = "ir.ui.view"
+
         self._cr.execute("""INSERT INTO %s (name, lang, res_id, src, type, imd_model, module, imd_name, value, state, comments)
                             VALUES (%%(name)s, %%(lang)s, %%(res_id)s, %%(src)s, %%(type)s, %%(imd_model)s, %%(module)s,
                                     %%(imd_name)s, %%(value)s, %%(state)s, %%(comments)s)""" % self._table_name,
@@ -98,15 +103,14 @@ class ir_translation_import_cursor(object):
             FROM ir_model_data AS imd
             WHERE ti.res_id IS NULL
                 AND ti.module IS NOT NULL AND ti.imd_name IS NOT NULL
-
                 AND ti.module = imd.module AND ti.imd_name = imd.name
                 AND ti.imd_model = imd.model; """ % self._table_name)
 
         if self._debug:
-            cr.execute("SELECT module, imd_model, imd_name FROM %s " \
+            cr.execute("SELECT module, imd_name, imd_model FROM %s " \
                 "WHERE res_id IS NULL AND module IS NOT NULL" % self._table_name)
             for row in cr.fetchall():
-                _logger.debug("ir.translation.cursor: missing res_id for %s. %s/%s ", *row)
+                _logger.info("ir.translation.cursor: missing res_id for %s.%s <%s> ", *row)
 
         # Records w/o res_id must _not_ be inserted into our db, because they are
         # referencing non-existent data.
@@ -297,7 +301,7 @@ class ir_translation(osv.osv):
                         AND src=%s"""
             params = (lang or '', types, tools.ustr(source))
             if res_id:
-                query += "AND res_id=%s"
+                query += " AND res_id=%s"
                 params += (res_id,)
             if name:
                 query += " AND name=%s"

openerp/addons/base/ir/ir_ui_view.py

@@ -38,7 +38,7 @@ import openerp
 from openerp import tools, api
 from openerp.http import request
 from openerp.osv import fields, osv, orm
-from openerp.tools import graph, SKIPPED_ELEMENT_TYPES
+from openerp.tools import graph, SKIPPED_ELEMENT_TYPES, SKIPPED_ELEMENTS
 from openerp.tools.parse_version import parse_version
 from openerp.tools.safe_eval import safe_eval as eval
 from openerp.tools.view_validation import valid_view
@@ -957,7 +957,7 @@ class view(osv.osv):
                 return None
             return Translations._get_source(cr, uid, 'website', 'view', lang, text, id_)
 
-        if arch.tag not in ['script']:
+        if type(arch) not in SKIPPED_ELEMENT_TYPES and arch.tag not in SKIPPED_ELEMENTS:
             text = get_trans(arch.text)
             if text:
                 arch.text = arch.text.replace(arch.text.strip(), text)
@@ -965,7 +965,7 @@ class view(osv.osv):
             if tail:
                 arch.tail = arch.tail.replace(arch.tail.strip(), tail)
 
-            for attr_name in ('title', 'alt', 'placeholder'):
+            for attr_name in ('title', 'alt', 'label', 'placeholder'):
                 attr = get_trans(arch.get(attr_name))
                 if attr:
                     arch.set(attr_name, attr)

openerp/tools/translate.py

@@ -49,6 +49,8 @@ _logger = logging.getLogger(__name__)
 # used to notify web client that these translations should be loaded in the UI
 WEB_TRANSLATION_COMMENT = "openerp-web"
 
+SKIPPED_ELEMENTS = ('script', 'style')
+
 _LOCALE2WIN32 = {
     'af_ZA': 'Afrikaans_South Africa',
     'sq_AL': 'Albanian_Albania',
@@ -536,28 +538,34 @@ def trans_parse_rml(de):
         res.extend(trans_parse_rml(n))
     return res
 
-def trans_parse_view(de):
+def _push(callback, term, source_line):
-    res = []
+    """ Sanity check before pushing translation terms """
-    if not isinstance(de, SKIPPED_ELEMENT_TYPES) and de.text and de.text.strip():
+    term = (term or "").strip().encode('utf8')
-        res.append(de.text.strip().encode("utf8"))
+    # Avoid non-char tokens like ':' '...' '.00' etc.
-    if de.tail and de.tail.strip():
+    if len(term) > 8 or any(x.isalpha() for x in term):
-        res.append(de.tail.strip().encode("utf8"))
+        callback(term, source_line)
-    if de.tag == 'attribute' and de.get("name") == 'string':
+
-        if de.text:
+def trans_parse_view(element, callback):
-            res.append(de.text.encode("utf8"))
+    """ Helper method to recursively walk an etree document representing a
-    if de.get("string"):
+        regular view and call ``callback(term)`` for each translatable term
-        res.append(de.get('string').encode("utf8"))
+        that is found in the document.
-    if de.get("help"):
+
-        res.append(de.get('help').encode("utf8"))
+        :param ElementTree element: root of etree document to extract terms from
-    if de.get("sum"):
+        :param callable callback: a callable in the form ``f(term, source_line)``,
-        res.append(de.get('sum').encode("utf8"))
+            that will be called for each extracted term.
-    if de.get("confirm"):
+    """
-        res.append(de.get('confirm').encode("utf8"))
+    if (not isinstance(element, SKIPPED_ELEMENT_TYPES)
-    if de.get("placeholder"):
+            and element.tag.lower() not in SKIPPED_ELEMENTS
-        res.append(de.get('placeholder').encode("utf8"))
+            and element.text):
-    for n in de:
+        _push(callback, element.text, element.sourceline)
-        res.extend(trans_parse_view(n))
+    if element.tail:
-    return res
+        _push(callback, element.tail, element.sourceline)
+    for attr in ('string', 'help', 'sum', 'confirm', 'placeholder'):
+        value = element.get(attr)
+        if value:
+            _push(callback, value, element.sourceline)
+    for n in element:
+        trans_parse_view(n, callback)
 
 # tests whether an object is in a list of modules
 def in_modules(object_name, modules):
@@ -573,6 +581,30 @@ def in_modules(object_name, modules):
     module = module_dict.get(module, module)
     return module in modules
 
+def _extract_translatable_qweb_terms(element, callback):
+    """ Helper method to walk an etree document representing
+        a QWeb template, and call ``callback(term)`` for each
+        translatable term that is found in the document.
+
+        :param ElementTree element: root of etree document to extract terms from
+        :param callable callback: a callable in the form ``f(term, source_line)``,
+            that will be called for each extracted term.
+    """
+    # not using elementTree.iterparse because we need to skip sub-trees in case
+    # the ancestor element had a reason to be skipped
+    for el in element:
+        if isinstance(el, SKIPPED_ELEMENT_TYPES): continue
+        if (el.tag.lower() not in SKIPPED_ELEMENTS
+                and "t-js" not in el.attrib
+                and not ("t-jquery" in el.attrib and "t-operation" not in el.attrib)
+                and not ("t-translation" in el.attrib and
+                         el.attrib["t-translation"].strip() == "off")):
+            _push(callback, el.text, el.sourceline)
+            for att in ('title', 'alt', 'label', 'placeholder'):
+                if att in el.attrib:
+                    _push(callback, el.attrib[att], el.sourceline)
+            _extract_translatable_qweb_terms(el, callback)
+        _push(callback, el.tail, el.sourceline)
 
 def babel_extract_qweb(fileobj, keywords, comment_tags, options):
     """Babel message extractor for qweb template files.
@@ -588,31 +620,11 @@ def babel_extract_qweb(fileobj, keywords, comment_tags, options):
     """
     result = []
     def handle_text(text, lineno):
-        text = (text or "").strip()
+        result.append((lineno, None, text, []))
-        if len(text) > 1: # Avoid mono-char tokens like ':' ',' etc.
-            result.append((lineno, None, text, []))
-
-    # not using elementTree.iterparse because we need to skip sub-trees in case
-    # the ancestor element had a reason to be skipped
-    def iter_elements(current_element):
-        for el in current_element:
-            if isinstance(el, SKIPPED_ELEMENT_TYPES): continue
-            if "t-js" not in el.attrib and \
-                    not ("t-jquery" in el.attrib and "t-operation" not in el.attrib) and \
-                    not ("t-translation" in el.attrib and el.attrib["t-translation"].strip() == "off"):
-                handle_text(el.text, el.sourceline)
-                for att in ('title', 'alt', 'label', 'placeholder'):
-                    if att in el.attrib:
-                        handle_text(el.attrib[att], el.sourceline)
-                iter_elements(el)
-            handle_text(el.tail, el.sourceline)
-
     tree = etree.parse(fileobj)
-    iter_elements(tree.getroot())
+    _extract_translatable_qweb_terms(tree.getroot(), handle_text)
-
     return result
 
-
 def trans_generate(lang, modules, cr):
     dbname = cr.dbname
 
@@ -649,7 +661,6 @@ def trans_generate(lang, modules, cr):
         # empty and one-letter terms are ignored, they probably are not meant to be
         # translated, and would be very hard to translate anyway.
         if not source or len(source.strip()) <= 1:
-            _logger.debug("Ignoring empty or 1-letter source term: %r", tuple)
             return
         if tuple not in _to_translate:
             _to_translate.append(tuple)
@@ -659,6 +670,19 @@ def trans_generate(lang, modules, cr):
             return s.encode('utf8')
         return s
 
+    def push(mod, type, name, res_id, term):
+        term = (term or '').strip()
+        if len(term) > 2:
+            push_translation(mod, type, name, res_id, term)
+
+    def get_root_view(xml_id):
+        view = model_data_obj.xmlid_to_object(cr, uid, xml_id)
+        if view:
+            while view.mode != 'primary':
+                view = view.inherit_id
+        xml_id = view.get_external_id(cr, uid).get(view.id, xml_id)
+        return xml_id
+
     for (xml_name,model,res_id,module) in cr.fetchall():
         module = encode(module)
         model = encode(model)
@@ -680,8 +704,13 @@ def trans_generate(lang, modules, cr):
 
         if model=='ir.ui.view':
             d = etree.XML(encode(obj.arch))
-            for t in trans_parse_view(d):
+            if obj.type == 'qweb':
-                push_translation(module, 'view', encode(obj.model), 0, t)
+                view_id = get_root_view(xml_name)
+                push_qweb = lambda t,l: push(module, 'view', 'website', view_id, t)
+                _extract_translatable_qweb_terms(d, push_qweb)
+            else:
+                push_view = lambda t,l: push(module, 'view', obj.model, xml_name, t)
+                trans_parse_view(d, push_view)
         elif model=='ir.actions.wizard':
             pass # TODO Can model really be 'ir.actions.wizard' ?
 
@@ -745,13 +774,16 @@ def trans_generate(lang, modules, cr):
                     _logger.exception("couldn't export translation for report %s %s %s", name, report_type, fname)
 
         for field_name, field_def in obj._columns.items():
+            if model == 'ir.model' and field_name == 'name' and obj.name == obj.model:
+                # ignore model name if it is the technical one, nothing to translate
+                continue
             if field_def.translate:
                 name = model + "," + field_name
                 try:
-                    trad = getattr(obj, field_name) or ''
+                    term = obj[field_name] or ''
                 except:
-                    trad = ''
+                    term = ''
-                push_translation(module, 'model', name, xml_name, encode(trad))
+                push_translation(module, 'model', name, xml_name, encode(term))
 
         # End of data for ir.model.data query results