373 lines
15 KiB
Python
Executable File
373 lines
15 KiB
Python
Executable File
# -*- coding: utf-8 -*-
|
|
# ast.py
|
|
# Copyright (C) Mako developers
|
|
#
|
|
# This module is part of Mako and is released under
|
|
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
|
|
|
"""Handles parsing of Python code.
|
|
|
|
Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
|
|
module is used.
|
|
"""
|
|
|
|
from StringIO import StringIO
|
|
from mako import exceptions, util
|
|
|
|
# words that cannot be assigned to (notably smaller than the total keys in __builtins__)
|
|
reserved = util.Set(['True', 'False', 'None'])
|
|
|
|
try:
|
|
import _ast
|
|
util.restore__ast(_ast)
|
|
import _ast_util
|
|
except ImportError:
|
|
_ast = None
|
|
from compiler import parse as compiler_parse
|
|
from compiler import visitor
|
|
|
|
|
|
def parse(code, mode='exec', **exception_kwargs):
|
|
"""Parse an expression into AST"""
|
|
try:
|
|
if _ast:
|
|
return _ast_util.parse(code, '<unknown>', mode)
|
|
else:
|
|
return compiler_parse(code, mode)
|
|
except Exception, e:
|
|
raise exceptions.SyntaxException("(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), **exception_kwargs)
|
|
|
|
|
|
if _ast:
|
|
class FindIdentifiers(_ast_util.NodeVisitor):
|
|
def __init__(self, listener, **exception_kwargs):
|
|
self.in_function = False
|
|
self.in_assign_targets = False
|
|
self.local_ident_stack = {}
|
|
self.listener = listener
|
|
self.exception_kwargs = exception_kwargs
|
|
def _add_declared(self, name):
|
|
if not self.in_function:
|
|
self.listener.declared_identifiers.add(name)
|
|
def visit_ClassDef(self, node):
|
|
self._add_declared(node.name)
|
|
def visit_Assign(self, node):
|
|
# flip around the visiting of Assign so the expression gets evaluated first,
|
|
# in the case of a clause like "x=x+5" (x is undeclared)
|
|
self.visit(node.value)
|
|
in_a = self.in_assign_targets
|
|
self.in_assign_targets = True
|
|
for n in node.targets:
|
|
self.visit(n)
|
|
self.in_assign_targets = in_a
|
|
def visit_FunctionDef(self, node):
|
|
self._add_declared(node.name)
|
|
# push function state onto stack. dont log any
|
|
# more identifiers as "declared" until outside of the function,
|
|
# but keep logging identifiers as "undeclared".
|
|
# track argument names in each function header so they arent counted as "undeclared"
|
|
saved = {}
|
|
inf = self.in_function
|
|
self.in_function = True
|
|
for arg in node.args.args:
|
|
if arg.id in self.local_ident_stack:
|
|
saved[arg.id] = True
|
|
else:
|
|
self.local_ident_stack[arg.id] = True
|
|
for n in node.body:
|
|
self.visit(n)
|
|
self.in_function = inf
|
|
for arg in node.args.args:
|
|
if arg.id not in saved:
|
|
del self.local_ident_stack[arg.id]
|
|
def visit_For(self, node):
|
|
# flip around visit
|
|
self.visit(node.iter)
|
|
self.visit(node.target)
|
|
for statement in node.body:
|
|
self.visit(statement)
|
|
for statement in node.orelse:
|
|
self.visit(statement)
|
|
def visit_Name(self, node):
|
|
if isinstance(node.ctx, _ast.Store):
|
|
self._add_declared(node.id)
|
|
if node.id not in reserved and node.id not in self.listener.declared_identifiers and node.id not in self.local_ident_stack:
|
|
self.listener.undeclared_identifiers.add(node.id)
|
|
def visit_Import(self, node):
|
|
for name in node.names:
|
|
if name.asname is not None:
|
|
self._add_declared(name.asname)
|
|
else:
|
|
self._add_declared(name.name.split('.')[0])
|
|
def visit_ImportFrom(self, node):
|
|
for name in node.names:
|
|
if name.asname is not None:
|
|
self._add_declared(name.asname)
|
|
else:
|
|
if name.name == '*':
|
|
raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared. Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", **self.exception_kwargs)
|
|
self._add_declared(name.name)
|
|
|
|
class FindTuple(_ast_util.NodeVisitor):
|
|
def __init__(self, listener, code_factory, **exception_kwargs):
|
|
self.listener = listener
|
|
self.exception_kwargs = exception_kwargs
|
|
self.code_factory = code_factory
|
|
def visit_Tuple(self, node):
|
|
for n in node.elts:
|
|
p = self.code_factory(n, **self.exception_kwargs)
|
|
self.listener.codeargs.append(p)
|
|
self.listener.args.append(ExpressionGenerator(n).value())
|
|
self.listener.declared_identifiers = self.listener.declared_identifiers.union(p.declared_identifiers)
|
|
self.listener.undeclared_identifiers = self.listener.undeclared_identifiers.union(p.undeclared_identifiers)
|
|
|
|
class ParseFunc(_ast_util.NodeVisitor):
|
|
def __init__(self, listener, **exception_kwargs):
|
|
self.listener = listener
|
|
self.exception_kwargs = exception_kwargs
|
|
def visit_FunctionDef(self, node):
|
|
self.listener.funcname = node.name
|
|
argnames = [arg.id for arg in node.args.args]
|
|
if node.args.vararg:
|
|
argnames.append(node.args.vararg)
|
|
if node.args.kwarg:
|
|
argnames.append(node.args.kwarg)
|
|
self.listener.argnames = argnames
|
|
self.listener.defaults = node.args.defaults # ast
|
|
self.listener.varargs = node.args.vararg
|
|
self.listener.kwargs = node.args.kwarg
|
|
|
|
class ExpressionGenerator(object):
|
|
def __init__(self, astnode):
|
|
self.generator = _ast_util.SourceGenerator(' ' * 4)
|
|
self.generator.visit(astnode)
|
|
def value(self):
|
|
return ''.join(self.generator.result)
|
|
else:
|
|
class FindIdentifiers(object):
|
|
def __init__(self, listener, **exception_kwargs):
|
|
self.in_function = False
|
|
self.local_ident_stack = {}
|
|
self.listener = listener
|
|
self.exception_kwargs = exception_kwargs
|
|
def _add_declared(self, name):
|
|
if not self.in_function:
|
|
self.listener.declared_identifiers.add(name)
|
|
def visitClass(self, node, *args):
|
|
self._add_declared(node.name)
|
|
def visitAssName(self, node, *args):
|
|
self._add_declared(node.name)
|
|
def visitAssign(self, node, *args):
|
|
# flip around the visiting of Assign so the expression gets evaluated first,
|
|
# in the case of a clause like "x=x+5" (x is undeclared)
|
|
self.visit(node.expr, *args)
|
|
for n in node.nodes:
|
|
self.visit(n, *args)
|
|
def visitFunction(self,node, *args):
|
|
self._add_declared(node.name)
|
|
# push function state onto stack. dont log any
|
|
# more identifiers as "declared" until outside of the function,
|
|
# but keep logging identifiers as "undeclared".
|
|
# track argument names in each function header so they arent counted as "undeclared"
|
|
saved = {}
|
|
inf = self.in_function
|
|
self.in_function = True
|
|
for arg in node.argnames:
|
|
if arg in self.local_ident_stack:
|
|
saved[arg] = True
|
|
else:
|
|
self.local_ident_stack[arg] = True
|
|
for n in node.getChildNodes():
|
|
self.visit(n, *args)
|
|
self.in_function = inf
|
|
for arg in node.argnames:
|
|
if arg not in saved:
|
|
del self.local_ident_stack[arg]
|
|
def visitFor(self, node, *args):
|
|
# flip around visit
|
|
self.visit(node.list, *args)
|
|
self.visit(node.assign, *args)
|
|
self.visit(node.body, *args)
|
|
def visitName(self, node, *args):
|
|
if node.name not in reserved and node.name not in self.listener.declared_identifiers and node.name not in self.local_ident_stack:
|
|
self.listener.undeclared_identifiers.add(node.name)
|
|
def visitImport(self, node, *args):
|
|
for (mod, alias) in node.names:
|
|
if alias is not None:
|
|
self._add_declared(alias)
|
|
else:
|
|
self._add_declared(mod.split('.')[0])
|
|
def visitFrom(self, node, *args):
|
|
for (mod, alias) in node.names:
|
|
if alias is not None:
|
|
self._add_declared(alias)
|
|
else:
|
|
if mod == '*':
|
|
raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared. Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", **self.exception_kwargs)
|
|
self._add_declared(mod)
|
|
def visit(self, expr):
|
|
visitor.walk(expr, self) #, walker=walker())
|
|
|
|
class FindTuple(object):
|
|
def __init__(self, listener, code_factory, **exception_kwargs):
|
|
self.listener = listener
|
|
self.exception_kwargs = exception_kwargs
|
|
self.code_factory = code_factory
|
|
def visitTuple(self, node, *args):
|
|
for n in node.nodes:
|
|
p = self.code_factory(n, **self.exception_kwargs)
|
|
self.listener.codeargs.append(p)
|
|
self.listener.args.append(ExpressionGenerator(n).value())
|
|
self.listener.declared_identifiers = self.listener.declared_identifiers.union(p.declared_identifiers)
|
|
self.listener.undeclared_identifiers = self.listener.undeclared_identifiers.union(p.undeclared_identifiers)
|
|
def visit(self, expr):
|
|
visitor.walk(expr, self) #, walker=walker())
|
|
|
|
class ParseFunc(object):
|
|
def __init__(self, listener, **exception_kwargs):
|
|
self.listener = listener
|
|
self.exception_kwargs = exception_kwargs
|
|
def visitFunction(self, node, *args):
|
|
self.listener.funcname = node.name
|
|
self.listener.argnames = node.argnames
|
|
self.listener.defaults = node.defaults
|
|
self.listener.varargs = node.varargs
|
|
self.listener.kwargs = node.kwargs
|
|
def visit(self, expr):
|
|
visitor.walk(expr, self)
|
|
|
|
class ExpressionGenerator(object):
|
|
"""given an AST node, generates an equivalent literal Python expression."""
|
|
def __init__(self, astnode):
|
|
self.buf = StringIO()
|
|
visitor.walk(astnode, self) #, walker=walker())
|
|
def value(self):
|
|
return self.buf.getvalue()
|
|
def operator(self, op, node, *args):
|
|
self.buf.write("(")
|
|
self.visit(node.left, *args)
|
|
self.buf.write(" %s " % op)
|
|
self.visit(node.right, *args)
|
|
self.buf.write(")")
|
|
def booleanop(self, op, node, *args):
|
|
self.visit(node.nodes[0])
|
|
for n in node.nodes[1:]:
|
|
self.buf.write(" " + op + " ")
|
|
self.visit(n, *args)
|
|
def visitConst(self, node, *args):
|
|
self.buf.write(repr(node.value))
|
|
def visitAssName(self, node, *args):
|
|
# TODO: figure out OP_ASSIGN, other OP_s
|
|
self.buf.write(node.name)
|
|
def visitName(self, node, *args):
|
|
self.buf.write(node.name)
|
|
def visitMul(self, node, *args):
|
|
self.operator("*", node, *args)
|
|
def visitAnd(self, node, *args):
|
|
self.booleanop("and", node, *args)
|
|
def visitOr(self, node, *args):
|
|
self.booleanop("or", node, *args)
|
|
def visitBitand(self, node, *args):
|
|
self.booleanop("&", node, *args)
|
|
def visitBitor(self, node, *args):
|
|
self.booleanop("|", node, *args)
|
|
def visitBitxor(self, node, *args):
|
|
self.booleanop("^", node, *args)
|
|
def visitAdd(self, node, *args):
|
|
self.operator("+", node, *args)
|
|
def visitGetattr(self, node, *args):
|
|
self.visit(node.expr, *args)
|
|
self.buf.write(".%s" % node.attrname)
|
|
def visitSub(self, node, *args):
|
|
self.operator("-", node, *args)
|
|
def visitNot(self, node, *args):
|
|
self.buf.write("not ")
|
|
self.visit(node.expr)
|
|
def visitDiv(self, node, *args):
|
|
self.operator("/", node, *args)
|
|
def visitFloorDiv(self, node, *args):
|
|
self.operator("//", node, *args)
|
|
def visitSubscript(self, node, *args):
|
|
self.visit(node.expr)
|
|
self.buf.write("[")
|
|
[self.visit(x) for x in node.subs]
|
|
self.buf.write("]")
|
|
def visitUnarySub(self, node, *args):
|
|
self.buf.write("-")
|
|
self.visit(node.expr)
|
|
def visitUnaryAdd(self, node, *args):
|
|
self.buf.write("-")
|
|
self.visit(node.expr)
|
|
def visitSlice(self, node, *args):
|
|
self.visit(node.expr)
|
|
self.buf.write("[")
|
|
if node.lower is not None:
|
|
self.visit(node.lower)
|
|
self.buf.write(":")
|
|
if node.upper is not None:
|
|
self.visit(node.upper)
|
|
self.buf.write("]")
|
|
def visitDict(self, node):
|
|
self.buf.write("{")
|
|
c = node.getChildren()
|
|
for i in range(0, len(c), 2):
|
|
self.visit(c[i])
|
|
self.buf.write(": ")
|
|
self.visit(c[i+1])
|
|
if i<len(c) -2:
|
|
self.buf.write(", ")
|
|
self.buf.write("}")
|
|
def visitTuple(self, node):
|
|
self.buf.write("(")
|
|
c = node.getChildren()
|
|
for i in range(0, len(c)):
|
|
self.visit(c[i])
|
|
if i<len(c) - 1:
|
|
self.buf.write(", ")
|
|
self.buf.write(")")
|
|
def visitList(self, node):
|
|
self.buf.write("[")
|
|
c = node.getChildren()
|
|
for i in range(0, len(c)):
|
|
self.visit(c[i])
|
|
if i<len(c) - 1:
|
|
self.buf.write(", ")
|
|
self.buf.write("]")
|
|
def visitListComp(self, node):
|
|
self.buf.write("[")
|
|
self.visit(node.expr)
|
|
self.buf.write(" ")
|
|
for n in node.quals:
|
|
self.visit(n)
|
|
self.buf.write("]")
|
|
def visitListCompFor(self, node):
|
|
self.buf.write(" for ")
|
|
self.visit(node.assign)
|
|
self.buf.write(" in ")
|
|
self.visit(node.list)
|
|
for n in node.ifs:
|
|
self.visit(n)
|
|
def visitListCompIf(self, node):
|
|
self.buf.write(" if ")
|
|
self.visit(node.test)
|
|
def visitCompare(self, node):
|
|
self.visit(node.expr)
|
|
for tup in node.ops:
|
|
self.buf.write(tup[0])
|
|
self.visit(tup[1])
|
|
def visitCallFunc(self, node, *args):
|
|
self.visit(node.node)
|
|
self.buf.write("(")
|
|
if len(node.args):
|
|
self.visit(node.args[0])
|
|
for a in node.args[1:]:
|
|
self.buf.write(", ")
|
|
self.visit(a)
|
|
self.buf.write(")")
|
|
|
|
class walker(visitor.ASTVisitor):
|
|
def dispatch(self, node, *args):
|
|
print "Node:", str(node)
|
|
#print "dir:", dir(node)
|
|
return visitor.ASTVisitor.dispatch(self, node, *args)
|