<commit_before>#!/usr/bin/env python
"""Safe(ish) evaluation of mathematical expression using Python's ast
module.

This module provides an Interpreter class that compiles a restricted set of
Python expressions and statements to Python's AST representation, and then
executes that representation using values held in a symbol table.

The symbol table is a simple dictionary, giving a simple, flat namespace.
This comes pre-loaded with many functions from Python's builtin and math
module.  If numpy is installed, many numpy functions are also included.
Additional symbols can be added when an Interpreter is created, but the
user of that interpreter will not be able to import additional modules.

Expressions, including loops, conditionals, and function definitions can be
compiled into ast node and then evaluated later, using the current values
in the symbol table.

The result is a restricted, simplified version of Python meant for
numerical caclulations that is somewhat safer than 'eval' because many
unsafe operations (such as 'import' and 'eval') are simply not allowed.

Many parts of Python syntax are supported, including:
     for loops, while loops, if-then-elif-else conditionals
     try-except (including 'finally')
     function definitions with def
     advanced slicing:    a[::-1], array[-3:, :, ::2]
     if-expressions:      out = one_thing if TEST else other
     list comprehension   out = [sqrt(i) for i in values]

The following Python syntax elements are not supported:
     Import, Exec, Lambda, Class, Global, Generators,
     Yield, Decorators

In addition, while many builtin functions are supported, several builtin
functions that are considered unsafe are missing ('eval', 'exec', and
'getattr' for example)
"""

from __future__ import division, print_function

import ast
import time
import inspect
from sys import exc_info, stdout, stderr, version_info

from .astutils import (UNSAFE_ATTRS, HAS_NUMPY, make_symbol_table, numpy,
                       op2func, ExceptionHolder, ReturnedNone,
                       valid_symbol_name, check_pyversion)

builtins = __builtins__
if not isinstance(builtins, dict):
    builtins = builtins.__dict__

ALL_NODES = ['arg', 'assert', 'assign', 'attribute', 'augassign', 'binop',
             'boolop', 'break', 'call', 'compare', 'continue', 'delete',
             'dict', 'ellipsis', 'excepthandler', 'expr', 'extslice',
             'for', 'functiondef', 'if', 'ifexp', 'index', 'interrupt',
             'list', 'listcomp', 'module', 'name', 'nameconstant', 'num',
             'pass', 'print', 'raise', 'repr', 'return', 'slice', 'str',
             'subscript', 'try', 'tuple', 'unaryop', 'while', 'constant']


PY3 = check_pyversion()

class Interpreter(object):
    """create an asteval Interpreter: a restricted, simplified interpreter
    of mathematical expressions using Python syntax.

    Parameters
    ----------
    symtable : dict or `None`
        dictionary to use as symbol table (if `None`, one will be created).
    usersyms : dict or `None`
        dictionary of user-defined symbols to add to symbol table.
    writer : file-like or `None`
        callable file-like object where standard output will be sent.
    err_writer : file-like or `None`
        callable file-like object where standard error will be sent.
    use_numpy : bool
        whether to use functions from numpy.
    minimal : bool
        create a minimal interpreter: disable all options (see Note 1).
    no_if : bool
        whether to support `if` blocks
    no_for : bool
        whether to support `for` blocks.
    no_while : bool
        whether to support `while` blocks.
    no_try : bool
        whether to support `try` blocks.
    no_functiondef : bool
        whether to support user-defined functions.
    no_ifexp : bool
        whether to support if expressions.
    no_listcomp : bool
        whether to support list comprehension.
    no_augassign : bool
        whether to support augemented assigments (`a += 1`, etc).
    no_assert : bool
        whether to support `assert`.
    no_delete : bool
        whether to support `del`.
    no_raise : bool
        whether to support `raise`.
    no_print : bool
        whether to support `print`.
    readonly_symbols : iterable or `None`
        symbols that the user can not assign to
    builtins_readonly : bool
        whether to blacklist all symbols that are in the initial symtable

    Notes
    -----
    1. setting `minimal=True` is equivalent to setting all
       `no_***` options to `True`.
    """

    def __init__(self, symtable=None, usersyms=None, writer=None,
                 err_writer=None, use_numpy=True, minimal=False,
                 no_if=False, no_for=False, no_while=False, no_try=False,
                 no_functiondef=False, no_ifexp=False, no_listcomp=False,
                 no_augassign=False, no_assert=False, no_delete=False,
                 no_raise=False, no_print=False, max_time=None,
                 readonly_symbols=None, builtins_readonly=False):

        self.writer = writer or stdout
        self.err_writer = err_writer or stderr

        if symtable is None:
            if usersyms is None:
                usersyms = {}
            symtable = make_symbol_table(use_numpy=use_numpy, **usersyms)

        if no_print:
            symtable['print'] = self._nullprinter
        else:
            symtable['print'] = self._printer

        self.symtable = symtable
        self._interrupt = None
        self.error = []
        self.error_msg = None
        self.expr = None
        self.retval = None
        self.lineno = 0
        self.start_time = time.time()
        self.use_numpy = HAS_NUMPY and use_numpy

        nodes = ALL_NODES[:]

        if minimal or no_if:
            nodes.remove('if')
        if minimal or no_for:
            nodes.remove('for')
        if minimal or no_while:
            nodes.remove('while')
        if minimal or no_try:
            nodes.remove('try')
        if minimal or no_functiondef:
            nodes.remove('functiondef')
        if minimal or no_ifexp:
            nodes.remove('ifexp')
        if minimal or no_assert:
            nodes.remove('assert')
        if minimal or no_delete:
            nodes.remove('delete')
        if minimal or no_raise:
            nodes.remove('raise')
        if minimal or no_print:
            nodes.remove('print')
        if minimal or no_listcomp:
            nodes.remove('listcomp')
        if minimal or no_augassign:
            nodes.remove('augassign')

        self.node_handlers = {}
        for node in nodes:
            self.node_handlers[node] = getattr(self, "on_%s" % node)

        # to rationalize try/except try/finally for Python2.6 through Python3.3
        if 'try' in self.node_handlers:
            self.node_handlers['tryexcept'] = self.node_handlers['try']
            self.node_handlers['tryfinally'] = self.node_handlers['try']

        if readonly_symbols is None:
            self.readonly_symbols = set()
        else:
            self.readonly_symbols = set(readonly_symbols)

        if builtins_readonly:
            self.readonly_symbols |= set(self.symtable)

        self.no_deepcopy = [key for key, val in symtable.items()
                            if (callable(val)
                                or 'numpy.lib.index_tricks' in repr(val)
                                or inspect.ismodule(val))]

    def remove_nodehandler(self, node):
        """remove support for a node
        returns current node handler, so that it
        might be re-added with add_nodehandler()
        """
        out = None
        if node in self.node_handlers:
            out = self.node_handlers.pop(node)
        return out

    def set_nodehandler(self, node, handler):
        """set node handler"""
        self.node_handlers[node] = handler


    def user_defined_symbols(self):
        """Return a set of symbols that have been added to symtable after
        construction.

        I.e., the symbols from self.symtable that are not in
        self.no_deepcopy.

        Returns
        -------
        unique_symbols : set
            symbols in symtable that are not in self.no_deepcopy

        """
        sym_in_current = set(self.symtable.keys())
        sym_from_construction = set(self.no_deepcopy)
        unique_symbols = sym_in_current.difference(sym_from_construction)
        return unique_symbols

    def unimplemented(self, node):
        """Unimplemented nodes."""
        self.raise_exception(node, exc=NotImplementedError,
                             msg="'%s' not supported" %
                             (node.__class__.__name__))

    def raise_exception(self, node, exc=None, msg='', expr=None,
                        lineno=None):
        """Add an exception."""
        if self.error is None:
            self.error = []
        if expr is None:
            expr = self.expr
        if len(self.error) > 0 and not isinstance(node, ast.Module):
            msg = '%s' % msg
        err = ExceptionHolder(node, exc=exc, msg=msg, expr=expr, lineno=lineno)
        self._interrupt = ast.Break()
        self.error.append(err)
        if self.error_msg is None:
            self.error_msg = "at expr='%s'" % (self.expr)
        elif len(msg) > 0:
            self.error_msg = msg
        if exc is None:
            try:
                exc = self.error[0].exc
            except:
                exc = RuntimeError
        raise exc(self.error_msg)

    # main entry point for Ast node evaluation
    #  parse:  text of statements -> ast
    #  run:    ast -> result
    #  eval:   string statement -> result = run(parse(statement))
    def parse(self, text):
        """Parse statement/expression to Ast representation."""
        self.expr = text
        try:
            out = ast.parse(text)
        except SyntaxError:
            self.raise_exception(None, msg='Syntax Error', expr=text)
        except:
            self.raise_exception(None, msg='Runtime Error', expr=text)

        return out

    def run(self, node, expr=None, lineno=None, with_raise=True):
        """Execute parsed Ast representation for an expression."""
        # Note: keep the 'node is None' test: internal code here may run
        #    run(None) and expect a None in return.
        out = None
        if len(self.error) > 0:
            return out
        if node is None:
            return out
        if isinstance(node, str):
            node = self.parse(node)
        if lineno is not None:
            self.lineno = lineno
        if expr is not None:
            self.expr = expr

        # get handler for this node:
        #   on_xxx with handle nodes of type 'xxx', etc
        try:
            handler = self.node_handlers[node.__class__.__name__.lower()]
        except KeyError:
            return self.unimplemented(node)

        # run the handler:  this will likely generate
        # recursive calls into this run method.
        try:
            ret = handler(node)
            if isinstance(ret, enumerate):
                ret = list(ret)
            return ret
        except:
            if with_raise:
                self.raise_exception(node, expr=expr)

    def __call__(self, expr, **kw):
        """Call class instance as function."""
        return self.eval(expr, **kw)

    def eval(self, expr, lineno=0, show_errors=True):
        """Evaluate a single statement."""
        self.lineno = lineno
        self.error = []
        self.start_time = time.time()
        try:
            node = self.parse(expr)<commit_msg>Raise immediately on the first error<commit_after>#!/usr/bin/env python
"""Safe(ish) evaluation of mathematical expression using Python's ast
module.

This module provides an Interpreter class that compiles a restricted set of
Python expressions and statements to Python's AST representation, and then
executes that representation using values held in a symbol table.

The symbol table is a simple dictionary, giving a simple, flat namespace.
This comes pre-loaded with many functions from Python's builtin and math
module.  If numpy is installed, many numpy functions are also included.
Additional symbols can be added when an Interpreter is created, but the
user of that interpreter will not be able to import additional modules.

Expressions, including loops, conditionals, and function definitions can be
compiled into ast node and then evaluated later, using the current values
in the symbol table.

The result is a restricted, simplified version of Python meant for
numerical caclulations that is somewhat safer than 'eval' because many
unsafe operations (such as 'import' and 'eval') are simply not allowed.

Many parts of Python syntax are supported, including:
     for loops, while loops, if-then-elif-else conditionals
     try-except (including 'finally')
     function definitions with def
     advanced slicing:    a[::-1], array[-3:, :, ::2]
     if-expressions:      out = one_thing if TEST else other
     list comprehension   out = [sqrt(i) for i in values]

The following Python syntax elements are not supported:
     Import, Exec, Lambda, Class, Global, Generators,
     Yield, Decorators

In addition, while many builtin functions are supported, several builtin
functions that are considered unsafe are missing ('eval', 'exec', and
'getattr' for example)
"""

from __future__ import division, print_function

import ast
import time
import inspect
from sys import exc_info, stdout, stderr, version_info

from .astutils import (UNSAFE_ATTRS, HAS_NUMPY, make_symbol_table, numpy,
                       op2func, ExceptionHolder, ReturnedNone,
                       valid_symbol_name, check_pyversion)

builtins = __builtins__
if not isinstance(builtins, dict):
    builtins = builtins.__dict__

ALL_NODES = ['arg', 'assert', 'assign', 'attribute', 'augassign', 'binop',
             'boolop', 'break', 'call', 'compare', 'continue', 'delete',
             'dict', 'ellipsis', 'excepthandler', 'expr', 'extslice',
             'for', 'functiondef', 'if', 'ifexp', 'index', 'interrupt',
             'list', 'listcomp', 'module', 'name', 'nameconstant', 'num',
             'pass', 'print', 'raise', 'repr', 'return', 'slice', 'str',
             'subscript', 'try', 'tuple', 'unaryop', 'while', 'constant']


PY3 = check_pyversion()

class Interpreter(object):
    """create an asteval Interpreter: a restricted, simplified interpreter
    of mathematical expressions using Python syntax.

    Parameters
    ----------
    symtable : dict or `None`
        dictionary to use as symbol table (if `None`, one will be created).
    usersyms : dict or `None`
        dictionary of user-defined symbols to add to symbol table.
    writer : file-like or `None`
        callable file-like object where standard output will be sent.
    err_writer : file-like or `None`
        callable file-like object where standard error will be sent.
    use_numpy : bool
        whether to use functions from numpy.
    minimal : bool
        create a minimal interpreter: disable all options (see Note 1).
    no_if : bool
        whether to support `if` blocks
    no_for : bool
        whether to support `for` blocks.
    no_while : bool
        whether to support `while` blocks.
    no_try : bool
        whether to support `try` blocks.
    no_functiondef : bool
        whether to support user-defined functions.
    no_ifexp : bool
        whether to support if expressions.
    no_listcomp : bool
        whether to support list comprehension.
    no_augassign : bool
        whether to support augemented assigments (`a += 1`, etc).
    no_assert : bool
        whether to support `assert`.
    no_delete : bool
        whether to support `del`.
    no_raise : bool
        whether to support `raise`.
    no_print : bool
        whether to support `print`.
    readonly_symbols : iterable or `None`
        symbols that the user can not assign to
    builtins_readonly : bool
        whether to blacklist all symbols that are in the initial symtable

    Notes
    -----
    1. setting `minimal=True` is equivalent to setting all
       `no_***` options to `True`.
    """

    def __init__(self, symtable=None, usersyms=None, writer=None,
                 err_writer=None, use_numpy=True, minimal=False,
                 no_if=False, no_for=False, no_while=False, no_try=False,
                 no_functiondef=False, no_ifexp=False, no_listcomp=False,
                 no_augassign=False, no_assert=False, no_delete=False,
                 no_raise=False, no_print=False, max_time=None,
                 readonly_symbols=None, builtins_readonly=False):

        self.writer = writer or stdout
        self.err_writer = err_writer or stderr

        if symtable is None:
            if usersyms is None:
                usersyms = {}
            symtable = make_symbol_table(use_numpy=use_numpy, **usersyms)

        if no_print:
            symtable['print'] = self._nullprinter
        else:
            symtable['print'] = self._printer

        self.symtable = symtable
        self._interrupt = None
        self.error = []
        self.error_msg = None
        self.expr = None
        self.retval = None
        self.lineno = 0
        self.start_time = time.time()
        self.use_numpy = HAS_NUMPY and use_numpy

        nodes = ALL_NODES[:]

        if minimal or no_if:
            nodes.remove('if')
        if minimal or no_for:
            nodes.remove('for')
        if minimal or no_while:
            nodes.remove('while')
        if minimal or no_try:
            nodes.remove('try')
        if minimal or no_functiondef:
            nodes.remove('functiondef')
        if minimal or no_ifexp:
            nodes.remove('ifexp')
        if minimal or no_assert:
            nodes.remove('assert')
        if minimal or no_delete:
            nodes.remove('delete')
        if minimal or no_raise:
            nodes.remove('raise')
        if minimal or no_print:
            nodes.remove('print')
        if minimal or no_listcomp:
            nodes.remove('listcomp')
        if minimal or no_augassign:
            nodes.remove('augassign')

        self.node_handlers = {}
        for node in nodes:
            self.node_handlers[node] = getattr(self, "on_%s" % node)

        # to rationalize try/except try/finally for Python2.6 through Python3.3
        if 'try' in self.node_handlers:
            self.node_handlers['tryexcept'] = self.node_handlers['try']
            self.node_handlers['tryfinally'] = self.node_handlers['try']

        if readonly_symbols is None:
            self.readonly_symbols = set()
        else:
            self.readonly_symbols = set(readonly_symbols)

        if builtins_readonly:
            self.readonly_symbols |= set(self.symtable)

        self.no_deepcopy = [key for key, val in symtable.items()
                            if (callable(val)
                                or 'numpy.lib.index_tricks' in repr(val)
                                or inspect.ismodule(val))]

    def remove_nodehandler(self, node):
        """remove support for a node
        returns current node handler, so that it
        might be re-added with add_nodehandler()
        """
        out = None
        if node in self.node_handlers:
            out = self.node_handlers.pop(node)
        return out

    def set_nodehandler(self, node, handler):
        """set node handler"""
        self.node_handlers[node] = handler


    def user_defined_symbols(self):
        """Return a set of symbols that have been added to symtable after
        construction.

        I.e., the symbols from self.symtable that are not in
        self.no_deepcopy.

        Returns
        -------
        unique_symbols : set
            symbols in symtable that are not in self.no_deepcopy

        """
        sym_in_current = set(self.symtable.keys())
        sym_from_construction = set(self.no_deepcopy)
        unique_symbols = sym_in_current.difference(sym_from_construction)
        return unique_symbols

    def unimplemented(self, node):
        """Unimplemented nodes."""
        self.raise_exception(node, exc=NotImplementedError,
                             msg="'%s' not supported" %
                             (node.__class__.__name__))

    def raise_exception(self, node, exc=None, msg='', expr=None,
                        lineno=None):
        """Add an exception."""
        if self.error is None:
            self.error = []
        if expr is None:
            expr = self.expr
        if len(self.error) > 0 and not isinstance(node, ast.Module):
            msg = '%s' % msg
        err = ExceptionHolder(node, exc=exc, msg=msg, expr=expr, lineno=lineno)
        self._interrupt = ast.Break()
        self.error.append(err)
        if self.error_msg is None:
            self.error_msg = "at expr='%s'" % (self.expr)
        elif len(msg) > 0:
            self.error_msg = msg
        if exc is None:
            try:
                exc = self.error[0].exc
            except:
                exc = RuntimeError
        raise exc(self.error_msg)

    # main entry point for Ast node evaluation
    #  parse:  text of statements -> ast
    #  run:    ast -> result
    #  eval:   string statement -> result = run(parse(statement))
    def parse(self, text):
        """Parse statement/expression to Ast representation."""
        self.expr = text
        try:
            out = ast.parse(text)
        except SyntaxError:
            self.raise_exception(None, msg='Syntax Error', expr=text)
        except:
            self.raise_exception(None, msg='Runtime Error', expr=text)

        return out

    def run(self, node, expr=None, lineno=None, with_raise=True):
        """Execute parsed Ast representation for an expression."""
        # Note: keep the 'node is None' test: internal code here may run
        #    run(None) and expect a None in return.
        out = None
        if len(self.error) > 0:
            return out
        if node is None:
            return out
        if isinstance(node, str):
            node = self.parse(node)
        if lineno is not None:
            self.lineno = lineno
        if expr is not None:
            self.expr = expr

        # get handler for this node:
        #   on_xxx with handle nodes of type 'xxx', etc
        try:
            handler = self.node_handlers[node.__class__.__name__.lower()]
        except KeyError:
            return self.unimplemented(node)

        # run the handler:  this will likely generate
        # recursive calls into this run method.
        try:
            ret = handler(node)
            if isinstance(ret, enumerate):
                ret = list(ret)
            return ret
        except:
            if with_raise:
                if len(self.error) == 0:
                    # Unhandled exception that didn't go through raise_exception
                    self.raise_exception(node, expr=expr)
                raise

    def __call__(self, expr, **kw):
        """Call class instance as function."""
        return self.eval(expr, **kw)

    def eval(self, expr, lineno=0, show_errors=True):
        """Evaluate a single statement."""
        self.lineno = lineno
        self.error = []
        self.start_time = time.time()
        try:
            node = self.parse(expr)
{'fraction_non_alphanumeric': 0.08913481748380732, 'fraction_numerical': 0.0015870973276712564, 'mean_word_length': 2.1745642701525054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '305137', 'n_tokens_mistral': 6438, 'n_tokens_neox': 6273, 'n_words': 2143}