<commit_before>"""
This parser will parse the given input and build an expression tree. Grammar
file for the supported mathematical expressions.
"""

import os.path
PYBISON_BUILD = os.path.realpath('build/external/pybison')
EXTERNAL_MODS = os.path.realpath('external')

import sys
sys.path.insert(0, PYBISON_BUILD)
sys.path.insert(1, EXTERNAL_MODS)

from pybison import BisonParser, BisonSyntaxError
from graph_drawing.graph import generate_graph

from node import ExpressionNode as Node, ExpressionLeaf as Leaf, OP_MAP, \
        TOKEN_MAP, TYPE_OPERATOR, OP_COMMA, OP_NEG, OP_MUL, Scope
from rules import RULES
from possibilities import filter_duplicates, pick_suggestion, apply_suggestion

import Queue


# Check for n-ary operator in child nodes
def combine(op, op_type, *nodes):
    # At least return the operator.
    res = [op]

    for n in nodes:
        # Merge the children for all nodes which have the same operator.
        if n.type == TYPE_OPERATOR and n.op == op_type:
            res += n.nodes
        else:
            res.append(n)

    return res


class Parser(BisonParser):
    """
    Implements the calculator parser. Grammar rules are defined in the method
    docstrings. Scanner rules are in the 'lexscript' attribute.
    """

    # Words to be ignored by preprocessor
    words = zip(*filter(lambda (s, op): TOKEN_MAP[op] == 'FUNCTION', \
                        OP_MAP.iteritems()))[0] + ('raise', 'graph')

    # Output directory of generated pybison files, including a trailing slash.
    buildDirectory = PYBISON_BUILD + '/'

    # ----------------------------------------------------------------
    # lexer tokens - these must match those in your lex script (below)
    # ----------------------------------------------------------------
    # TODO: add a runtime check to verify that this token list match the list
    # of tokens of the lex script.
    tokens = ['NUMBER', 'IDENTIFIER', 'NEWLINE', 'QUIT', 'RAISE', 'GRAPH',
              'LPAREN', 'RPAREN', 'FUNCTION'] \
             + filter(lambda t: t != 'FUNCTION', TOKEN_MAP.values())

    # ------------------------------
    # precedences
    # ------------------------------
    precedences = (
        ('left', ('COMMA', )),
        ('left', ('MINUS', 'PLUS')),
        ('left', ('TIMES', 'DIVIDE')),
        ('left', ('EQ', )),
        ('left', ('NEG', )),
        ('right', ('POW', )),
        ('right', ('FUNCTION', )),
        #('right', ('SIN', 'COS', 'TAN', 'SOLVE', 'INT', 'SQRT')),
        )

    interactive = 0

    def __init__(self, **kwargs):
        BisonParser.__init__(self, **kwargs)
        self.interactive = kwargs.get('interactive', 0)
        self.timeout = kwargs.get('timeout', 0)

        self.reset()

    def reset(self):
        self.read_buffer = ''
        self.read_queue = Queue.Queue()

        #self.subtree_map = {}
        self.root_node = None
        self.possibilities = self.last_possibilities = []

    def run(self, *args, **kwargs):
        self.reset()
        return super(Parser, self).run(*args, **kwargs)

    # Override default read method with a version that prompts for input.
    def read(self, nbytes):
        if self.file == sys.stdin and self.file.closed:
            return ''

        if not self.read_buffer and not self.read_queue.empty():
            self.read_buffer = self.read_queue.get_nowait() + '\n'

        if self.read_buffer:
            read_buffer = self.read_buffer[:nbytes]

            self.read_buffer = self.read_buffer[nbytes:]
            return read_buffer

        try:
            read_buffer = raw_input('>>> ' if self.interactive else '') + '\n'
        except EOFError:
            return ''

        self.read_buffer = read_buffer[nbytes:]
        return read_buffer[:nbytes]

    def hook_read_before(self):
        if self.possibilities:
            if self.verbose:  # pragma: nocover
                print 'possibilities:'

            items = filter_duplicates(self.possibilities)
            self.last_possibilities = self.possibilities

            if self.verbose:  # pragma: nocover
                print '  ' + '\n  '.join(map(str, items))

    def hook_read_after(self, data):
        """
        This hook will be called when the read() method returned. The data
        argument points to the data read by the read() method. This hook
        function should return the data to be used by the parser.
        """
        if not data.strip():
            return data

        self.possibilities = []

        import re

        # TODO: remove this quick preprocessing hack. This hack enables
        # concatenated expressions, since the grammar currently does not
        # support those. This workaround will replace:
        #   - ")(" with ")*(".
        #   - "a(" with "a*(".
        #   - ")a" with ")*a".
        #   - "ab" with "a*b".
        #   - "4a" with "4*a".
        #   - "a4" with "a^4".

        pattern = ('(?:(\))\s*(\()'       # match: )(  result: ) * (
                + '|([a-z0-9])\s*(\()'    # match: a(  result: a * (
                + '|(\))\s*([a-z0-9])'    # match: )a  result: ) * a
                + '|([a-z])\s*([a-z]+)'   # match: ab  result: a * b
                + '|([0-9])\s*([a-z])'    # match: 4a  result: 4 * a
                + '|([a-z])\s*([0-9])'    # match: a4  result: a ^ 4
                + '|([0-9])\s+([0-9]))')  # match: 4 4 result: 4 * 4

        def preprocess_data(match):
            left, right = filter(None, match.groups())

            # Filter words (otherwise they will be preprocessed as well)
            if left + right in Parser.words:
                return left + right

            # If all characters on the right are numbers. e.g. "a4", the
            # expression implies exponentiation. Make sure ")4" is not
            # converted into an exponentiation, because that's multiplication.
            if left != ')' and not 48 <= ord(left) < 58 \
                    and all(map(lambda x: 48 <= ord(x) < 58, list(right))):
                return '%s^%s' % (left, right)

            # match: ab | abc | abcd (where left = "a")
            return '*'.join([left] + list(right))

        # Iteratively replace all matches.
        while True:
            data_after = re.sub(pattern, preprocess_data, data)

            if data == data_after:
                break

            if self.verbose:  # pragma: nocover
                print 'hook_read_after() modified the input data:'
                print 'before:', data.replace('\n', '\\n')
                print 'after :', data_after.replace('\n', '\\n')

            data = data_after

        return data

    def hook_handler(self, target, option, names, values, retval):
        if target in ['exp', 'line', 'input'] or not retval:
            return retval
<commit_msg>Print modifications done by hook_read_after() only once.<commit_after>"""
This parser will parse the given input and build an expression tree. Grammar
file for the supported mathematical expressions.
"""

import os.path
PYBISON_BUILD = os.path.realpath('build/external/pybison')
EXTERNAL_MODS = os.path.realpath('external')

import sys
sys.path.insert(0, PYBISON_BUILD)
sys.path.insert(1, EXTERNAL_MODS)

from pybison import BisonParser, BisonSyntaxError
from graph_drawing.graph import generate_graph

from node import ExpressionNode as Node, ExpressionLeaf as Leaf, OP_MAP, \
        TOKEN_MAP, TYPE_OPERATOR, OP_COMMA, OP_NEG, OP_MUL, Scope
from rules import RULES
from possibilities import filter_duplicates, pick_suggestion, apply_suggestion

import Queue


# Check for n-ary operator in child nodes
def combine(op, op_type, *nodes):
    # At least return the operator.
    res = [op]

    for n in nodes:
        # Merge the children for all nodes which have the same operator.
        if n.type == TYPE_OPERATOR and n.op == op_type:
            res += n.nodes
        else:
            res.append(n)

    return res


class Parser(BisonParser):
    """
    Implements the calculator parser. Grammar rules are defined in the method
    docstrings. Scanner rules are in the 'lexscript' attribute.
    """

    # Words to be ignored by preprocessor
    words = zip(*filter(lambda (s, op): TOKEN_MAP[op] == 'FUNCTION', \
                        OP_MAP.iteritems()))[0] + ('raise', 'graph')

    # Output directory of generated pybison files, including a trailing slash.
    buildDirectory = PYBISON_BUILD + '/'

    # ----------------------------------------------------------------
    # lexer tokens - these must match those in your lex script (below)
    # ----------------------------------------------------------------
    # TODO: add a runtime check to verify that this token list match the list
    # of tokens of the lex script.
    tokens = ['NUMBER', 'IDENTIFIER', 'NEWLINE', 'QUIT', 'RAISE', 'GRAPH',
              'LPAREN', 'RPAREN', 'FUNCTION'] \
             + filter(lambda t: t != 'FUNCTION', TOKEN_MAP.values())

    # ------------------------------
    # precedences
    # ------------------------------
    precedences = (
        ('left', ('COMMA', )),
        ('left', ('MINUS', 'PLUS')),
        ('left', ('TIMES', 'DIVIDE')),
        ('left', ('EQ', )),
        ('left', ('NEG', )),
        ('right', ('POW', )),
        ('right', ('FUNCTION', )),
        #('right', ('SIN', 'COS', 'TAN', 'SOLVE', 'INT', 'SQRT')),
        )

    interactive = 0

    def __init__(self, **kwargs):
        BisonParser.__init__(self, **kwargs)
        self.interactive = kwargs.get('interactive', 0)
        self.timeout = kwargs.get('timeout', 0)

        self.reset()

    def reset(self):
        self.read_buffer = ''
        self.read_queue = Queue.Queue()

        #self.subtree_map = {}
        self.root_node = None
        self.possibilities = self.last_possibilities = []

    def run(self, *args, **kwargs):
        self.reset()
        return super(Parser, self).run(*args, **kwargs)

    # Override default read method with a version that prompts for input.
    def read(self, nbytes):
        if self.file == sys.stdin and self.file.closed:
            return ''

        if not self.read_buffer and not self.read_queue.empty():
            self.read_buffer = self.read_queue.get_nowait() + '\n'

        if self.read_buffer:
            read_buffer = self.read_buffer[:nbytes]

            self.read_buffer = self.read_buffer[nbytes:]
            return read_buffer

        try:
            read_buffer = raw_input('>>> ' if self.interactive else '') + '\n'
        except EOFError:
            return ''

        self.read_buffer = read_buffer[nbytes:]
        return read_buffer[:nbytes]

    def hook_read_before(self):
        if self.possibilities:
            if self.verbose:  # pragma: nocover
                print 'possibilities:'

            items = filter_duplicates(self.possibilities)
            self.last_possibilities = self.possibilities

            if self.verbose:  # pragma: nocover
                print '  ' + '\n  '.join(map(str, items))

    def hook_read_after(self, data):
        """
        This hook will be called when the read() method returned. The data
        argument points to the data read by the read() method. This hook
        function should return the data to be used by the parser.
        """
        if not data.strip():
            return data

        self.possibilities = []

        import re

        # TODO: remove this quick preprocessing hack. This hack enables
        # concatenated expressions, since the grammar currently does not
        # support those. This workaround will replace:
        #   - ")(" with ")*(".
        #   - "a(" with "a*(".
        #   - ")a" with ")*a".
        #   - "ab" with "a*b".
        #   - "4a" with "4*a".
        #   - "a4" with "a^4".

        pattern = ('(?:(\))\s*(\()'       # match: )(  result: ) * (
                + '|([a-z0-9])\s*(\()'    # match: a(  result: a * (
                + '|(\))\s*([a-z0-9])'    # match: )a  result: ) * a
                + '|([a-z])\s*([a-z]+)'   # match: ab  result: a * b
                + '|([0-9])\s*([a-z])'    # match: 4a  result: 4 * a
                + '|([a-z])\s*([0-9])'    # match: a4  result: a ^ 4
                + '|([0-9])\s+([0-9]))')  # match: 4 4 result: 4 * 4

        def preprocess_data(match):
            left, right = filter(None, match.groups())

            # Filter words (otherwise they will be preprocessed as well)
            if left + right in Parser.words:
                return left + right

            # If all characters on the right are numbers. e.g. "a4", the
            # expression implies exponentiation. Make sure ")4" is not
            # converted into an exponentiation, because that's multiplication.
            if left != ')' and not 48 <= ord(left) < 58 \
                    and all(map(lambda x: 48 <= ord(x) < 58, list(right))):
                return '%s^%s' % (left, right)

            # match: ab | abc | abcd (where left = "a")
            return '*'.join([left] + list(right))

        if self.verbose:  # pragma: nocover
            data_before = data

        # Iteratively replace all matches.
        while True:
            data_after = re.sub(pattern, preprocess_data, data)

            if data == data_after:
                break

            data = data_after

        if self.verbose and data_before != data_after:  # pragma: nocover
            print 'hook_read_after() modified the input data:'
            print 'before:', repr(data_before)
            print 'after :', repr(data_after)

        return data

    def hook_handler(self, target, option, names, values, retval):
        if target in ['exp', 'line', 'input'] or not retval:
            return retval

{'fraction_non_alphanumeric': 0.1560262965668371, 'fraction_numerical': 0.005843681519357195, 'mean_word_length': 1.8963401734715464, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793720', 'n_tokens_mistral': 4162, 'n_tokens_neox': 3964, 'n_words': 1201}