<commit_before>import time
import re
import sys
import antlr4
from antlr4.error.ErrorListener import ErrorListener

from gen.PSParser import PSParser
from gen.PSLexer import PSLexer
from gen.PSListener import PSListener

def process_sympy(sympy):

    matherror = MathErrorListener(sympy)

    stream = antlr4.InputStream(sympy)
    lex    = PSLexer(stream)
    lex.removeErrorListeners()
    lex.addErrorListener(matherror)

    tokens = antlr4.CommonTokenStream(lex)
    parser = PSParser(tokens)

    # remove default console error listener
    parser.removeErrorListeners()
    parser.addErrorListener(matherror)

    t0 = time.time()

    relation = parser.math().relation()
    sympy = stringify_relation(relation)

    t1 = time.time()
    return sympy

class MathErrorListener(ErrorListener):
    def __init__(self, src):
        super(ErrorListener, self).__init__()
        self.src = src

    def syntaxError(self, recog, symbol, line, col, msg, e):
        fmt = "%s\n%s\n%s"
        marker = "~" * col + "^"
        
        if msg.startswith("missing"):
            err = fmt % (msg, self.src, marker)
        elif msg.startswith("no viable"):
            err = fmt % ("I expected something else here", self.src, marker)
        elif msg.startswith("mismatched"):
            names = PSParser.literalNames
            expected = [names[i] for i in e.getExpectedTokens() if i < len(names)]
            if expected < 10:
                expected = " ".join(expected)
                err = (fmt % ("I expected one of these: " + expected,
                    self.src, marker))
            else:
                err = (fmt % ("I expected something else here", self.src, marker))
        else:
            err = fmt % ("I don't understand this", self.src, marker)
        raise Exception(err)

def stringify_relation(rel):
    if rel.expr():
        return stringify_expr(rel.expr())

    lh = stringify_relation(rel.relation(0))
    rh = stringify_relation(rel.relation(1))
    if rel.LT():
        fmt = "%s < %s"
    elif rel.LTE():
        fmt = "%s <= %s"
    elif rel.GT():
        fmt = "%s > %s"
    elif rel.GTE():
        fmt = "%s >= %s"
    elif rel.EQUAL():
        fmt = "solve(%s -(%s))"
    return fmt % (lh, rh)

def stringify_expr(expr):
    return stringify_add(expr.additive())

def stringify_add(add):
    if add.ADD():
       lh = stringify_add(add.additive(0))
       rh = stringify_add(add.additive(1))
       return "%s + %s" % (lh, rh)
    elif add.SUB():
        lh = stringify_add(add.additive(0))
        rh = stringify_add(add.additive(1))
        return "%s - %s" % (lh, rh)
    else:
        return stringify_mp(add.mp())

def stringify_mult(mult):
    arr = map(stringify_mp, mult.mp())
    return "*".join(arr)

def stringify_mp(mp):
    if mp.MUL() or mp.CMD_TIMES() or mp.CMD_CDOT():
        lh = stringify_mp(mp.mp(0))
        rh = stringify_mp(mp.mp(1))
        return "%s * %s" % (lh, rh)
    elif mp.DIV() or mp.CMD_DIV():
        lh = stringify_mp(mp.mp(0))
        rh = stringify_mp(mp.mp(1))
        return "%s / %s" % (lh, rh)
    elif mp.unary():
        return stringify_unary(mp.unary())

def stringify_unary(unary):
    if unary.ADD():
        return "+%s" % (stringify_unary(unary.unary()))
    elif unary.SUB():
        return "-%s" % (stringify_unary(unary.unary()))
    elif unary.postfix():
        return stringify_postfix_list(unary.postfix())

def stringify_postfix_list(arr, i=0):
    if i >= len(arr):
        raise Exception("Index out of bounds")

    res = stringify_postfix(arr[i])
    if isinstance(res, basestring):
        if i == len(arr) - 1:
            return res # nothing to multiply by
        else:
            # multiply by next
            return res + "*" + stringify_postfix_list(arr, i + 1)<commit_msg>Interpret 5x3 as multiplication<commit_after>import time
import re
import sys
import antlr4
from antlr4.error.ErrorListener import ErrorListener

from gen.PSParser import PSParser
from gen.PSLexer import PSLexer
from gen.PSListener import PSListener

def process_sympy(sympy):

    matherror = MathErrorListener(sympy)

    stream = antlr4.InputStream(sympy)
    lex    = PSLexer(stream)
    lex.removeErrorListeners()
    lex.addErrorListener(matherror)

    tokens = antlr4.CommonTokenStream(lex)
    parser = PSParser(tokens)

    # remove default console error listener
    parser.removeErrorListeners()
    parser.addErrorListener(matherror)

    t0 = time.time()

    relation = parser.math().relation()
    sympy = stringify_relation(relation)

    t1 = time.time()
    return sympy

class MathErrorListener(ErrorListener):
    def __init__(self, src):
        super(ErrorListener, self).__init__()
        self.src = src

    def syntaxError(self, recog, symbol, line, col, msg, e):
        fmt = "%s\n%s\n%s"
        marker = "~" * col + "^"
        
        if msg.startswith("missing"):
            err = fmt % (msg, self.src, marker)
        elif msg.startswith("no viable"):
            err = fmt % ("I expected something else here", self.src, marker)
        elif msg.startswith("mismatched"):
            names = PSParser.literalNames
            expected = [names[i] for i in e.getExpectedTokens() if i < len(names)]
            if expected < 10:
                expected = " ".join(expected)
                err = (fmt % ("I expected one of these: " + expected,
                    self.src, marker))
            else:
                err = (fmt % ("I expected something else here", self.src, marker))
        else:
            err = fmt % ("I don't understand this", self.src, marker)
        raise Exception(err)

def stringify_relation(rel):
    if rel.expr():
        return stringify_expr(rel.expr())

    lh = stringify_relation(rel.relation(0))
    rh = stringify_relation(rel.relation(1))
    if rel.LT():
        fmt = "%s < %s"
    elif rel.LTE():
        fmt = "%s <= %s"
    elif rel.GT():
        fmt = "%s > %s"
    elif rel.GTE():
        fmt = "%s >= %s"
    elif rel.EQUAL():
        fmt = "solve(%s -(%s))"
    return fmt % (lh, rh)

def stringify_expr(expr):
    return stringify_add(expr.additive())

def stringify_add(add):
    if add.ADD():
       lh = stringify_add(add.additive(0))
       rh = stringify_add(add.additive(1))
       return "%s + %s" % (lh, rh)
    elif add.SUB():
        lh = stringify_add(add.additive(0))
        rh = stringify_add(add.additive(1))
        return "%s - %s" % (lh, rh)
    else:
        return stringify_mp(add.mp())

def stringify_mult(mult):
    arr = map(stringify_mp, mult.mp())
    return "*".join(arr)

def stringify_mp(mp):
    if mp.MUL() or mp.CMD_TIMES() or mp.CMD_CDOT():
        lh = stringify_mp(mp.mp(0))
        rh = stringify_mp(mp.mp(1))
        return "%s * %s" % (lh, rh)
    elif mp.DIV() or mp.CMD_DIV():
        lh = stringify_mp(mp.mp(0))
        rh = stringify_mp(mp.mp(1))
        return "%s / %s" % (lh, rh)
    elif mp.unary():
        return stringify_unary(mp.unary())

def stringify_unary(unary):
    if unary.ADD():
        return "+%s" % (stringify_unary(unary.unary()))
    elif unary.SUB():
        return "-%s" % (stringify_unary(unary.unary()))
    elif unary.postfix():
        return stringify_postfix_list(unary.postfix())

def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def stringify_postfix_list(arr, i=0):
    if i >= len(arr):
        raise Exception("Index out of bounds")

    res = stringify_postfix(arr[i])
    if isinstance(res, basestring):
        if i == len(arr) - 1:
            return res # nothing to multiply by
        else:
            if i > 0:
                left  = arr[i - 1].getText()
                right = arr[i + 1].getText()
                if is_number(left) and is_number(right) and res == "x":
                    return stringify_postfix_list(arr, i + 1)
            # multiply by next
            return res + "*" + stringify_postfix_list(arr, i + 1)
{'fraction_non_alphanumeric': 0.1343227267010439, 'fraction_numerical': 0.006036976480945793, 'mean_word_length': 2.335570469798658, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4691940', 'n_tokens_mistral': 2682, 'n_tokens_neox': 2623, 'n_words': 644}