<commit_before>import ast
import re

def varrepl(matched_text):
    num = int(matched_text[1:])
    if num > 0:
        num -= 1
    return f'result.vars.get({num})'

VAR_PATTERN = re.compile(r'\$-?\d+')
VAR_PATTERN_END = re.compile(r'\$-?\d+$')

def compile_python_expressions(input_string, validator=lambda expr: True, raise_on_error=True):
    expressions = []
    remaining_text = input_string
    while remaining_text:
        try:
            expr_text, remaining_text = greedy_parse(remaining_text, validator)
        except Exception as e:
            if raise_on_error:
                raise e
            else:
                break
        expressions.append(expr_text)
    return expressions

def greedy_parse(s, validator):
    assert s
    first_error = None
    expr_text = None
    expr_matches = None
    remaining_text = None
    seen_string = ''
    for char in s:
        seen_string += char
        try_parse_string = re.sub(VAR_PATTERN, 'variables[0]', seen_string)
        try:
            expr = ast.parse(try_parse_string, mode='eval')
        except SyntaxError as e:
            first_error = first_error or e
        else:
            if not validator(expr):
                remaining_text = None
                break
            expr_matches = re.finditer(VAR_PATTERN, seen_string) 
            expr_text = seen_string
            remaining_text = s[len(seen_string):]
    if expr_text is None:
        raise first_error
    replaced_text = replace_matches(expr_matches, expr_text)
    return replaced_text, remaining_text

def replace_matches(matches, expr_text):
    '''
    find which $int matches we need to replace (Names) and which we
    don't (inside strings). Then replace whichever matches raised
    syntax errors
    '''
    replace_matches = []
    # replace in reversed order to preserve positions of earlier matches
    matches = list(reversed(list(matches)))
    for match in matches:
        teststr = expr_text
        test_matches = (m for m in matches if m is not match)
        for m in test_matches:
            teststr = teststr[:m.start()] + 'variables[0]' + teststr[m.end():]
        try:
            ast.parse(teststr, mode='eval')
        except SyntaxError:
            replace_matches.append(match)
    # replace the matches that raised syntax error if not changed
    replaced_text = expr_text
    for m in replace_matches:
        old = expr_text[m.start():m.end()]
        assert VAR_PATTERN_END.match(old)
        replaced_text = replaced_text[:m.start()] + varrepl(old) + replaced_text[m.end():]
    return replaced_text
    <commit_msg>pyexpr rewrite<commit_after>import ast
import re

def varrepl(_, num):
    num = int(num)
    if num > 0:
        num -= 1
    return f'result.vars.get({num})'

error_handler_strings = {
    (r'\$', r'-?\d+'): varrepl
}

error_handlers = {}
for (before_pattern, after_pattern), handler in error_handler_strings.items():
    before_pattern = None if before_pattern is None else re.compile(f'(.*)({before_pattern})$')
    after_pattern = None if after_pattern is None else re.compile(f'({after_pattern})(.*)')
    error_handlers[(before_pattern, after_pattern)] = handler

def compile_python_expressions(input_string, validator=lambda expr: True, raise_on_error=True):
    expressions = []
    remaining_text = input_string
    while remaining_text:
        try:
            expr_text, remaining_text = greedy_parse(remaining_text, validator)
        except Exception as e:
            remaining_text = handle_parse_error(remaining_text[:e.offset], remaining_text[e.offset:])
            if remaining_text is None:
                if raise_on_error:
                    raise e
                break
        else:
            expressions.append(expr_text)
    expressions = merge_expressions(expressions)
    return expressions

def merge_expressions(expressions):
    merged = []
    for expr in expressions:
        if not merged:
            merged.append(expr)
        else:
            merged_expr = merged[-1] + expr
            try:
                ast.parse(merged_expr, mode='eval')
            except:
                merged.append(expr)
            else:
                merged[-1] = merged_expr
    return merged

def handle_parse_error(before, after):
    for (before_pattern, after_pattern), handler in error_handlers.items():
        start, end = before, after
        before_error_text, after_error_text = None, None
        if before_pattern:
            bmatch = before_pattern.match(before)
            if not bmatch:
                break
            start, before_error_text = bmatch.group(1), bmatch.group(2)
        if after_pattern:
            amatch = after_pattern.match(after)
            if not amatch:
                break
            after_error_text, after = amatch.group(1), amatch.group(2)
        return start + handler(before_error_text, after_error_text) + after

def greedy_parse(s, validator):
    assert s
    last_error = None
    expr_text = None
    remaining_text = None
    seen_string = ''
    for char in s:
        seen_string += char
        try:
            expr = ast.parse(seen_string, mode='eval')
        except SyntaxError as e:
            last_error = e
        else:
            if not validator(expr):
                remaining_text = None
                break
            expr_text = seen_string
            remaining_text = s[len(seen_string):]
    if expr_text is None:
        raise last_error
    return expr_text, remaining_text
{'fraction_non_alphanumeric': 0.08755005460502366, 'fraction_numerical': 0.0023662176920276664, 'mean_word_length': 2.3102409638554215, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4653418', 'n_tokens_mistral': 1608, 'n_tokens_neox': 1589, 'n_words': 409}