<commit_before>from charcoaltoken import CharcoalToken as CT
from unicodegrammars import UnicodeGrammars


def PassThrough(r):
    return r

def GetFreeVariable(s, n=1):
    r = ""
    for _ in range(n):
        r += next(filter(lambda c: c not in s + r, "ικλμνξπρςστυφχψωαβγδεζηθ"))
    return r

def VerbosifyVariable(c):
    return "iklmnxprsvtufcywabgdezhq"["ικλμνξπρςστυφχψωαβγδεζηθ".find(c)]

def EvaluateFunctionOrList(f, s):
    if isinstance(f, list):
        return f[0](s)
    return f(s)

ASTProcessor = {
    CT.Arrow: [
        lambda r: [lambda s="": [r[0] + ": Left"]],
        lambda r: [lambda s="": [r[0] + ": Up"]],
        lambda r: [lambda s="": [r[0] + ": Right"]],
        lambda r: [lambda s="": [r[0] + ": Down"]],
        lambda r: [lambda s="": [r[0] + ": Up Left"]],
        lambda r: [lambda s="": [r[0] + ": Up Right"]],
        lambda r: [lambda s="": [r[0] + ": Down Right"]],
        lambda r: [lambda s="": [r[0] + ": Down Left"]],
        lambda r: [lambda s="": [r[0] + ": Direction", r[1][0](s)]]
    ],
    CT.Multidirectional: [
        lambda r: [lambda s="": ["Multidirectional"] + r[0][0](s)[1:] + r[1][0](s)[1:]],
        lambda r: [lambda s="": [r[1][0](s)[0], r[0] + ": Right, down, left, up"] + r[1][0](s)[1:]],
        lambda r: [lambda s="": [r[1][0](s)[0], r[0] + ": Down right, down left, up left, up right, "] + r[1][0](s)[1:]],
        lambda r: [lambda s="": [r[1][0](s)[0], r[0] + ": Right, down right, down, down left, left, up left, up, up right, "] + r[1][0](s)[1:]],
        lambda r: [lambda s="": [r[1][0](s)[0], r[0] + ": Down, up"] + r[1][0](s)[1:]],<commit_msg>Fix s and v variables in AST output<commit_after>from charcoaltoken import CharcoalToken as CT
from unicodegrammars import UnicodeGrammars


def PassThrough(r):
    return r

def GetFreeVariable(s, n=1):
    r = ""
    for _ in range(n):
        r += next(filter(lambda c: c not in s + r, "ικλμνξπρςστυφχψωαβγδεζηθ"))
    return r

def VerbosifyVariable(c):
    return "iklmnxprvstufcywabgdezhq"["ικλμνξπρςστυφχψωαβγδεζηθ".find(c)]

def EvaluateFunctionOrList(f, s):
    if isinstance(f, list):
        return f[0](s)
    return f(s)

ASTProcessor = {
    CT.Arrow: [
        lambda r: [lambda s="": [r[0] + ": Left"]],
        lambda r: [lambda s="": [r[0] + ": Up"]],
        lambda r: [lambda s="": [r[0] + ": Right"]],
        lambda r: [lambda s="": [r[0] + ": Down"]],
        lambda r: [lambda s="": [r[0] + ": Up Left"]],
        lambda r: [lambda s="": [r[0] + ": Up Right"]],
        lambda r: [lambda s="": [r[0] + ": Down Right"]],
        lambda r: [lambda s="": [r[0] + ": Down Left"]],
        lambda r: [lambda s="": [r[0] + ": Direction", r[1][0](s)]]
    ],
    CT.Multidirectional: [
        lambda r: [lambda s="": ["Multidirectional"] + r[0][0](s)[1:] + r[1][0](s)[1:]],
        lambda r: [lambda s="": [r[1][0](s)[0], r[0] + ": Right, down, left, up"] + r[1][0](s)[1:]],
        lambda r: [lambda s="": [r[1][0](s)[0], r[0] + ": Down right, down left, up left, up right, "] + r[1][0](s)[1:]],
        lambda r: [lambda s="": [r[1][0](s)[0], r[0] + ": Right, down right, down, down left, left, up left, up, up right, "] + r[1][0](s)[1:]],
        lambda r: [lambda s="": [r[1][0](s)[0], r[0] + ": Down, up"] + r[1][0](s)[1:]],
{'fraction_non_alphanumeric': 0.24572127139364303, 'fraction_numerical': 0.028728606356968216, 'mean_word_length': 3.16412213740458, 'pattern_counts': {'":': 54, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7080534', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1357, 'n_words': 344}