<commit_before>
import re
import string

from common import *
<commit_msg>assume the common api exists<commit_after>
import re
import string




######## COMMON API ##########

E_INTEGER = 'INT'
E_STRING  = 'STR'
E_LAMBDA  = 'LAM'
E_VARREF  = 'VAR'

E_APPLICATION = 'APP'

E_OFFSET_LABEL = 'OFF'

E_OFFSET_REF = 'OFFREF'

E_BIN_RAW = 'RAW'
E_BIN_CONCAT = 'BINCAT'

E_BLOCK = 'BLK'

E_BUILTIN_FUNC = 'BUILTIN'

E_NEEDS_WORK = 'NEEDS_WORK'


def getbin(sx):
    if sx['type'] == E_BIN_RAW:
        return sx

    if sx['type'] == E_BIN_CONCAT:
        out = []
        for x in sx['data']:
            b = getbin(x)
            if b['type'] != E_BIN_RAW:
                raise Exception("Expected bin: " + str(b))
            out.append(b)
        sumlen = 0
        joined = []
        if out:
            sumlen = reduce(lambda x, y: x + y, [x['len'] for x in out])
            joined = reduce(lambda x, y: x + y, [x['data'] for x in out])
        return e_bin_raw(sumlen, joined)

    if sx['type'] == E_OFFSET_LABEL:
        return e_bin_raw(0, [])

    raise ValueError("Tried to get binary from " + str(sx))


# final: whether this is in its final form
# len:   the length of binary content, or None if n/a / unknown

# An easy way to tell the caller we need to be reconstructed (instead
# of having to reconstruct ourselves). This concept is possibly a bad
# idea.
def e_needs_work(length=None):
    return {'len': length, 'final': False, 'type':E_NEEDS_WORK, 'data': None}

def e_integer(n):
    return {'len': None, 'final': True, 'type':E_INTEGER, 'data':n}

def e_string(n):
    return {'len': None, 'final': True, 'type':E_STRING, 'data':n}

def e_varref(x):
    return {'len': None, 'final': False, 'type':E_VARREF, 'data': x}

def e_offset_label(x):
    return {'len': 0, 'final': True, 'type':E_OFFSET_LABEL, 'data': x}

def e_offset_ref(x):
    return {'len': None, 'final': False, 'type':E_OFFSET_REF, 'data': x}

def get_bin_concat_len(lst):
    lens = [x['len'] for x in lst]
    if not any([l is None for l in lens]):
        return sum(lens)
    return None

def e_bin_concat(lst, labels=None):
    if not labels:
        labels = {}
    return {'len': get_bin_concat_len(lst), 'final': all([x['final'] for x in lst]),
            'type':E_BIN_CONCAT, 'data': lst}

# x is a list of integers
def e_bin_raw(bitlen, x):
    return {'len':bitlen, 'final': True, 'type':E_BIN_RAW, 'data': x}

def e_block(assignments, value, labels=None):
    if not labels:
        labels = {}
    return {'len': value['len'], 'final': value['final'], 'type':E_BLOCK,
            'data': {'vars':assignments, 'val':value, 'labels':labels}}

def e_application(f, args, env=None):
    if f['type'] != E_VARREF:
        raise Exception("Application can only be performed on named functions")
    return {'len': None, 'final': False, 'type':E_APPLICATION, 'data': {'f': f, 'args': args, 'env':env}}

def e_lambda(params, body, env=None):
    return {'len': None, 'final': True, 'type':E_LAMBDA, 'data': {'params':params, 'body':body, 'env':env}}

def e_builtin_func(paramsnum, f):
    return {'len': None, 'final': True, 'type':E_BUILTIN_FUNC,
            'data': {'paramsnum': paramsnum, 'func': f}}

######## / COMMON API ##########





{'fraction_non_alphanumeric': 0.17006802721088435, 'fraction_numerical': 0.0009276437847866419, 'mean_word_length': 3.3716216216216215, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5634984', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1171, 'n_words': 344}