<commit_before>import base64
import jobs
import bottle
import json
import time
import uuid
import re
import picarus_takeout
import functools
import msgpack
from driver import PicarusManager
from parameters import PARAM_SCHEMAS_SERVE
from model_factories import FACTORIES

# These need to be set before using this module
thrift_lock = None
thrift_new = None
VERSION = None
JOBS = None


def dod_to_lod_b64(dod):
    # Converts from dod[row][column] to list of {row, col0_ub64:val0_b64, ...}
    # dod: dict of dicts
    # lod: list of dicts
    outs = []
    for row, columns in sorted(dod.items(), key=lambda x: x[0]):
        out = {'row': base64.b64encode(row)}
        out.update({base64.b64encode(x): base64.b64encode(y) if isinstance(y, str) else base64.b64encode(json.dumps(y)) for x, y in columns.items()})
        outs.append(out)
    return outs

PARAM_SCHEMAS_B64 = dod_to_lod_b64(PARAM_SCHEMAS_SERVE)


def encode_row(row, columns):
    out = {base64.b64encode(k): base64.b64encode(v) for k, v in columns.items()}
    out['row'] = base64.b64encode(row)
    return out


def key_to_model(manager, *args, **kw):
    try:
        return manager.key_to_model(*args, **kw)
    except IndexError:
        bottle.abort(404)


def _takeout_model_link_from_key(manager, key):
    print('Manager key[%r]' % key)
    model_binary, columns = key_to_model(manager, key, 'link')
    model = msgpack.loads(model_binary)
    if not isinstance(model, dict):
        bottle.abort(400)
    return model


def _takeout_model_chain_from_key(manager, key):
    if key == 'data:image':
        return []
    columns = key_to_model(manager, key)
    if columns['input_type'] == 'raw_image':
        return [_takeout_model_link_from_key(manager, key)]
    return _takeout_model_chain_from_key(manager, columns['input']) + [_takeout_model_link_from_key(manager, key)]


def _takeout_input_model_link_from_key(manager, key):
    model_binary, columns = key_to_model(manager, key, 'link')
    model = msgpack.loads(model_binary)
    if not isinstance(model, dict):
        bottle.abort(400)
    return columns['input'], model


def _takeout_input_model_chain_from_key(manager, key):
    columns = key_to_model(manager, key)
    if columns['input_type'] == 'raw_image':
        return [_takeout_input_model_link_from_key(manager, key)]
    return _takeout_input_model_chain_from_key(manager, columns['input']) + [_takeout_input_model_link_from_key(manager, key)]


def _parse_params(params, schema):
    kw = {}
    schema_params = schema['params']
    prefix = 'param'

    def get_param(x, func=str, exception=False):
        try:
            return func(params[prefix + '-' + x])
        except (KeyError, ValueError):
            if exception:
                raise
            bottle.abort(400)
    for param_name, param in schema_params.items():
        if param['type'] == 'enum':
            param_value = get_param(param_name)
            if param_value not in param['values']:
                bottle.abort(400)
            kw[param_name] = param_value
        elif param['type'] == 'int':
            param_value = get_param(param_name, int)
            if not (param['min'] <= param_value < param['max']):
                bottle.abort(400)
            kw[param_name] = param_value
        elif param['type'] == 'float':
            param_value = get_param(param_name, float)
            if not (param['min'] <= param_value < param['max']):<commit_msg>Added slightly more informative error message<commit_after>import base64
import jobs
import bottle
import json
import time
import uuid
import re
import picarus_takeout
import functools
import msgpack
from driver import PicarusManager
from parameters import PARAM_SCHEMAS_SERVE
from model_factories import FACTORIES

# These need to be set before using this module
thrift_lock = None
thrift_new = None
VERSION = None
JOBS = None


def dod_to_lod_b64(dod):
    # Converts from dod[row][column] to list of {row, col0_ub64:val0_b64, ...}
    # dod: dict of dicts
    # lod: list of dicts
    outs = []
    for row, columns in sorted(dod.items(), key=lambda x: x[0]):
        out = {'row': base64.b64encode(row)}
        out.update({base64.b64encode(x): base64.b64encode(y) if isinstance(y, str) else base64.b64encode(json.dumps(y)) for x, y in columns.items()})
        outs.append(out)
    return outs

PARAM_SCHEMAS_B64 = dod_to_lod_b64(PARAM_SCHEMAS_SERVE)


def encode_row(row, columns):
    out = {base64.b64encode(k): base64.b64encode(v) for k, v in columns.items()}
    out['row'] = base64.b64encode(row)
    return out


def key_to_model(manager, *args, **kw):
    try:
        return manager.key_to_model(*args, **kw)
    except IndexError:
        bottle.abort(404)


def _takeout_model_link_from_key(manager, key):
    print('Manager key[%r]' % key)
    model_binary, columns = key_to_model(manager, key, 'link')
    model = msgpack.loads(model_binary)
    if not isinstance(model, dict):
        bottle.abort(400)
    return model


def _takeout_model_chain_from_key(manager, key):
    if key == 'data:image':
        return []
    columns = key_to_model(manager, key)
    if columns['input_type'] == 'raw_image':
        return [_takeout_model_link_from_key(manager, key)]
    return _takeout_model_chain_from_key(manager, columns['input']) + [_takeout_model_link_from_key(manager, key)]


def _takeout_input_model_link_from_key(manager, key):
    model_binary, columns = key_to_model(manager, key, 'link')
    model = msgpack.loads(model_binary)
    if not isinstance(model, dict):
        bottle.abort(400)
    return columns['input'], model


def _takeout_input_model_chain_from_key(manager, key):
    columns = key_to_model(manager, key)
    if columns['input_type'] == 'raw_image':
        return [_takeout_input_model_link_from_key(manager, key)]
    return _takeout_input_model_chain_from_key(manager, columns['input']) + [_takeout_input_model_link_from_key(manager, key)]


def _parse_params(params, schema):
    kw = {}
    schema_params = schema['params']
    prefix = 'param'

    def get_param(x, func=str, exception=False):
        param_key = prefix + '-' + x
        try:
            return func(params[param_key])
        except (KeyError, ValueError):
            if exception:
                raise
            bottle.abort(400, 'Parameter not found [%s]' % (param_key,))
    for param_name, param in schema_params.items():
        if param['type'] == 'enum':
            param_value = get_param(param_name)
            if param_value not in param['values']:
                bottle.abort(400)
            kw[param_name] = param_value
        elif param['type'] == 'int':
            param_value = get_param(param_name, int)
            if not (param['min'] <= param_value < param['max']):
                bottle.abort(400)
            kw[param_name] = param_value
        elif param['type'] == 'float':
            param_value = get_param(param_name, float)
            if not (param['min'] <= param_value < param['max']):
{'fraction_non_alphanumeric': 0.11425711425711425, 'fraction_numerical': 0.017446017446017446, 'mean_word_length': 3.3959773727215588, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4392777', 'n_tokens_mistral': 2366, 'n_tokens_neox': 2301, 'n_words': 539}