<commit_before>import os
import json
import six
import logging


from babbage import BabbageException
from flask import Blueprint, request, current_app
from flask.ext.jsonpify import jsonify

backwardAPI = Blueprint('BackwardAPI', __name__)

TAXONOMIES = json.load(open(os.path.join(os.path.dirname(__file__), 'taxonomies.json')))


log = logging.getLogger()


def configure_backward_api(app, manager):
    """ Configure the current Flask app with an instance of ``CubeManager`` that
    will be used to load and query data. """
    if not hasattr(app, 'extensions'):
        app.extensions = {}  # pragma: nocover
    app.extensions['cube_manager'] = manager
    log.info('SETUP Backward API')
    return backwardAPI


def get_attr_for_dimension_name(model, orig_dim):
    canonized = model.setdefault('_canonized', {})
    if orig_dim in canonized:
        return canonized[orig_dim]
    dim = orig_dim.replace('.', '_')
    attr = None
    if dim in model['dimensions']:
        dim = model['dimensions'][dim]
        attr = dim['attributes'][dim['key_attribute']]
    else:
        for name, dimension in model['dimensions'].items():
            logging.info('%s? %s:%r',
                         dim, name, list(dimension['attributes'].keys()))
            attributes = dimension['attributes']
            name = '_'.join(name.split('_')[1:])
            if name in [dim, dim+'_name']:
                attr = dimension
                attr['datatype'] = attr['attributes'][attr['key_attribute']]['datatype']
            else:
                for attr_name in [dim, dim+'_label', dim+'_name']:
                    if attr_name in attributes:
                        attr = attributes[attr_name]
                        break
            if attr is not None:
                break
        if attr is None:
            for name, dimension in model['dimensions'].items():
                for attr_name, attribute in dimension['attributes'].items():
                    if attr_name.endswith('_'+dim):
                        attr = attribute
                        break
                if attr is not None:
                    break
    assert attr is not None, "Failed to find dimension %s" % dim
    canonized[orig_dim] = attr
    return attr


def canonize_dimension(model, dim, val):
    """
    Convert old-style dimension name to one understandable by babbage.
    Check the datatype of the dimension and cast the given value to the correct type.

    Examples:
        year -> year
        time.year -> year
        functional-classification.func1_label -> func1_code.func1_label

    :param model: the babbage model as Python object
    :param dim: the dimension to be converted (as string)
    :param val: the provided value
    :return: tuple with new dimension name and value
    """
    ret_val = val
    attr = get_attr_for_dimension_name(model, dim)
    ref = attr['ref']
    datatype = attr['datatype']
    if datatype == 'string':
        ret_val = str(ret_val)
    elif datatype == 'integer':
        ret_val = int(ret_val)
    return ref, ret_val


def get_arg_with_default(param, default=None):
    """
    Get from the request's parameters a specific argument.
    If the argument doesn't appear in the request or the value is 'undefined', return the default.
    Use conv callable to attempt a conversion of the resulting value. If the conversion fails, the default is returned.
    :param param: which parameter to fetch
    :param default: default value to use in case of failures
    :param conv: callable to be used to convert the value, may be None in case no conversion is necessary
    :return: fetched value or default
    """
    arg = request.args.get(param,default)
    if arg == 'undefined':
        arg = default
    return arg


@backwardAPI.route('/aggregate')
def backward_compat_aggregate_api():
    """ Implement the old OS aggregate API over Babbage """

    # Fetch general parameters from the request
    orig_dataset = request.args.get('dataset')
    measure_name = get_arg_with_default('measure', 'amount')
    pagesize = request.args.get('pagesize', 10000, int)
    page = request.args.get('page', 1, int)

    dataset = '6018ab87076187018fc29c94a68a3cd2:__os_imported__' + orig_dataset
    log.info('AGGREGATE dataset:%s', dataset)

    # Check if the dataset even exists
    cm = current_app.extensions['cube_manager']
    if cm.has_cube(dataset):<commit_msg>Reduce default page size in bw compat api<commit_after>import os
import json
import six
import logging


from babbage import BabbageException
from flask import Blueprint, request, current_app
from flask.ext.jsonpify import jsonify

backwardAPI = Blueprint('BackwardAPI', __name__)

TAXONOMIES = json.load(open(os.path.join(os.path.dirname(__file__), 'taxonomies.json')))


log = logging.getLogger()


def configure_backward_api(app, manager):
    """ Configure the current Flask app with an instance of ``CubeManager`` that
    will be used to load and query data. """
    if not hasattr(app, 'extensions'):
        app.extensions = {}  # pragma: nocover
    app.extensions['cube_manager'] = manager
    log.info('SETUP Backward API')
    return backwardAPI


def get_attr_for_dimension_name(model, orig_dim):
    canonized = model.setdefault('_canonized', {})
    if orig_dim in canonized:
        return canonized[orig_dim]
    dim = orig_dim.replace('.', '_')
    attr = None
    if dim in model['dimensions']:
        dim = model['dimensions'][dim]
        attr = dim['attributes'][dim['key_attribute']]
    else:
        for name, dimension in model['dimensions'].items():
            logging.info('%s? %s:%r',
                         dim, name, list(dimension['attributes'].keys()))
            attributes = dimension['attributes']
            name = '_'.join(name.split('_')[1:])
            if name in [dim, dim+'_name']:
                attr = dimension
                attr['datatype'] = attr['attributes'][attr['key_attribute']]['datatype']
            else:
                for attr_name in [dim, dim+'_label', dim+'_name']:
                    if attr_name in attributes:
                        attr = attributes[attr_name]
                        break
            if attr is not None:
                break
        if attr is None:
            for name, dimension in model['dimensions'].items():
                for attr_name, attribute in dimension['attributes'].items():
                    if attr_name.endswith('_'+dim):
                        attr = attribute
                        break
                if attr is not None:
                    break
    assert attr is not None, "Failed to find dimension %s" % dim
    canonized[orig_dim] = attr
    return attr


def canonize_dimension(model, dim, val):
    """
    Convert old-style dimension name to one understandable by babbage.
    Check the datatype of the dimension and cast the given value to the correct type.

    Examples:
        year -> year
        time.year -> year
        functional-classification.func1_label -> func1_code.func1_label

    :param model: the babbage model as Python object
    :param dim: the dimension to be converted (as string)
    :param val: the provided value
    :return: tuple with new dimension name and value
    """
    ret_val = val
    attr = get_attr_for_dimension_name(model, dim)
    ref = attr['ref']
    datatype = attr['datatype']
    if datatype == 'string':
        ret_val = str(ret_val)
    elif datatype == 'integer':
        ret_val = int(ret_val)
    return ref, ret_val


def get_arg_with_default(param, default=None):
    """
    Get from the request's parameters a specific argument.
    If the argument doesn't appear in the request or the value is 'undefined', return the default.
    Use conv callable to attempt a conversion of the resulting value. If the conversion fails, the default is returned.
    :param param: which parameter to fetch
    :param default: default value to use in case of failures
    :param conv: callable to be used to convert the value, may be None in case no conversion is necessary
    :return: fetched value or default
    """
    arg = request.args.get(param,default)
    if arg == 'undefined':
        arg = default
    return arg


@backwardAPI.route('/aggregate')
def backward_compat_aggregate_api():
    """ Implement the old OS aggregate API over Babbage """

    # Fetch general parameters from the request
    orig_dataset = request.args.get('dataset')
    measure_name = get_arg_with_default('measure', 'amount')
    pagesize = request.args.get('pagesize', 100, int)
    page = request.args.get('page', 1, int)

    dataset = '6018ab87076187018fc29c94a68a3cd2:__os_imported__' + orig_dataset
    log.info('AGGREGATE dataset:%s', dataset)

    # Check if the dataset even exists
    cm = current_app.extensions['cube_manager']
    if cm.has_cube(dataset):
{'fraction_non_alphanumeric': 0.09882406151062867, 'fraction_numerical': 0.007236544549977386, 'mean_word_length': 2.6762261014131337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6373182', 'n_tokens_mistral': 2532, 'n_tokens_neox': 2431, 'n_words': 808}