<commit_before>import logging
from collections import defaultdict

from pylons import request, response, app_globals
from pylons.controllers.util import abort

from openspending import model
from openspending.lib import calculator
from openspending.lib import solr_util as solr
from openspending.lib.cubes import find_cube
from openspending.ui.lib.base import BaseController
from openspending.ui.lib.jsonp import jsonpify

log = logging.getLogger(__name__)

def statistic_normalize(dataset, result, per, statistic):
    drilldowns = []
    values = {}
    for drilldown in result['drilldown']:
        per_value = drilldown.get(per)
        if not per_value in values:
            entry = model.entry.find_one({'dataset.name': dataset['name'],
                                          per: per_value})
            values[per_value] = entry.get(statistic, 0.0) if entry else 0.0
        if values[per_value]: # skip division by zero oppprtunities
            drilldown['amount'] /= values[per_value]
            drilldowns.append(drilldown)
    result['drilldown'] = drilldowns
    return result

def cellget(cell, key):
    val = cell.get(key)
    if isinstance(val, dict):
        return val.get('name', val.get('_id'))
    return val

class ApiController(BaseController):
    @jsonpify
    def index(self):
        out = {
            'doc': 'See %s' % app_globals.api_link
            }
        return out

    def search(self):
        solrargs = dict(request.params)
        rows = min(1000, request.params.get('rows', 10))
        q = request.params.get('q', '*:*')
        solrargs['q'] = q
        solrargs['rows'] = rows
        solrargs['wt'] = 'json'
        if 'callback' in solrargs and not 'json.wrf' in solrargs:
            solrargs['json.wrf'] = solrargs['callback']
        if not 'sort' in solrargs:
            solrargs['sort'] = 'score desc,amount desc'
        query = solr.get_connection().raw_query(**solrargs)
        response.content_type = 'application/json'
        return query

    @jsonpify
    def aggregate(self):
        dataset_name = request.params.get('dataset', request.params.get('slice'))
        dataset = model.dataset.find_one_by('name', dataset_name)

        if dataset is None:
            abort(400, "Dataset %s not found" % dataset_name)

        drilldowns, cuts, statistics = [], [], []
        for key, value in sorted(request.params.items()):
            if not '-' in key:
                continue
            op, key = key.split('-', 1)<commit_msg>Add hardwired dataset aliases to old API to avoid dataset reloads<commit_after>import logging
from collections import defaultdict

from pylons import request, response, app_globals
from pylons.controllers.util import abort

from openspending import model
from openspending.lib import calculator
from openspending.lib import solr_util as solr
from openspending.lib.cubes import find_cube
from openspending.ui.lib.base import BaseController
from openspending.ui.lib.jsonp import jsonpify
import re

log = logging.getLogger(__name__)

def statistic_normalize(dataset, result, per, statistic):
    drilldowns = []
    values = {}
    for drilldown in result['drilldown']:
        per_value = drilldown.get(per)
        if not per_value in values:
            entry = model.entry.find_one({'dataset.name': dataset['name'],
                                          per: per_value})
            values[per_value] = entry.get(statistic, 0.0) if entry else 0.0
        if values[per_value]: # skip division by zero oppprtunities
            drilldown['amount'] /= values[per_value]
            drilldowns.append(drilldown)
    result['drilldown'] = drilldowns
    return result

def cellget(cell, key):
    val = cell.get(key)
    if isinstance(val, dict):
        return val.get('name', val.get('_id'))
    return val

# This is a temporary hack-on-a-stick to get things working again
dataset_aliases = {
    'ukdepartments': 'ukgov-25k-spending'
}

pattern = re.compile(r'(dataset:\S+)')

def amend_query(q):
    matches = pattern.search(q)
    if not matches:
        return q
    _, dataset = matches.group(0).split(':')
    dataset = dataset_aliases.get(dataset, dataset)
    return pattern.sub('dataset:' + dataset, q)

class ApiController(BaseController):
    @jsonpify
    def index(self):
        out = {
            'doc': 'See %s' % app_globals.api_link
            }
        return out

    def search(self):
        solrargs = dict(request.params)
        rows = min(1000, request.params.get('rows', 10))
        q = request.params.get('q', '*:*')
        solrargs['q'] = amend_query(q)
        solrargs['rows'] = rows
        solrargs['wt'] = 'json'
        if 'callback' in solrargs and not 'json.wrf' in solrargs:
            solrargs['json.wrf'] = solrargs['callback']
        if not 'sort' in solrargs:
            solrargs['sort'] = 'score desc,amount desc'
        query = solr.get_connection().raw_query(**solrargs)
        response.content_type = 'application/json'
        return query

    @jsonpify
    def aggregate(self):
        dataset_name = request.params.get('dataset', request.params.get('slice'))
        dataset_name = dataset_aliases.get(dataset_name, dataset_name)
        dataset = model.dataset.find_one_by('name', dataset_name)

        if dataset is None:
            abort(400, "Dataset %s not found" % dataset_name)

        drilldowns, cuts, statistics = [], [], []
        for key, value in sorted(request.params.items()):
            if not '-' in key:
                continue
            op, key = key.split('-', 1)
{'fraction_non_alphanumeric': 0.11406980928391508, 'fraction_numerical': 0.005577545879812882, 'mean_word_length': 2.657236842105263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7039352', 'n_tokens_mistral': 1720, 'n_tokens_neox': 1675, 'n_words': 429}