<commit_before>from collections import OrderedDict
from lxml import etree
from flask import abort, flash, jsonify, redirect, render_template, request, url_for

from IATISimpleTester import app, db
from IATISimpleTester.lib import helpers
from IATISimpleTester.lib.exceptions import FileGoneException, InvalidXMLException, ActivityNotFoundException
from IATISimpleTester.models import SuppliedData, Results, TestSet


def package_overview(uuid):
    try:
        response = _package_overview(uuid)
    except (FileGoneException, InvalidXMLException) as e:
        flash(str(e), 'danger')
        return redirect(url_for('home'))
    context = response['data']
    context['params'] = response['params']
    return render_template('overview.html', **context)

def _package_overview(uuid):
    params = {'uuid': str(uuid)}
    params.update(request.args)
    supplied_data = SuppliedData.query.get_or_404(str(uuid))
    filter_activities = request.args.get('filter') != 'false'
    test_set_id = request.args.get('test_set')
    test_set = TestSet(test_set_id)

    all_tests = test_set.all_tests
    filter_ = test_set.filter if filter_activities else None

    results = Results(supplied_data, test_set, None, filter_)
    percentages = results.percentages
    components = OrderedDict()
    for component in test_set.components.values():
        items = [percentages[test] for test in component.tests if test in percentages]
        if len(items) == 0:
            continue
        components[component.name] = sum(items) / len(items)
    context = {
        'results': results,
        'components': components,
        'name': supplied_data.name,
        'test_set': test_set,
    }

    return {
        'success': True,
        'data': context,
        'params': params,
    }

def package_quality_by_component(uuid, component):
    try:
        response = _package_quality_by_component(uuid, component)
    except (FileGoneException, InvalidXMLException) as e:
        flash(str(e), 'danger')
        return redirect(url_for('home'))
    context = response['data']
    context['params'] = response['params']
    return render_template('quality_by_component.html', **context)

def _package_quality_by_component(uuid, component_filter):
    params = {'uuid': str(uuid), 'component': component_filter}
    params.update(request.args)
    supplied_data = SuppliedData.query.get_or_404(str(uuid))
    filter_activities = request.args.get('filter') != 'false'
    test_set = TestSet(request.args.get('test_set'))

    component = test_set.components.get(component_filter)
    if not component:
        return abort(404)
    filter_ = test_set.filter if filter_activities else None

    results = Results(supplied_data, test_set, component.tests, filter_)
    context = {
        'results': results,
        'test_set': test_set,
    }

    return {
        'success': True,
        'data': context,
        'params': params,
    }

def package_quality_by_test(uuid, test):
    try:
        response = _package_quality_by_test(uuid, test)
    except (FileGoneException, InvalidXMLException) as e:
        flash(str(e), 'danger')
        return redirect(url_for('home'))
    context = response['data']
    context['params'] = response['params']
    return render_template('quality_by_test.html', **context)

def _package_quality_by_test(uuid, test):
    params = {
        'uuid': str(uuid),
        'test': test,
    }
    params.update(request.args)
    supplied_data = SuppliedData.query.get_or_404(str(uuid))
    filter_activities = request.args.get('filter') != 'false'
    test_set = TestSet(request.args.get('test_set'))<commit_msg>Some faffing with ImmutableMultiDicts…<commit_after>from collections import OrderedDict
from lxml import etree
from flask import abort, flash, jsonify, redirect, render_template, request, url_for

from IATISimpleTester import app, db
from IATISimpleTester.lib import helpers
from IATISimpleTester.lib.exceptions import FileGoneException, InvalidXMLException, ActivityNotFoundException
from IATISimpleTester.models import SuppliedData, Results, TestSet


def package_overview(uuid):
    try:
        response = _package_overview(uuid)
    except (FileGoneException, InvalidXMLException) as e:
        flash(str(e), 'danger')
        return redirect(url_for('home'))
    context = response['data']
    context['params'] = response['params']
    return render_template('overview.html', **context)

def _package_overview(uuid):
    params = {'uuid': str(uuid)}
    params.update(dict(request.args.items()))

    supplied_data = SuppliedData.query.get_or_404(str(uuid))
    filter_activities = request.args.get('filter') != 'false'
    test_set_id = request.args.get('test_set')
    test_set = TestSet(test_set_id)

    all_tests = test_set.all_tests
    filter_ = test_set.filter if filter_activities else None

    results = Results(supplied_data, test_set, None, filter_)
    percentages = results.percentages
    components = OrderedDict()
    for component in test_set.components.values():
        items = [percentages[test] for test in component.tests if test in percentages]
        if len(items) == 0:
            continue
        components[component.name] = sum(items) / len(items)
    context = {
        'results': results,
        'components': components,
        'name': supplied_data.name,
        'test_set': test_set,
    }

    return {
        'success': True,
        'data': context,
        'params': params,
    }

def package_quality_by_component(uuid, component):
    try:
        response = _package_quality_by_component(uuid, component)
    except (FileGoneException, InvalidXMLException) as e:
        flash(str(e), 'danger')
        return redirect(url_for('home'))
    context = response['data']
    context['params'] = response['params']
    return render_template('quality_by_component.html', **context)

def _package_quality_by_component(uuid, component_filter):
    params = {'uuid': str(uuid), 'component': component_filter}
    params.update(dict(request.args.items()))
    supplied_data = SuppliedData.query.get_or_404(str(uuid))
    filter_activities = request.args.get('filter') != 'false'
    test_set = TestSet(request.args.get('test_set'))

    component = test_set.components.get(component_filter)
    if not component:
        return abort(404)
    filter_ = test_set.filter if filter_activities else None

    results = Results(supplied_data, test_set, component.tests, filter_)
    context = {
        'results': results,
        'test_set': test_set,
    }

    return {
        'success': True,
        'data': context,
        'params': params,
    }

def package_quality_by_test(uuid, test):
    try:
        response = _package_quality_by_test(uuid, test)
    except (FileGoneException, InvalidXMLException) as e:
        flash(str(e), 'danger')
        return redirect(url_for('home'))
    context = response['data']
    context['params'] = response['params']
    return render_template('quality_by_test.html', **context)

def _package_quality_by_test(uuid, test):
    params = {
        'uuid': str(uuid),
        'test': test,
    }
    params.update(dict(request.args.items()))
    supplied_data = SuppliedData.query.get_or_404(str(uuid))
    filter_activities = request.args.get('filter') != 'false'
    test_set = TestSet(request.args.get('test_set'))
{'fraction_non_alphanumeric': 0.11498637602179837, 'fraction_numerical': 0.003542234332425068, 'mean_word_length': 3.7361290322580647, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '907083', 'n_tokens_mistral': 2234, 'n_tokens_neox': 2188, 'n_words': 476}