<commit_before>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by <NAME>, <NAME>, <NAME> and <NAME>
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools
import models # damn!
import pprint

class NoRelevantResults(Exception): pass

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in 
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def remove_empty_dicts(h):
    has_keys = lambda kvp: len(kvp[1])
    return dict([ 
            (K, dict(filter(has_keys, V.items()))) 
            for K, V in h.items() 
            ])


class TestInfo(object):
    def __init__(self, test_id, results_pct, results_num, sampling_ok):
        test = models.Test.query.filter(models.Test.id == test_id).first()
        self.test_id = test_id
        self.test_name = test.name
        self.test_description = test.description
        self.test_group = test.test_group
        self.test_level = test.test_level
        self.results_pct = results_pct
        self.results_num = results_num
        self.sampling_ok = sampling_ok
        if sampling_ok:
            self.results_score = self.results_pct
        else:
            self.results_score = 0.0

    def as_dict(self):
        return {
            "test": {
                "id": self.test_id,
                "name": self.test_name,
                "description": self.test_description,
                "test_group": self.test_group,
                "test_level": self.test_level
                },
            "results_pct": self.results_pct,
            "results_num": self.results_num,
            "sampling_ok": self.sampling_ok,
            "results_score": self.results_score
            }


class IndicatorInfo(object):
    def __init__(self, indicator_id):
        self.indicator_id = indicator_id
        self.ind = models.Indicator.query.filter(
            models.Indicator.id == self.indicator_id).first()

    def as_dict(self):
        return self.ind.as_dict()

    def as_dict_minus_group(self):
        tmp = self.ind.as_dict()
        del(tmp['indicatorgroup_id'])
        return tmp


def publisher_indicators(indicators, indicators_tests, simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    def per_indicator(indicator):
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        relevant = lambda test: (indicator, test) in indicators_tests

        for test in filter(relevant, simple_out.keys()):
            indic_info = simple_out[test]
            results_pct += indic_info["results_pct"]
            results_num += indic_info["results_num"]
            results_weighted_pct_average_numerator += (
                indic_info["results_pct"] * 
                indic_info["results_num"]
                )
            indicator_test_data.append(indic_info)

        return {
            "indicator": IndicatorInfo(indicator).as_dict_minus_group(),
            "tests": indicator_test_data,
            "results_pct": (results_weighted_pct_average_numerator/results_num),
            "results_num": results_num
            }
    
    return dict([ (i, per_indicator(i)) for i in indicators ])


def make_summary(test_id, results_pct, results_num, sampling_ok):
    t = TestInfo(test_id, results_pct, results_num, sampling_ok)
    return t.as_dict()

def publisher_simple(out, cdtns):
    hierarchies = set(out)
    tests = set()
    for h in hierarchies:
        tests.update(set(out[h]))

    def per_test(t):
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        def relevant(hierarchy):
            key = (t,'activity hierarchy', str(hierarchy)) 

            if t not in out[hierarchy]:
                return False

            if not cdtns:
                return True

            if key not in cdtns:
                return True

            if cdtns[key][0] == 0:
                return False

            return True

        # This makes sure information about a test is returned.
        def get_okhierarchy(out, t):
            for h in out:
                if (t in out[h] and 'test' in out[h][t]):
                    return h
            raise NoRelevantResults(
                "Summary could not be generated for test %d" % t
                )

        for hierarchy in filter(relevant, hierarchies):
            test_info = out[hierarchy][t]
            
            results_pct += test_info["results_pct"]
            results_num += test_info["results_num"]
            results_weighted_pct_average_numerator += (
                test_info["results_pct"] * 
                test_info["results_num"]
                )

        if results_num == 0:
            raise NoRelevantResults("Results_num == 0 for test: %d" % t)

        # Result aggregation throws away hierarchies if there are 0 results.
        # This means some hierarchies won't have the 'test' dict. But, at
        # least one must, because we wouldn't be here otherwise.
        okhierarchy = get_okhierarchy(out, t)

        tmp = make_summary(
            out[okhierarchy][t]['test']["id"],
            (results_weighted_pct_average_numerator / results_num),
            results_num,
            True
            )
        tmp["indicator"] = out[okhierarchy][t]['indicator']
        return tmp

    return dict([ (t, per_test(t)) for t in tests ])


def sum_for_publishers(packages, d, h, t):
    # aggregate data across multiple packages for a single publisher ;
    # for each package, add percentage for each ;
    # need below to only include packages that are in this hierarchy

    relevant = lambda p: (h, t, p) in d
    relevant_data = map(lambda p: d[(h, t, p)], filter(relevant, packages))

    pct = lambda i: i[2]
    activities = lambda i: i[3]

    total_pct        = reduce(operator.add, map(pct,        relevant_data), 0)
    total_activities = reduce(operator.add, map(activities, relevant_data), 0)

    packages_in_hierarchy = len(relevant_data)

    if total_activities <= 0:
        return {}

    ok_tdata = relevant_data[-1] ## FIXME: this is obviously wrong

    test_id = ok_tdata[1]
         
    tmp = make_summary(
        test_id,
        float(total_pct/packages_in_hierarchy),
        total_activities,
        True
        )
    tmp["indicator"] = IndicatorInfo(ok_tdata[0]).as_dict()<commit_msg>fake results for a specific donor+test combination for testing<commit_after>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by <NAME>, <NAME>, <NAME> and <NAME>
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools
import models # damn!
import pprint

class NoRelevantResults(Exception): pass

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in 
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def remove_empty_dicts(h):
    has_keys = lambda kvp: len(kvp[1])
    return dict([ 
            (K, dict(filter(has_keys, V.items()))) 
            for K, V in h.items() 
            ])


class TestInfo(object):
    def __init__(self, test_id, results_pct, results_num, sampling_ok):
        test = models.Test.query.filter(models.Test.id == test_id).first()
        self.test_id = test_id
        self.test_name = test.name
        self.test_description = test.description
        self.test_group = test.test_group
        self.test_level = test.test_level
        self.results_pct = results_pct
        self.results_num = results_num
        self.sampling_ok = sampling_ok
        if test_id in [44, 43]:
            print test.name

            if test_id == 44:
                sampling_ok = False
        if sampling_ok:
            self.results_score = self.results_pct
        else:
            self.results_score = 0.0
        if test_id in [44, 43]:
            print self.results_pct
            print self.results_score

        self.results_pct = self.results_score


    def as_dict(self):
        return {
            "test": {
                "id": self.test_id,
                "name": self.test_name,
                "description": self.test_description,
                "test_group": self.test_group,
                "test_level": self.test_level
                },
            "results_pct": self.results_pct,
            "results_num": self.results_num,
            "sampling_ok": self.sampling_ok,
            "results_score": self.results_score
            }


class IndicatorInfo(object):
    def __init__(self, indicator_id):
        self.indicator_id = indicator_id
        self.ind = models.Indicator.query.filter(
            models.Indicator.id == self.indicator_id).first()

    def as_dict(self):
        return self.ind.as_dict()

    def as_dict_minus_group(self):
        tmp = self.ind.as_dict()
        del(tmp['indicatorgroup_id'])
        return tmp


def publisher_indicators(indicators, indicators_tests, simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    def per_indicator(indicator):
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        relevant = lambda test: (indicator, test) in indicators_tests

        for test in filter(relevant, simple_out.keys()):
            indic_info = simple_out[test]
            results_pct += indic_info["results_pct"]
            results_num += indic_info["results_num"]
            results_weighted_pct_average_numerator += (
                indic_info["results_pct"] * 
                indic_info["results_num"]
                )
            indicator_test_data.append(indic_info)

        return {
            "indicator": IndicatorInfo(indicator).as_dict_minus_group(),
            "tests": indicator_test_data,
            "results_pct": (results_weighted_pct_average_numerator/results_num),
            "results_num": results_num
            }
    
    return dict([ (i, per_indicator(i)) for i in indicators ])


def make_summary(test_id, results_pct, results_num, sampling_ok):
    t = TestInfo(test_id, results_pct, results_num, sampling_ok)
    return t.as_dict()

def publisher_simple(out, cdtns):
    hierarchies = set(out)
    tests = set()
    for h in hierarchies:
        tests.update(set(out[h]))

    def per_test(t):
        results_pct = 0.0
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        def relevant(hierarchy):
            key = (t,'activity hierarchy', str(hierarchy)) 

            if t not in out[hierarchy]:
                return False

            if not cdtns:
                return True

            if key not in cdtns:
                return True

            if cdtns[key][0] == 0:
                return False

            return True

        # This makes sure information about a test is returned.
        def get_okhierarchy(out, t):
            for h in out:
                if (t in out[h] and 'test' in out[h][t]):
                    return h
            raise NoRelevantResults(
                "Summary could not be generated for test %d" % t
                )

        for hierarchy in filter(relevant, hierarchies):
            test_info = out[hierarchy][t]
            
            results_pct += test_info["results_pct"]
            results_num += test_info["results_num"]
            results_weighted_pct_average_numerator += (
                test_info["results_pct"] * 
                test_info["results_num"]
                )

        if results_num == 0:
            raise NoRelevantResults("Results_num == 0 for test: %d" % t)

        # Result aggregation throws away hierarchies if there are 0 results.
        # This means some hierarchies won't have the 'test' dict. But, at
        # least one must, because we wouldn't be here otherwise.
        okhierarchy = get_okhierarchy(out, t)

        tmp = make_summary(
            out[okhierarchy][t]['test']["id"],
            (results_weighted_pct_average_numerator / results_num),
            results_num,
            True
            )
        tmp["indicator"] = out[okhierarchy][t]['indicator']
        return tmp

    return dict([ (t, per_test(t)) for t in tests ])


def sum_for_publishers(packages, d, h, t):
    # aggregate data across multiple packages for a single publisher ;
    # for each package, add percentage for each ;
    # need below to only include packages that are in this hierarchy
    
    relevant = lambda p: (h, t, p) in d
    relevant_data = map(lambda p: d[(h, t, p)], filter(relevant, packages))

    pct = lambda i: i[2]
    activities = lambda i: i[3]

    total_pct        = reduce(operator.add, map(pct,        relevant_data), 0)
    total_activities = reduce(operator.add, map(activities, relevant_data), 0)

    packages_in_hierarchy = len(relevant_data)

    if total_activities <= 0:
        return {}

    ok_tdata = relevant_data[-1] ## FIXME: this is obviously wrong

    test_id = ok_tdata[1]
         
    tmp = make_summary(
        test_id,
        float(total_pct/packages_in_hierarchy),
        total_activities,
        True
        )
    tmp["indicator"] = IndicatorInfo(ok_tdata[0]).as_dict()
{'fraction_non_alphanumeric': 0.09431395015754798, 'fraction_numerical': 0.007590948152391865, 'mean_word_length': 2.1151014945349096, 'pattern_counts': {'":': 28, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '63748', 'n_tokens_mistral': 4350, 'n_tokens_neox': 4137, 'n_words': 1078}