<commit_before>from cfda.models import Program, ProgramObligation, Agency
from metrics.models import *
from django.shortcuts import render_to_response
from django.db.models import Count, Sum
from django.db.models.query import QuerySet
from decimal import Decimal
from django.core.mail import send_mail
from helpers.format import moneyfmt
import math
from urllib import unquote
from haystack.query import SearchQuerySet
from haystack.models import SearchResult

FISCAL_YEARS = [2007, 2008, 2009]

def contact(request):
    #submission of contact form
    name = request.POST.get('name', '')
    email = request.POST.get('email', '')
    msg = request.POST.get('message', '')
    send_mail('Clearspending Feedback from '+name, msg, email, ['<EMAIL>'], fail_silently=True)
    return render_to_response('contact_thankyou.html')

def search_results(request, search_string, unit='pct', fiscal_year=2009):

    search = unquote(search_string)
    programs = SearchQuerySet().filter(content=search_string)
    result_count = programs.count()
    table_data = generic_program_table(programs, fiscal_year, unit)
    
    return render_to_response('generic_program_list.html', { 'table_data': table_data, 'fiscal_year': fiscal_year, 'unit': unit, 'search_string': search, 'result_count': result_count })
    
 

def get_css_color(pct, metric):
    if metric == 'con':  #consistency
        pct = Decimal(str(math.fabs(pct)))
        if pct > Decimal('50'): return 'bad'
        elif pct > Decimal('25'): return 'warn'
        else: return 'good'
    elif metric == 'time':
        if pct > Decimal('.1'): return 'bad'
        else: return 'good'
    elif metric == 'com':
        if pct > Decimal('5'): return 'bad'
        else: return 'good'
    
def get_first(set):
    if isinstance(set, QuerySet) and len(set) > 0:
        return set[0]
    else: return None

def get_timeliness(timeliness, unit):
    if timeliness:
        try:
            pct = timeliness.late_dollars / timeliness.total_dollars
        except Exception:
            pct = 0
        if unit == 'pct':
            return (pct*100, get_css_color(pct, 'time'))
        else:
            return (timeliness.late_dollars, get_css_color(pct, 'time'))
    else:
        return (None,'')

def get_completeness(unit, **completeness):
    if completeness:
        try:
            pct = (completeness['failed_total']/ completeness['total']) * 100
        except Exception:
            pct = 0
        if unit == 'pct':
            return (pct, get_css_color(pct, 'com'))
        else:
            return (completeness['failed_total'], get_css_color(pct, 'com'))
    else:
        return (None, '')

def get_consistency(consistency, unit):
    
    if consistency:
        over = math.fabs(consistency.__dict__['over_reported_'+unit] or 0)
        under = math.fabs(consistency.__dict__['under_reported_'+unit] or 0)
        non = math.fabs(consistency.__dict__['non_reported_'+unit] or 0)
        return (over, get_css_color(consistency.over_reported_pct or 0, 'con'), 
                under, get_css_color(consistency.under_reported_pct or 0, 'con'), 
                non, get_css_color(consistency.non_reported_pct or 0, 'con'))
    else:
        placeholders = []
        for i in range(0, 6):
            placeholders.append(None)
        return placeholders

def generic_program_table(programs, fiscal_year, unit):
    #builds a scorecard table for a generic list of programs
    table_data = []
    types = [None, "grants", "loans"]
    for p in programs:
        display_name = p.program_title
        program_number = p.program_number
         
        if p.__class__ == SearchResult:
            p = p.pk
            p_id = p
        else: 
            p_id = p.id
        consistency = ProgramConsistency.objects.filter(fiscal_year=fiscal_year, program=p)
        timeliness = ProgramTimeliness.objects.filter(fiscal_year=fiscal_year, program=p)
        completeness_totals = ProgramCompleteness.objects.filter(program=p, fiscal_year=fiscal_year).aggregate(failed_total=Sum('completeness_failed_dollars'), total=Sum('completeness_total_dollars'))
        if len(consistency) > 0:
            ob_type = "(%s)" % types[get_first(consistency).type]
        else: ob_type = ""

        #if len(display_name) > 35: display_name = "%s..." % display_name[0:32]
        row = [ program_number,
                p_id,
                "%s %s" % (display_name, ob_type),
                ]
        row.extend(get_consistency(get_first(consistency), unit))
        row.extend(get_timeliness(get_first(timeliness), unit))
        row.extend(get_completeness(unit, **completeness_totals))
        table_data.append(row)

        if len(consistency) > 1:
            for ob in consistency[1:]:
                row = [ program_number,
                        p_id,
                        "%s (%s)" % (display_name, types[ob.type]),
                        ]
                row.extend(get_consistency(ob, unit))
                row.extend(get_timeliness(get_first(timeliness), unit))
                row.extend(get_completeness(unit, **completeness_totals))
                table_data.append(row)

    return table_data

def index(request, unit='dollars', fiscal_year=2009):
    #get top level agency stats 
    consistency = AgencyConsistency.objects.filter(fiscal_year=fiscal_year).order_by('agency__name')
    timeliness = AgencyTimeliness.objects.filter(fiscal_year=fiscal_year).order_by('agency__name')
    agencies = Agency.objects.all().order_by('name')
    table_data = []
    for a in agencies:

        number_programs = Program.objects.filter(agency=a).count()
        display_name = a.name
      #  if len(display_name) > 35: display_name = "%s..." % display_name[0:35]
        a_data = [a.code,
                  number_programs, 
                  a.name,
                  display_name]

        a_data.extend(get_consistency(get_first(consistency.filter(agency=a)), unit))
        a_data.extend(get_timeliness(get_first(timeliness.filter(agency=a)), unit))
        completeness_totals = ProgramCompleteness.objects.filter(program__agency=a, fiscal_year=fiscal_year).aggregate(failed_total=Sum('completeness_failed_dollars'), total=Sum('completeness_total_dollars'))
        a_data.extend(get_completeness(unit, **completeness_totals))

        table_data.append(a_data) 

    return render_to_response('scorecard_index.html', {'table_data': table_data, 'fiscal_year': "%s" % fiscal_year, 'unit':unit})

def agencyDetail(request, agency_id, unit='dollars', fiscal_year=2009):
    
    summary_numbers = []
    summary_numbers.append(AgencyConsistency.objects.filter(agency=agency_id, fiscal_year=fiscal_year).aggregate(total=Sum('total_cfda_obligations'))['total'])
    summary_numbers.extend(get_consistency(get_first(AgencyConsistency.objects.filter(agency=agency_id, fiscal_year=fiscal_year)), 'dollars'))
    summary_numbers.extend(get_timeliness(get_first(AgencyTimeliness.objects.filter(agency=agency_id, fiscal_year=fiscal_year)), 'dollars'))
    completeness_totals = ProgramCompleteness.objects.filter(program__agency=agency_id, fiscal_year=fiscal_year).aggregate(failed_total=Sum('completeness_failed_dollars'), total=Sum('completeness_total_dollars'))
    summary_numbers.extend(get_completeness('dollars', **completeness_totals)) 
    summary_numbers = filter( lambda x: not isinstance(x, basestring) , summary_numbers) #remove css colors

    programs = Program.objects.filter(agency=agency_id)
    agency = Agency.objects.get(code=agency_id)
    table_data = generic_program_table(programs, fiscal_year, unit)

    return render_to_response('agency_detail.html', {'summary_numbers': summary_numbers, 'table_data': table_data, 'fiscal_year': fiscal_year, 'unit': unit, 'agency_name': agency.name, 'agency': agency_id, 'description': agency.description})


def programDetail(request, program_id, unit):
    program = Program.objects.get(id=program_id)
    
    program_total_number = ProgramObligation.objects.filter(program=program).aggregate(total=Sum('obligation'))['total']
    program_total = moneyfmt(Decimal(str(program_total_number)), places=0, curr='$', sep=',', dp='')

    consistency_block = programDetailConsistency(program, unit) 
    field_names = ['late_'+unit, 'avg_lag_rows']
    proper_names = ['Late', 'Average Late Records']
    coll = ProgramTimeliness.objects.filter(program=program_id).order_by('fiscal_year')
    timeliness_block = programDetailGeneral(program_id, unit, field_names, proper_names, coll, 'Timeliness')

    if len(program.objectives) < 1300:
        description = program.objectives
    else: description = program.objectives[:1300] + '... <a href="http://cfda.gov">read more at CFDA.gov</a>.'

    #update these lists with all the fields we want to show
    com_field_names = ['recipient_type_is_not_empty', 'federal_agency_code_is_not_empty', 'cfda_program_num_is_descriptive', 'federal_funding_amount_is_not_empty',
                        'principal_place_code_not_empty', 'recipient_name_not_empty', 'recipient_state_code_not_empty', 'recipient_county_code_not_empty_or_too_long',
                        'recipient_county_name_not_empty', 'recipient_city_code_not_empty', 'recipient_city_name_not_empty', 'principal_place_state_not_empty', 'record_type_is_not_empty', 
                        'action_type_is_not_empty', 'recipient_cong_district_is_not_empty', 'obligation_action_date_is_properly_formatted', 
                        'assistance_type_is_not_empty', 'federal_award_id_is_not_empty']
    
    com_proper_names = ['Recipient type', 'Federal Agency', 'CFDA Program Number', 'Funding Amount',
                        'Principal Place of Grant Code',  'Recipient Name', 'Recipient State', 'Recipient County Code', 
                        'Recipient County Name', 'Recipient City Code', 'Recipient City Name', 'Principal State of Grant', 'Record Type', 
                        'Action Type', 'Recipient Congressional District', 'Obligation Action Date Formatted Correctly', 
                        'Assistance Type', 'Federal Award ID' ]
    com_coll = ProgramCompletenessDetail.objects.filter(program=program_id).order_by('fiscal_year')
    completeness_block = programDetailGeneral(program_id, unit, com_field_names, com_proper_names, com_coll, 'Completeness')
    
    return render_to_response('program_detail.html', {'consistency':consistency_block, 'timeliness': timeliness_block, 'completeness': completeness_block, 'agency_name': program.agency.name, 'program_number': program.program_number, 'title': program.program_title, 'desc': description, 'unit': unit, 'program_total': program_total}) 
    <commit_msg>fixed decimal error<commit_after>from cfda.models import Program, ProgramObligation, Agency
from metrics.models import *
from django.shortcuts import render_to_response
from django.db.models import Count, Sum
from django.db.models.query import QuerySet
from decimal import Decimal
from django.core.mail import send_mail
from helpers.format import moneyfmt
import math
from urllib import unquote
from haystack.query import SearchQuerySet
from haystack.models import SearchResult

FISCAL_YEARS = [2007, 2008, 2009]

def contact(request):
    #submission of contact form
    name = request.POST.get('name', '')
    email = request.POST.get('email', '')
    msg = request.POST.get('message', '')
    send_mail('Clearspending Feedback from '+name, msg, email, ['<EMAIL>'], fail_silently=True)
    return render_to_response('contact_thankyou.html')

def search_results(request, search_string, unit='pct', fiscal_year=2009):

    search = unquote(search_string)
    programs = SearchQuerySet().filter(content=search_string)
    result_count = programs.count()
    table_data = generic_program_table(programs, fiscal_year, unit)
    
    return render_to_response('generic_program_list.html', { 'table_data': table_data, 'fiscal_year': fiscal_year, 'unit': unit, 'search_string': search, 'result_count': result_count })
    
 

def get_css_color(pct, metric):
    if metric == 'con':  #consistency
        pct = Decimal(str(math.fabs(pct)))
        if pct > Decimal('50'): return 'bad'
        elif pct > Decimal('25'): return 'warn'
        else: return 'good'
    elif metric == 'time':
        if pct > Decimal('.1'): return 'bad'
        else: return 'good'
    elif metric == 'com':
        if pct > Decimal('5'): return 'bad'
        else: return 'good'
    
def get_first(set):
    if isinstance(set, QuerySet) and len(set) > 0:
        return set[0]
    else: return None

def get_timeliness(timeliness, unit):
    if timeliness:
        try:
            pct = timeliness.late_dollars / timeliness.total_dollars
        except Exception:
            pct = 0
        if unit == 'pct':
            return (pct*100, get_css_color(pct, 'time'))
        else:
            return (timeliness.late_dollars, get_css_color(pct, 'time'))
    else:
        return (None,'')

def get_completeness(unit, **completeness):
    if completeness:
        try:
            pct = (completeness['failed_total']/ completeness['total']) * 100
        except Exception:
            pct = 0
        if unit == 'pct':
            return (pct, get_css_color(pct, 'com'))
        else:
            return (completeness['failed_total'], get_css_color(pct, 'com'))
    else:
        return (None, '')

def get_consistency(consistency, unit):
    
    if consistency:
        over = math.fabs(consistency.__dict__['over_reported_'+unit] or 0)
        under = math.fabs(consistency.__dict__['under_reported_'+unit] or 0)
        non = math.fabs(consistency.__dict__['non_reported_'+unit] or 0)
        return (over, get_css_color(consistency.over_reported_pct or 0, 'con'), 
                under, get_css_color(consistency.under_reported_pct or 0, 'con'), 
                non, get_css_color(consistency.non_reported_pct or 0, 'con'))
    else:
        placeholders = []
        for i in range(0, 6):
            placeholders.append(None)
        return placeholders

def generic_program_table(programs, fiscal_year, unit):
    #builds a scorecard table for a generic list of programs
    table_data = []
    types = [None, "grants", "loans"]
    for p in programs:
        display_name = p.program_title
        program_number = p.program_number
         
        if p.__class__ == SearchResult:
            p = p.pk
            p_id = p
        else: 
            p_id = p.id
        consistency = ProgramConsistency.objects.filter(fiscal_year=fiscal_year, program=p)
        timeliness = ProgramTimeliness.objects.filter(fiscal_year=fiscal_year, program=p)
        completeness_totals = ProgramCompleteness.objects.filter(program=p, fiscal_year=fiscal_year).aggregate(failed_total=Sum('completeness_failed_dollars'), total=Sum('completeness_total_dollars'))
        if len(consistency) > 0:
            ob_type = "(%s)" % types[get_first(consistency).type]
        else: ob_type = ""

        #if len(display_name) > 35: display_name = "%s..." % display_name[0:32]
        row = [ program_number,
                p_id,
                "%s %s" % (display_name, ob_type),
                ]
        row.extend(get_consistency(get_first(consistency), unit))
        row.extend(get_timeliness(get_first(timeliness), unit))
        row.extend(get_completeness(unit, **completeness_totals))
        table_data.append(row)

        if len(consistency) > 1:
            for ob in consistency[1:]:
                row = [ program_number,
                        p_id,
                        "%s (%s)" % (display_name, types[ob.type]),
                        ]
                row.extend(get_consistency(ob, unit))
                row.extend(get_timeliness(get_first(timeliness), unit))
                row.extend(get_completeness(unit, **completeness_totals))
                table_data.append(row)

    return table_data

def index(request, unit='dollars', fiscal_year=2009):
    #get top level agency stats 
    consistency = AgencyConsistency.objects.filter(fiscal_year=fiscal_year).order_by('agency__name')
    timeliness = AgencyTimeliness.objects.filter(fiscal_year=fiscal_year).order_by('agency__name')
    agencies = Agency.objects.all().order_by('name')
    table_data = []
    for a in agencies:

        number_programs = Program.objects.filter(agency=a).count()
        display_name = a.name
      #  if len(display_name) > 35: display_name = "%s..." % display_name[0:35]
        a_data = [a.code,
                  number_programs, 
                  a.name,
                  display_name]

        a_data.extend(get_consistency(get_first(consistency.filter(agency=a)), unit))
        a_data.extend(get_timeliness(get_first(timeliness.filter(agency=a)), unit))
        completeness_totals = ProgramCompleteness.objects.filter(program__agency=a, fiscal_year=fiscal_year).aggregate(failed_total=Sum('completeness_failed_dollars'), total=Sum('completeness_total_dollars'))
        a_data.extend(get_completeness(unit, **completeness_totals))

        table_data.append(a_data) 

    return render_to_response('scorecard_index.html', {'table_data': table_data, 'fiscal_year': "%s" % fiscal_year, 'unit':unit})

def agencyDetail(request, agency_id, unit='dollars', fiscal_year=2009):
    
    summary_numbers = []
    summary_numbers.append(AgencyConsistency.objects.filter(agency=agency_id, fiscal_year=fiscal_year).aggregate(total=Sum('total_cfda_obligations'))['total'])
    summary_numbers.extend(get_consistency(get_first(AgencyConsistency.objects.filter(agency=agency_id, fiscal_year=fiscal_year)), 'dollars'))
    summary_numbers.extend(get_timeliness(get_first(AgencyTimeliness.objects.filter(agency=agency_id, fiscal_year=fiscal_year)), 'dollars'))
    completeness_totals = ProgramCompleteness.objects.filter(program__agency=agency_id, fiscal_year=fiscal_year).aggregate(failed_total=Sum('completeness_failed_dollars'), total=Sum('completeness_total_dollars'))
    summary_numbers.extend(get_completeness('dollars', **completeness_totals)) 
    summary_numbers = filter( lambda x: not isinstance(x, basestring) , summary_numbers) #remove css colors

    programs = Program.objects.filter(agency=agency_id)
    agency = Agency.objects.get(code=agency_id)
    table_data = generic_program_table(programs, fiscal_year, unit)

    return render_to_response('agency_detail.html', {'summary_numbers': summary_numbers, 'table_data': table_data, 'fiscal_year': fiscal_year, 'unit': unit, 'agency_name': agency.name, 'agency': agency_id, 'description': agency.description})


def programDetail(request, program_id, unit):
    program = Program.objects.get(id=program_id)
    
    program_total_number = ProgramObligation.objects.filter(program=program).aggregate(total=Sum('obligation'))['total']
    program_total = moneyfmt(Decimal(str(program_total_number).replace('None', '0')), places=0, curr='$', sep=',', dp='')

    consistency_block = programDetailConsistency(program, unit) 
    field_names = ['late_'+unit, 'avg_lag_rows']
    proper_names = ['Late', 'Average Late Records']
    coll = ProgramTimeliness.objects.filter(program=program_id).order_by('fiscal_year')
    timeliness_block = programDetailGeneral(program_id, unit, field_names, proper_names, coll, 'Timeliness')

    if len(program.objectives) < 1300:
        description = program.objectives
    else: description = program.objectives[:1300] + '... <a href="http://cfda.gov">read more at CFDA.gov</a>.'

    #update these lists with all the fields we want to show
    com_field_names = ['recipient_type_is_not_empty', 'federal_agency_code_is_not_empty', 'cfda_program_num_is_descriptive', 'federal_funding_amount_is_not_empty',
                        'principal_place_code_not_empty', 'recipient_name_not_empty', 'recipient_state_code_not_empty', 'recipient_county_code_not_empty_or_too_long',
                        'recipient_county_name_not_empty', 'recipient_city_code_not_empty', 'recipient_city_name_not_empty', 'principal_place_state_not_empty', 'record_type_is_not_empty', 
                        'action_type_is_not_empty', 'recipient_cong_district_is_not_empty', 'obligation_action_date_is_properly_formatted', 
                        'assistance_type_is_not_empty', 'federal_award_id_is_not_empty']
    
    com_proper_names = ['Recipient type', 'Federal Agency', 'CFDA Program Number', 'Funding Amount',
                        'Principal Place of Grant Code',  'Recipient Name', 'Recipient State', 'Recipient County Code', 
                        'Recipient County Name', 'Recipient City Code', 'Recipient City Name', 'Principal State of Grant', 'Record Type', 
                        'Action Type', 'Recipient Congressional District', 'Obligation Action Date Formatted Correctly', 
                        'Assistance Type', 'Federal Award ID' ]
    com_coll = ProgramCompletenessDetail.objects.filter(program=program_id).order_by('fiscal_year')
    completeness_block = programDetailGeneral(program_id, unit, com_field_names, com_proper_names, com_coll, 'Completeness')
    
    return render_to_response('program_detail.html', {'consistency':consistency_block, 'timeliness': timeliness_block, 'completeness': completeness_block, 'agency_name': program.agency.name, 'program_number': program.program_number, 'title': program.program_title, 'desc': description, 'unit': unit, 'program_total': program_total}) 
    
{'fraction_non_alphanumeric': 0.11475179299676558, 'fraction_numerical': 0.0066094782730980175, 'mean_word_length': 3.6898219388876674, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1628133', 'n_tokens_mistral': 7011, 'n_tokens_neox': 6550, 'n_words': 1247}