<commit_before>from dataviva.stats.util import parse_year
from sqlalchemy import desc
from dataviva import __year_range__

from dataviva.attrs.models import  University, Course_hedu, Course_sc, Stat, Bs
from dataviva.attrs.models import Yb, Ybs, Bra, Hs, Cbo, Cnae, Wld
from dataviva.secex.models import Ymbp, Ymbw, Ympw, Ymb, Ymp, Ymw
from dataviva.rais.models import Ybi, Ybo, Yio, Yb_rais, Yi, Yo
from dataviva.hedu.models import Yu, Yuc, Ybu, Yc_hedu
from dataviva.sc.models import Ybc_sc, Yc_sc

from flask.ext.babel import gettext


def bra_stats(pobj, rais_year, secex_year):
    stats = []
    group = "General Stats"
    stat_ids = ["pop", "gini", "life_exp", "hdi"]
    stats_year = parse_year(__year_range__["stats"][-1].split("-")[0])
    results = batch_stats(Ybs, pobj.id, stat_ids, stats_year)
    for name, val in results.items():
        stats.append(make_stat(group, name, desc=val, year=stats_year))

    group = '{} Stats ({}):'.format(rais_year, "RAIS")
    filters = [Ybi.year == rais_year, Ybi.bra_id == pobj.id, Ybi.cnae_id_len == 6]
    result = get_top_stat(Ybi, Ybi.cnae_id, Ybi.num_emp, Cnae, filters)
    if result:
        profile, value = result
        stat = make_stat(group, gettext("Top Industry by Employment"), profile=profile, value=value, mode="num_emp")
        stats.append(stat)

    filters = [Ybo.year == rais_year, Ybo.bra_id == pobj.id, Ybo.cbo_id_len == 4]
    profile, value = get_top_stat(Ybo, Ybo.cbo_id, Ybo.num_emp, Cbo, filters)
    stat = make_stat(group, gettext("Top Occupation by Employment"), profile=profile, value=value, mode="num_emp")
    stats.append(stat)

    value = get_stat_val(Yb_rais, Yb_rais.wage, [Yb_rais.year == rais_year, Yb_rais.bra_id == pobj.id])
    stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=value, mode="wage"))

    group = '{} Stats ({}):'.format(secex_year, "SECEX")
    filters = [Ymbp.year == secex_year, Ymbp.month == 0, Ymbp.bra_id == pobj.id, Ymbp.hs_id_len == 6]
    result = get_top_stat(Ymbp, Ymbp.hs_id, Ymbp.export_val, Hs, filters)
    if result:
        profile, value = result
        stat = make_stat(group, gettext("Top Product by Export Value"), profile=profile, value=value, mode="export_val")
        stats.append(stat)

    filters = [Ymbw.year == secex_year, Ymbw.month == 0, Ymbw.bra_id == pobj.id, Ymbw.wld_id_len == 5]
    result = get_top_stat(Ymbw, Ymbw.wld_id, Ymbw.export_val, Wld, filters)
    if result:
        profile,value=result
        stats.append(make_stat(group, gettext("Top Destination by Export Value"), profile=profile, value=value, mode="export_val"))
    
    filters = [Ymb.year == secex_year, Ymb.month == 0, Ymb.bra_id == pobj.id]
    result = get_stat_val(Ymb, [Ymb.export_val, Ymb.import_val], filters)
    if result:
        export_val, import_val = result
        stats.append(make_stat(group, gettext("Total Exports"), desc=export_val, mode="export_val"))
        stats.append(make_stat(group, gettext("Total Imports"), desc=import_val, mode="export_val"))
    
    if len(pobj.id) == 9:
        group = "General Stats"
        stat_ids = ["airport", "airport_dist", "seaport", "seaport_dist"]
        results = batch_stats(Bs, pobj.id, stat_ids)
        for name, val in results.items():
            stats.append(make_stat(group, name, desc=val))
    return stats

def cnae_stats(pobj, rais_year): 
    stats = []   
    five_years_ago = rais_year - 5
    group = '{} Stats ({}):'.format(rais_year, "RAIS")
    
    filters = [Ybi.year == rais_year, Ybi.cnae_id == pobj.id, Ybi.bra_id_len == 9]
    profile, value = get_top_stat(Ybi, Ybi.bra_id, Ybi.num_emp, Bra, filters)
    stats.append(make_stat(group, gettext("Top Municipality by Employment"), profile=profile, value=value, mode="num_emp"))

    filters = [Yio.year == rais_year, Yio.cnae_id == pobj.id, Yio.cbo_id_len == 4]
    profile, value = get_top_stat(Yio, Yio.cbo_id, Yio.num_emp, Cbo, filters)
    stats.append(make_stat(group, gettext("Top Occupation by Employment"), profile=profile, value=value, mode="num_emp"))

    filters = [Yi.year == rais_year, Yi.cnae_id == pobj.id]
    wage, wage_avg = get_stat_val(Yi, [Yi.wage, Yi.wage_avg], filters)
    stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=wage, mode="wage"))
    stats.append(make_stat(group, gettext("Average Monthly Wage"), desc=wage_avg, mode="wage"))

    group = '{} Stats ({}):'.format(five_years_ago, "RAIS")
    wage, wage_avg = get_stat_val(Yi, [Yi.wage, Yi.wage_avg], filters)
    filters = [Yi.year == five_years_ago, Yi.cnae_id == pobj.id]
    wage, wage_avg = get_stat_val(Yi, [Yi.wage, Yi.wage_avg], filters)
    stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=wage, mode="wage"))
    stats.append(make_stat(group, gettext("Average Monthly Wage"), desc=wage_avg, mode="wage"))
    return stats

def cbo_stats(pobj, rais_year):
    stats = []
    five_years_ago = rais_year - 5
    group = '{} Stats ({}):'.format(rais_year, "RAIS")

    filters = [Ybo.year == rais_year, Ybo.cbo_id == pobj.id, Ybo.bra_id_len == 9]
    profile, value = get_top_stat(Ybo, Ybo.bra_id, Ybo.num_emp, Bra, filters)
    stats.append(make_stat(group, gettext("Top Municipality by Employment"), profile=profile, value=value, mode="num_emp"))

    filters = [Yio.year == rais_year, Yio.cbo_id == pobj.id, Yio.cnae_id_len == 6]
    profile, value = get_top_stat(Yio, Yio.cnae_id, Yio.num_emp, Cnae, filters)
    stats.append(make_stat(group, gettext("Top Occupation by Employment"), profile=profile, value=value, mode="num_emp"))

    filters = [Yo.year == rais_year, Yo.cbo_id == pobj.id]
    res = get_stat_val(Yo, [Yo.wage, Yo.wage_avg], filters)
    if res:
        wage, wage_avg = res
        stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=wage, mode="wage"))
        stats.append(make_stat(group, gettext("Average Monthly Wage"), desc=wage_avg, mode="wage"))

    group = '{} Stats ({}):'.format(five_years_ago, "RAIS")
    filters = [Yo.year == five_years_ago, Yo.cbo_id == pobj.id]
    res = get_stat_val(Yo, [Yo.wage, Yo.wage_avg], filters)
    if res:
        wage, wage_avg = res
        stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=wage, mode="wage"))
        stats.append(make_stat(group, gettext("Average Monthly Wage"), desc=wage_avg, mode="wage"))
    return stats


def hs_stats(pobj, secex_year):
    stats =[]
    group = '{} Stats ({}):'.format(rais_year, "SECEX")<commit_msg>stat bugfix for secex year<commit_after>from dataviva.stats.util import parse_year
from sqlalchemy import desc
from dataviva import __year_range__

from dataviva.attrs.models import  University, Course_hedu, Course_sc, Stat, Bs
from dataviva.attrs.models import Yb, Ybs, Bra, Hs, Cbo, Cnae, Wld
from dataviva.secex.models import Ymbp, Ymbw, Ympw, Ymb, Ymp, Ymw
from dataviva.rais.models import Ybi, Ybo, Yio, Yb_rais, Yi, Yo
from dataviva.hedu.models import Yu, Yuc, Ybu, Yc_hedu
from dataviva.sc.models import Ybc_sc, Yc_sc

from flask.ext.babel import gettext


def bra_stats(pobj, rais_year, secex_year):
    stats = []
    group = "General Stats"
    stat_ids = ["pop", "gini", "life_exp", "hdi"]
    stats_year = parse_year(__year_range__["stats"][-1].split("-")[0])
    results = batch_stats(Ybs, pobj.id, stat_ids, stats_year)
    for name, val in results.items():
        stats.append(make_stat(group, name, desc=val, year=stats_year))

    group = '{} Stats ({}):'.format(rais_year, "RAIS")
    filters = [Ybi.year == rais_year, Ybi.bra_id == pobj.id, Ybi.cnae_id_len == 6]
    result = get_top_stat(Ybi, Ybi.cnae_id, Ybi.num_emp, Cnae, filters)
    if result:
        profile, value = result
        stat = make_stat(group, gettext("Top Industry by Employment"), profile=profile, value=value, mode="num_emp")
        stats.append(stat)

    filters = [Ybo.year == rais_year, Ybo.bra_id == pobj.id, Ybo.cbo_id_len == 4]
    profile, value = get_top_stat(Ybo, Ybo.cbo_id, Ybo.num_emp, Cbo, filters)
    stat = make_stat(group, gettext("Top Occupation by Employment"), profile=profile, value=value, mode="num_emp")
    stats.append(stat)

    value = get_stat_val(Yb_rais, Yb_rais.wage, [Yb_rais.year == rais_year, Yb_rais.bra_id == pobj.id])
    stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=value, mode="wage"))

    group = '{} Stats ({}):'.format(secex_year, "SECEX")
    filters = [Ymbp.year == secex_year, Ymbp.month == 0, Ymbp.bra_id == pobj.id, Ymbp.hs_id_len == 6]
    result = get_top_stat(Ymbp, Ymbp.hs_id, Ymbp.export_val, Hs, filters)
    if result:
        profile, value = result
        stat = make_stat(group, gettext("Top Product by Export Value"), profile=profile, value=value, mode="export_val")
        stats.append(stat)

    filters = [Ymbw.year == secex_year, Ymbw.month == 0, Ymbw.bra_id == pobj.id, Ymbw.wld_id_len == 5]
    result = get_top_stat(Ymbw, Ymbw.wld_id, Ymbw.export_val, Wld, filters)
    if result:
        profile,value=result
        stats.append(make_stat(group, gettext("Top Destination by Export Value"), profile=profile, value=value, mode="export_val"))
    
    filters = [Ymb.year == secex_year, Ymb.month == 0, Ymb.bra_id == pobj.id]
    result = get_stat_val(Ymb, [Ymb.export_val, Ymb.import_val], filters)
    if result:
        export_val, import_val = result
        stats.append(make_stat(group, gettext("Total Exports"), desc=export_val, mode="export_val"))
        stats.append(make_stat(group, gettext("Total Imports"), desc=import_val, mode="export_val"))
    
    if len(pobj.id) == 9:
        group = "General Stats"
        stat_ids = ["airport", "airport_dist", "seaport", "seaport_dist"]
        results = batch_stats(Bs, pobj.id, stat_ids)
        for name, val in results.items():
            stats.append(make_stat(group, name, desc=val))
    return stats

def cnae_stats(pobj, rais_year): 
    stats = []   
    five_years_ago = rais_year - 5
    group = '{} Stats ({}):'.format(rais_year, "RAIS")
    
    filters = [Ybi.year == rais_year, Ybi.cnae_id == pobj.id, Ybi.bra_id_len == 9]
    profile, value = get_top_stat(Ybi, Ybi.bra_id, Ybi.num_emp, Bra, filters)
    stats.append(make_stat(group, gettext("Top Municipality by Employment"), profile=profile, value=value, mode="num_emp"))

    filters = [Yio.year == rais_year, Yio.cnae_id == pobj.id, Yio.cbo_id_len == 4]
    profile, value = get_top_stat(Yio, Yio.cbo_id, Yio.num_emp, Cbo, filters)
    stats.append(make_stat(group, gettext("Top Occupation by Employment"), profile=profile, value=value, mode="num_emp"))

    filters = [Yi.year == rais_year, Yi.cnae_id == pobj.id]
    wage, wage_avg = get_stat_val(Yi, [Yi.wage, Yi.wage_avg], filters)
    stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=wage, mode="wage"))
    stats.append(make_stat(group, gettext("Average Monthly Wage"), desc=wage_avg, mode="wage"))

    group = '{} Stats ({}):'.format(five_years_ago, "RAIS")
    wage, wage_avg = get_stat_val(Yi, [Yi.wage, Yi.wage_avg], filters)
    filters = [Yi.year == five_years_ago, Yi.cnae_id == pobj.id]
    wage, wage_avg = get_stat_val(Yi, [Yi.wage, Yi.wage_avg], filters)
    stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=wage, mode="wage"))
    stats.append(make_stat(group, gettext("Average Monthly Wage"), desc=wage_avg, mode="wage"))
    return stats

def cbo_stats(pobj, rais_year):
    stats = []
    five_years_ago = rais_year - 5
    group = '{} Stats ({}):'.format(rais_year, "RAIS")

    filters = [Ybo.year == rais_year, Ybo.cbo_id == pobj.id, Ybo.bra_id_len == 9]
    profile, value = get_top_stat(Ybo, Ybo.bra_id, Ybo.num_emp, Bra, filters)
    stats.append(make_stat(group, gettext("Top Municipality by Employment"), profile=profile, value=value, mode="num_emp"))

    filters = [Yio.year == rais_year, Yio.cbo_id == pobj.id, Yio.cnae_id_len == 6]
    profile, value = get_top_stat(Yio, Yio.cnae_id, Yio.num_emp, Cnae, filters)
    stats.append(make_stat(group, gettext("Top Occupation by Employment"), profile=profile, value=value, mode="num_emp"))

    filters = [Yo.year == rais_year, Yo.cbo_id == pobj.id]
    res = get_stat_val(Yo, [Yo.wage, Yo.wage_avg], filters)
    if res:
        wage, wage_avg = res
        stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=wage, mode="wage"))
        stats.append(make_stat(group, gettext("Average Monthly Wage"), desc=wage_avg, mode="wage"))

    group = '{} Stats ({}):'.format(five_years_ago, "RAIS")
    filters = [Yo.year == five_years_ago, Yo.cbo_id == pobj.id]
    res = get_stat_val(Yo, [Yo.wage, Yo.wage_avg], filters)
    if res:
        wage, wage_avg = res
        stats.append(make_stat(group, gettext("Total Monthly Wage"), desc=wage, mode="wage"))
        stats.append(make_stat(group, gettext("Average Monthly Wage"), desc=wage_avg, mode="wage"))
    return stats


def hs_stats(pobj, secex_year):
    stats =[]
    group = '{} Stats ({}):'.format(secex_year, "SECEX")
{'fraction_non_alphanumeric': 0.1504959702417855, 'fraction_numerical': 0.0024798512089274642, 'mean_word_length': 4.147586757080176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985579', 'n_tokens_mistral': 5012, 'n_tokens_neox': 4845, 'n_words': 1027}