<commit_before>import json
import yaml

from status.util import SafeHandler, UnsafeHandler

def get_default_stats_data(view):
    data={}
    for row in view:
        try:
            data[row.key[0]][row.key[1]]=row.value
        except (TypeError, KeyError):
            data[row.key[0]]={}
            data[row.key[0]][row.key[1]]=row.value
    return data

def get_stats_data(db,view,gl=None):
    data={}
    if gl==2:
        view=db.view(view, group_level=gl)
        data=get_default_stats_data(view)
    elif gl==1:
        for row in db.view(view, group_level=gl):
            data[row.key]=row.value
    return data

def clean_application_keys_y(raw_data):
    # Normalise Application names to approved list
    # Don't let Denis see this, it will make him cry
    with open('application_keys.yaml') as f:
        app_cats=yaml.load(f)
    data={}
    for year in raw_data:
        data[year] = {}
        for c in raw_data[year]:
            try:
                data[year][app_cats[c]] += raw_data[year][c]
            except KeyError:
                try:
                    data[year][app_cats[c]] = raw_data[year][c]
                except KeyError:
                    data[year][c] = raw_data[year][c]
    return data

def clean_application_keys(raw_data):
    # Normalise Application names to approved list
    # Don't let Denis see this, it will make him cry
    with open('application_keys.yaml') as f:
        app_cats=yaml.load(f)
    data={}
    for c in raw_data:
        try:
            data[app_cats[c]] += raw_data[c]
        except KeyError:
            try:
                data[app_cats[c]] = raw_data[c]
            except KeyError:
                data[c] = raw_data[c]<commit_msg>Used nicer code as suggested by @senthil10<commit_after>import json
import yaml

from status.util import SafeHandler, UnsafeHandler

def get_default_stats_data(view):
    data={}
    for row in view:
        try:
            data[row.key[0]][row.key[1]]=row.value
        except (TypeError, KeyError):
            data[row.key[0]]={}
            data[row.key[0]][row.key[1]]=row.value
    return data

def get_stats_data(db,view,gl=None):
    data={}
    if gl==2:
        view=db.view(view, group_level=gl)
        data=get_default_stats_data(view)
    elif gl==1:
        for row in db.view(view, group_level=gl):
            data[row.key]=row.value
    return data

def clean_application_keys_y(raw_data):
    # Normalise Application names to approved list
    # Don't let Denis see this, it will make him cry
    with open('application_keys.yaml') as f:
        app_cats=yaml.load(f)
    data={}
    for year in raw_data:
        data[year] = {}
        for c in raw_data[year]:
            data[year][app_cats.get(c, c)] = data[year].get(app_cats.get(c, c),0) + raw_data[year][c]
    return data

def clean_application_keys(raw_data):
    # Normalise Application names to approved list
    # Don't let Denis see this, it will make him cry
    with open('application_keys.yaml') as f:
        app_cats=yaml.load(f)
    data={}
    for c in raw_data:
        data[app_cats.get(c, c)] = data.get(app_cats.get(c, c),0) + raw_data[c]
{'fraction_non_alphanumeric': 0.11781517942203874, 'fraction_numerical': 0.0057161003493172435, 'mean_word_length': 2.420195439739414, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5133381', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1041, 'n_words': 251}