<commit_before># -*- coding: utf-8 -*-

import pandas as pd
import numpy as np
import os
from collections import OrderedDict


def to_dict_dropna(data):
    """return a dictionary that do not contain any NaN values from a dataframe."""
    return dict((k, v.dropna().to_dict()) for k, v in data.iterrows())


def generate_metadata(ddf_path, outpath):

    try:
        ddf_concept = pd.read_csv(os.path.join(ddf_path, 'ddf--concepts.csv'))
    except:
        ddf_concept = pd.read_csv(os.path.join(ddf_path, 'ddf--concepts--continous.csv'))

    # use OrderedDict in order to keep the order of insertion.
    indb = OrderedDict([['indicatorsDB', OrderedDict()]])

    # TODO:
    # 1. geo concepts?
    # 2. oneset?

    # TODO: see if below columns are engough
    measure_cols = ['concept', 'sourceLink', 'scales', 'interpolation', 'color']

    mdata = ddf_concept[ddf_concept['concept_type'] == 'measure'][measure_cols]
    mdata = mdata.set_index('concept')
    mdata = mdata.drop(['longitude', 'latitude'])
    mdata.columns = ['sourceLink', 'scales', 'interpolation', 'color']
    mdata['use'] = 'indicator'

    mdata_dict = to_dict_dropna(mdata)
    for k in sorted(mdata_dict.keys()):
        indb['indicatorsDB'][k] = mdata_dict.get(k)

    for i in indb['indicatorsDB'].keys():
        fname = os.path.join(ddf_path, 'ddf--datapoints--'+i+'--by--geo--time.csv')
        try:
            df = pd.read_csv(fname, dtype={i: float, 'time': int})
        except (OSError, IOError):
            print('no datapoints for ', i)
            continue

        # domain and availability
        dm = [float(df[i].min()), float(df[i].max())]
        av = [int(df['time'].min()), int(df['time'].max())]

        # domain_quantiles_10_90:
        # 1) sort by indicator value
        # 2) remove top and bottom 10% of values (os if 100 points, remove 10 from top and bottom)
        # 3) take first and last value of what's left as min and max in the property above.
        values_sorted = df[i].sort_values().values
        q_10 = np.round(len(values_sorted) / 10)
        q_90 = -1 * q_10 - 1

        # values_sorted = values_sorted[q_10:q_90]
        # domain_quantiles_10_90 = [values_sorted.min(), values_sorted.max()]

        domain_quantiles_10_90 = [values_sorted[q_10], values_sorted[q_90]]

        indb['indicatorsDB'][i].update({
            'domain': dm, 'availability': av,
            'domain_quantiles_10_90': domain_quantiles_10_90
        })

    return indb['indicatorsDB']<commit_msg>no need for indicatorsTree part<commit_after># -*- coding: utf-8 -*-

import pandas as pd
import numpy as np
import os
from collections import OrderedDict


def to_dict_dropna(data):
    """return a dictionary that do not contain any NaN values from a dataframe."""
    return dict((k, v.dropna().to_dict()) for k, v in data.iterrows())


def generate_metadata(ddf_path, outpath):

    try:
        ddf_concept = pd.read_csv(os.path.join(ddf_path, 'ddf--concepts.csv'))
    except:
        ddf_concept = pd.read_csv(os.path.join(ddf_path, 'ddf--concepts--continuous.csv'))

    # use OrderedDict in order to keep the order of insertion.
    indb = OrderedDict()

    # TODO:
    # 1. geo concepts?
    # 2. oneset?

    # FIXME: see if below columns are engough
    measure_cols = ['concept', 'sourceLink', 'scales', 'interpolation', 'color']

    mdata = ddf_concept[ddf_concept['concept_type'] == 'measure'][measure_cols]
    mdata = mdata.set_index('concept')
    mdata = mdata.drop(['longitude', 'latitude'])
    mdata.columns = ['sourceLink', 'scales', 'interpolation', 'color']
    mdata['use'] = 'indicator'

    mdata_dict = to_dict_dropna(mdata)
    for k in sorted(mdata_dict.keys()):
        indb[k] = mdata_dict.get(k)

    for i in indb.keys():
        fname = os.path.join(ddf_path, 'ddf--datapoints--'+i+'--by--geo--time.csv')
        try:
            df = pd.read_csv(fname, dtype={i: float, 'time': int})
        except (OSError, IOError):
            print('no datapoints for ', i)
            continue

        # domain and availability
        dm = [float(df[i].min()), float(df[i].max())]
        av = [int(df['time'].min()), int(df['time'].max())]

        # domain_quantiles_10_90:
        # 1) sort by indicator value
        # 2) remove top and bottom 10% of values (os if 100 points, remove 10 from top and bottom)
        # 3) take first and last value of what's left as min and max in the property above.
        values_sorted = df[i].sort_values().values
        q_10 = np.round(len(values_sorted) / 10)
        q_90 = -1 * q_10 - 1

        # values_sorted = values_sorted[q_10:q_90]
        # domain_quantiles_10_90 = [values_sorted.min(), values_sorted.max()]

        domain_quantiles_10_90 = [values_sorted[q_10], values_sorted[q_90]]

        indb[i].update({
            'domain': dm, 'availability': av,
            'domain_quantiles_10_90': domain_quantiles_10_90
        })

    return indb
{'fraction_non_alphanumeric': 0.14164133738601822, 'fraction_numerical': 0.02066869300911854, 'mean_word_length': 2.9050632911392404, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1837145', 'n_tokens_mistral': 1783, 'n_tokens_neox': 1662, 'n_words': 410}