<commit_before>import pandas as pd
import numpy as np
import json as json


data = pd.read_csv('../../data/raw/orbf_benin.csv' , delimiter = ';')

indicators_names = pd.read_json('../../data/raw/pbf_indicatorstranslations.json')

## Keep only data with claimed and verified values
print('Length Complete Data :' + str(len(data)))
data = data[(np.isnan(data.indicator_claimed_value) == False) & (np.isnan(data.indicator_verified_value) == False)]
print('Length Data with claimed and verified complete :' + str(len(data)))

## Formatting facilities names for simplicity
data.entity_name[data.entity_name == 'CNHU-PPC (Centre National Hospitalier Universitaire Pneumo Phtysiologique de Cot CSC'] = "CNHU-PPC"
data.entity_name = data.entity_name.str.title()

## Format Date Variables
u = data['datafile_year'].astype(str) + '-' +  data['datafile_month'].astype(str)
data['date'] = pd.to_datetime(u)
data['period'] = data['date'].dt.to_period('M')

## Label Indicator names
indicators_list_names = indicators_names.loc[indicators_names.indicator_language == 'en' , ['indicator_id' , 'indicator_title']]
indicators_list_names.columns = ['indicator_id' , 'indicator_label']

print(len(data))
data = pd.merge(data , indicators_list_names)
print(len(data))


to_export = ['entity_id' , 'entity_name' , 'entity_type' , 'parent_geozone_name' , 'geozone_name' , 'entity_fosa_id' ,  'content' , 'entity_status' , 'entity_active' , 'filetype_name' , 'filetype_id' , 'datafile_total' , 'datafile_author_id' , 'indicator_id' , 'indicator_label' , 'indicator_claimed_value' , 'indicator_verified_value' ,   'indicator_tarif' , 'indicator_montant' , 'entity_pop' , 'entity_pop_year' , 'geozone_pop' , 'geozone_pop_year' , 'parent_geozone_pop' , 'parent_geozone_pop_year' , 'period' , 'date']

data_out = data[to_export]<commit_msg>Correcting indicators name encoding<commit_after>import pandas as pd
import numpy as np
import json as json


data = pd.read_csv('../../data/raw/orbf_benin.csv' , delimiter = ';')

import codecs
indicators_names = pd.read_json(codecs.open('../../data/raw/pbf_indicatorstranslations.json' , encoding = 'utf-8'))

## Keep only data with claimed and verified values
print('Length Complete Data :' + str(len(data)))
data = data[(np.isnan(data.indicator_claimed_value) == False) & (np.isnan(data.indicator_verified_value) == False)]
print('Length Data with claimed and verified complete :' + str(len(data)))

## Formatting facilities names for simplicity
data.entity_name[data.entity_name == 'CNHU-PPC (Centre National Hospitalier Universitaire Pneumo Phtysiologique de Cot CSC'] = "CNHU-PPC"
data.entity_name = data.entity_name.str.title()




## Format Date Variables
u = data['datafile_year'].astype(str) + '-' +  data['datafile_month'].astype(str)
data['date'] = pd.to_datetime(u)
data['period'] = data['date'].dt.to_period('M')

## Label Indicator names
indicators_list_names = indicators_names.loc[indicators_names.indicator_language == 'en' , ['indicator_id' , 'indicator_title']]
indicators_list_names.columns = ['indicator_id' , 'indicator_label']

print(len(data))
data = pd.merge(data , indicators_list_names)
print(len(data))


to_export = ['entity_id' , 'entity_name' , 'entity_type' , 'parent_geozone_name' , 'geozone_name' , 'entity_fosa_id' ,  'content' , 'entity_status' , 'entity_active' , 'filetype_name' , 'filetype_id' , 'datafile_total' , 'datafile_author_id' , 'indicator_id' , 'indicator_label' , 'indicator_claimed_value' , 'indicator_verified_value' ,   'indicator_tarif' , 'indicator_montant' , 'entity_pop' , 'entity_pop_year' , 'geozone_pop' , 'geozone_pop_year' , 'parent_geozone_pop' , 'parent_geozone_pop_year' , 'period' , 'date']

data_out = data[to_export]
{'fraction_non_alphanumeric': 0.15085158150851583, 'fraction_numerical': 0.0002703433360367667, 'mean_word_length': 6.628865979381444, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3811985', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1304, 'n_words': 240}