<commit_before>import os
from dotenv import load_dotenv, find_dotenv
import get_kenya
import get_haiti

# find .env automagically by walking up directories until it's found
dotenv_path = find_dotenv()

# load up the entries as environment variables
load_dotenv(dotenv_path)

kenya_1_url = os.environ.get("kenya_1_url")
kenya_2_visits_url = os.environ.get("kenya_2_visits_url")
kenya_2_patients_url = os.environ.get("kenya_2_patients_url")
kenya_3_visits_url = os.environ.get("kenya_3_visits_url")
kenya_3_patients_url = os.environ.get("kenya_3_patients_url")

haiti_metadata_url = os.environ.get("haiti_metadata_url")

### Loading Kenya
visits_1 , patients_1 = get_kenya.read_access_data(kenya_1_url)
visits_2 , patients_2 = get_kenya.read_csv_data(kenya_2_visits_url , kenya_2_patients_url)
visits_3 , patients_3 = get_kenya.read_csv_data(kenya_3_visits_url , kenya_3_patients_url)

data_ken_1 = get_kenya.format_tables(visits_1 , patients_1 , 'ken_fac_1')
data_ken_2 = get_kenya.format_tables(visits_2 , patients_2 , 'ken_fac_2')
data_ken_3 = get_kenya.format_tables(visits_3 , patients_3 , 'ken_fac_3')

### Loading Haiti
data_haiti = get_haiti.read_metadata(haiti_metadata_url)


## Combining all the data
data_all = data_ken_1.append(data_ken_2).append(data_ken_3).append(data_haiti)
data_complete = data_all.dropna()

data_complete.to_csv('../../data/processed/complete_data.csv')

### TODO Check completeness of data exported from access to csv for Kenya facilities 2 and 3
### IDEA Only load visit data ?
### TODO for Haiti, get dict of form types
<commit_msg>removing index from exported output<commit_after>import os
from dotenv import load_dotenv, find_dotenv
import get_kenya
import get_haiti

# find .env automagically by walking up directories until it's found
dotenv_path = find_dotenv()

# load up the entries as environment variables
load_dotenv(dotenv_path)

kenya_1_url = os.environ.get("kenya_1_url")
kenya_2_visits_url = os.environ.get("kenya_2_visits_url")
kenya_2_patients_url = os.environ.get("kenya_2_patients_url")
kenya_3_visits_url = os.environ.get("kenya_3_visits_url")
kenya_3_patients_url = os.environ.get("kenya_3_patients_url")

haiti_metadata_url = os.environ.get("haiti_metadata_url")

### Loading Kenya
visits_1 , patients_1 = get_kenya.read_access_data(kenya_1_url)
visits_2 , patients_2 = get_kenya.read_csv_data(kenya_2_visits_url , kenya_2_patients_url)
visits_3 , patients_3 = get_kenya.read_csv_data(kenya_3_visits_url , kenya_3_patients_url)

data_ken_1 = get_kenya.format_tables(visits_1 , patients_1 , 'ken_fac_1')
data_ken_2 = get_kenya.format_tables(visits_2 , patients_2 , 'ken_fac_2')
data_ken_3 = get_kenya.format_tables(visits_3 , patients_3 , 'ken_fac_3')

### Loading Haiti
data_haiti = get_haiti.read_metadata(haiti_metadata_url)


## Combining all the data
data_all = data_ken_1.append(data_ken_2).append(data_ken_3).append(data_haiti)
data_complete = data_all.dropna()

data_complete.to_csv('../../data/processed/complete_data.csv' , index = False)

### TODO Check completeness of data exported from access to csv for Kenya facilities 2 and 3
### IDEA Only load visit data ?
### TODO for Haiti, get dict of form types

{'fraction_non_alphanumeric': 0.09325771896660366, 'fraction_numerical': 0.023944549464398234, 'mean_word_length': 7.120204603580563, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2232720', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1208, 'n_words': 226}