<jupyter_start><jupyter_text>Import Modules<jupyter_code>import pandas as pd
import seaborn as sns
import pickle

from sklearn.preprocessing import StandardScaler, Imputer
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Load Dataset<jupyter_code>filepath = '/Users/tim/src/Metis/Project_3/data/interim/df_merge3.pkl'
with open(filepath, 'rb') as pkl:
    df = pickle.load(pkl)
df.info()
df.describe()<jupyter_output><empty_output><jupyter_text>Split training and test data<jupyter_code># Create feature matrix (X) and target vector (y)
y = df['coup']
X = df.drop(['year', 'coup', 'country'], axis=1)


# Split the data into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=13, stratify=y)
X_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 4110 entries, 4484 to 3754
Data columns (total 23 columns):
mil_career        4110 non-null int64
tenure            4110 non-null float64
exec_ant          4110 non-null int64
irreg_lead_ant    4110 non-null int64
exec_recent       4110 non-null int64
lead_recent       4110 non-null int64
prev_conflict     4110 non-null int64
nat_resources     3313 non-null float64
food_scarcity     1809 non-null float64
education         1736 non-null float64
healthcare        1899 non-null float64
reserves          1825 non-null float64
revenue           1980 non-null float64
electricity       3387 non-null float64
debt_forgiven     2135 non-null float64
life_exp          3250 non-null float64
mil_spending      2659 non-null float64
sanitation        2009 non-null float64
water             2006 non-null float64
tax               1991 non-null float64
military_pop      2904 non-null float64
rgdpe             3023 non-null float64
hc                2589[...]<jupyter_text>Impute with Mean<jupyter_code>imputer = Imputer(strategy='mean', axis=0)
imputer.fit(X_train)
train_mean = imputer.transform(X_train)

# filepath = '/Users/tim/src/Metis/Project_3/data/interim/train_mean.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(train_mean, pkl)
train_mean.shape
imputer = Imputer(strategy='mean', axis=0)
imputer.fit(X_train)
test_mean = imputer.transform(X_test)

# filepath = /Users/tim/src/Metis/Project_3/data/interim/train_mean.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(test_mean, pkl)
train_mean.shape<jupyter_output><empty_output><jupyter_text>Impute with Median<jupyter_code>imputer2 = Imputer(strategy='median', axis=0)
imputer2.fit(X_train)
train_median = imputer.transform(X_train)

# filepath = '/Users/tim/src/Metis/Project_3/data/interim/train_median.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(train_median, pkl)
train_median.shape<jupyter_output><empty_output><jupyter_text>Future Work to Finish  Impute with Mean by Country<jupyter_code># fill_mean = X_train.groupby('country').mean().reset_index()

# train_mean_grouped = impute_all_the_things(X_train, fill_mean)

# filepath = '/Users/tim/src/Metis/Project_3/data/interim/train_mean_grouped.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(train_mean_grouped, pkl)
#train_mean_grouped.info()<jupyter_output><empty_output><jupyter_text>Aggregate Similar Data<jupyter_code>def impute_all_the_things(df, fill_in):
    merged = pd.merge(df, fill_in, left_on='country', right_on='country')
    col_to_replace = [c for c in merged.columns if c.endswith('_x')]
    for col in col_to_replace:
        col_replace_with = col[:-2] + '_y'
        merged[col].fillna(merged[col_replace_with], inplace=True)
        merged.drop(col_replace_with, axis=1, inplace=True)
        merged.rename(columns={col: col[:-2]}, inplace=True)
    return merged
# df_by_country =  df.groupby(['country'])
# scaler = StandardScaler()

# # Development Index
# df_dev = df_by_country[['electricity', 'water', 'sanitation', 'life_exp']]
# dev_scaled = scaler.fit_transform(df_dev)
# full_dev = df_dev.notnull().sum(axis=1)
# df_dev.fillna(0, inplace=True)
# df_by_country['dev_index'] = (dev_scaled['electricity'] + dev_scaled['water'] + dev_scaled['sanitation'])/full_dev

# # Military Focus
# df_mil = df_by_country[['military_pop', 'mil_spending']]
# mil_scaled = scaler.fit_transform(df_mil)
# full_mil = df_mil.notnull().sum(axis=1)
# df_mil.fillna(0, inplace=True)
# df_by_country['mil_index'] = (mil_scaled['military_pop'] + mil_scaled['mil_spending'])/full_mil
# # Development Index
# df_dev = df[['country', 'year', 'electricity', 'water', 'sanitation']]
# full = df_dev.notnull().sum(axis=1) - 2
# df_dev.fillna(0, inplace=True)
# df['dev_index'] = (df_dev['electricity'] + df_dev['water'] + df_dev['sanitation'])/full

# # Government Money
# df_mon = df[['tax', 'revenue']]
# full_mon = df_mon.notnull().sum(axis=1)
# df_mon.fillna(0, inplace=True)
# df['mon_index'] = (df_mon['tax'] + df_mon['revenue'])/full_mon<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1294524730307348, 'fraction_numerical': 0.03643395074292693, 'mean_word_length': 5.059186189889026, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '273314', 'license': 'MIT', 'n_tokens_mistral': 1949, 'n_tokens_neox': 1833, 'n_words': 341, 'path': 'Notebooks/2.Missing-Data.ipynb', 'repo_name': 'broco347/coup-predictor', 'size': '4913'}