<jupyter_start><jupyter_text>Import helpers and packages<jupyter_code>import pandas as pd
from autoc import DataExploration, PreProcessor, NaImputer
from autoc.utils.getdata import get_dataset
import numpy as np

# skicit learn 
from sklearn.linear_model import LogisticRegression
from sklearn.cross_validation import cross_val_score,train_test_split
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.metrics import roc_curve, accuracy_score, auc, classification_report

# matplotlib
%matplotlib inline
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Playing with titanic data  Two approaches : * Simulate missing data and look if we can impute them correctly using machine learning algo* Impute the real ones and look if it is improving performance on the prediction<jupyter_code>titanic = get_dataset("titanic")
titanic.who.dtype.kind == 'O'
titanic.head()
exploration_titanic = DataExploration(titanic)
exploration_titanic.print_infos() # there is duplicates here because no id, interesting !
exploration_titanic.nacolcount()
exploration_titanic.structure()
titanic.corr()
titanic.loc[titanic.age.isnull(),:].head(5)<jupyter_output><empty_output><jupyter_text>Preprocessing data<jupyter_code>preprocessor = PreProcessor(titanic)
preprocessor.infer_subtypes()
titanic = titanic.drop('alive', axis = 1)<jupyter_output><empty_output><jupyter_text>Transform everything to numeric variables for skicit learn model  The dataset is not clean enough to be directly transformed as a numpy array and used for ml with skicit learn * The target variable (alive) is not a a integer dummy variable (0 or 1) * The values are heterogeneous (string labels for categories, integers and floating point numbers). Numpy array for skicit learn only supports integers and floating point numbers.* Some observations are missing (np.nan)<jupyter_code>features_full = pd.concat([titanic.loc[:, ['fare', 'age', 'pclass', 'sibsp', 'parch']],
                      pd.get_dummies(titanic['sex'], prefix='sex'),
                      pd.get_dummies(titanic['who'], prefix='who'),
                      pd.get_dummies(titanic['alone'], prefix='alone'),
                      pd.get_dummies(titanic['embarked'], prefix='embarked')],
                     axis=1)
features = pd.concat([titanic[['fare', 'age', 'pclass']],
                      pd.get_dummies(titanic['sex'], prefix='sex'),
                      pd.get_dummies(titanic['who'], prefix='who'),
                      pd.get_dummies(titanic['embarked'], prefix='embarked')],
                     axis=1)
target = titanic.survived<jupyter_output><empty_output><jupyter_text>Simple Model to understand variable importances   Prepare data<jupyter_code># Impute missing values 
imp = NaImputer(features_full)
features_full = imp.basic_naimputation(['age']) # this is still a pandas Dataframe but imputed 
target = titanic.survived
# Creating test train 
features_train, features_test, target_train, target_test = train_test_split(
    features_full.values, target.values, test_size=0.25, random_state=0)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>logreg = LogisticRegression(C=1)
logreg.fit(features_train, target_train)
target_pred = logreg.predict(features_test)


feature_names = features_full.columns
print("Accuracy : {}".format(accuracy_score(target_test, target_pred)))
weights = logreg.coef_.flatten()
dict_weights = {k:v for k,v in zip(feature_names, weights)}
def plot_simple_imp(imp, features_names,sort = True, absolute=False):
    serie = pd.Series(index=feature_names, data=imp)
    if absolute : 
        serie = np.abs(serie)
    if sort :
        serie.sort_values(inplace=True, ascending=False)
    serie.plot(kind='barh')
plot_simple_imp(weights, feature_names)
# Looking at weights 
feature_names = features_full.columns

def plot_abs_weights(coeff_arr, features_name, title=None,legend_size=12,figsize=(15,7)):
    coeff_arr = np.abs(coeff_arr)# take absolute value
    coeff_arr.sort()
    plt.figure(figsize=figsize)
    plt.barh(range(len(feature_names)), coeff_arr)
    plt.yticks(range(len(feature_names)), feature_names, size=legend_size)
    if title:
        plt.title(title)

plot_abs_weights(logreg.coef_.ravel(), feature_names, title="Absolute Coefficient Logistic Regression")<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>rf_full = RandomForestClassifier(n_estimators=500)
rf_full.fit(features_train, target_train)
rf_full.score(features_test, target_test)
plot_simple_imp(rf_full.feature_importances_, feature_names)<jupyter_output><empty_output><jupyter_text>Studying simple imputation technique for accuracy performance  Here we are trying to predict who survives using with the variable age having natural missing values  * Deletion of row containing missing values * Deletion of col containig missing values * Imputation of missing values with median * Imputation of missing values with -1<jupyter_code>def rf_cv(features, target,random_state=1, n_estimators=200,scoring='accuracy',n_jobs=4, verbose=True):
    """ Print scores of a random forest cross validation  """
    rf = RandomForestClassifier(n_estimators=n_estimators, random_state=random_state)
    scores = cross_val_score(rf, features, target, cv=4,scoring=scoring,n_jobs=4)
    if verbose :
        print("Random Forest CV scores:min: {:.3f}, mean: {:.3f}, max: {:.3f}".format(
                scores.min(), scores.mean(), scores.max()))
    return scores

def logreg_cv(features, target, scoring='accuracy',n_jobs=4, verbose=True):
    """ Print scores of a  forest cross validation  """
    logreg = LogisticRegression(C=1)
    scores = cross_val_score(logreg, features, target, cv=4,scoring=scoring,n_jobs=4)
    if verbose : 
        print("Logistic Regression CV scores: min: {:.3f}, mean: {:.3f}, max: {:.3f}".format(
                scores.min(), scores.mean(), scores.max()))
    return scores
    

def plot_roc_curve(target_test, target_predicted_proba):
    fpr, tpr, thresholds = roc_curve(target_test, target_predicted_proba[:, 1])
    
    roc_auc = auc(fpr, tpr)
    # Plot ROC curve
    plt.plot(fpr, tpr, label='ROC curve (AUC = %0.3f)' % roc_auc)
    plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.0])
    plt.xlabel('False Positive Rate or (1 - Specifity)')
    plt.ylabel('True Positive Rate or (Sensitivity)')
    plt.title('Receiver Operating Characteristic')
    plt.legend(loc="lower right")<jupyter_output><empty_output><jupyter_text>Row Deletion strategy<jupyter_code># selecting index and transforming into numpy array 
index_missing_age = features.age.isnull()
features_rm_a, target_rm_a = features.loc[~index_missing_age, :].values, target[~index_missing_age].values
features_rm_a.shape
rf_cv(features_rm_a, target_rm_a, scoring='accuracy')
features_train_rm, features_test_rm, target_train_rm, target_test_rm = train_test_split(
    features_rm_a, target_rm_a, test_size=0.25, random_state=0)
rf = RandomForestClassifier(n_estimators=200)
rf.fit(features_train_rm, target_train_rm)
target_predicted_proba = rf.predict_proba(features_test_rm)
plot_roc_curve(target_test_rm, target_predicted_proba)
rf_cv(features_rm_a, target_rm_a,random_state=0, scoring='roc_auc')<jupyter_output>Random Forest CV scores:min: 0.812, mean: 0.849, max: 0.865<jupyter_text>Col Deletion strategy<jupyter_code># selecting index and transforming into numpy array 
features_cm_a, target_cm_a = features.drop('age', axis =1).values, target.values
features_cm_a.shape
rf_cv(features_cm_a, target_cm_a, scoring="accuracy")
features_train_cm, features_test_cm, target_train_cm, target_test_cm = train_test_split(
    features_cm_a, target_cm_a, test_size=0.25, random_state=0)
rf = RandomForestClassifier(n_estimators=200)
rf.fit(features_train_cm, target_train_cm)
target_predicted_proba = rf.predict_proba(features_test_cm)
plot_roc_curve(target_test_cm, target_predicted_proba)
rf_cv(features_cm_a, target_cm_a, scoring="roc_auc")<jupyter_output>Random Forest CV scores:min: 0.805, mean: 0.850, max: 0.880<jupyter_text>Median Imputation strategy<jupyter_code># selecting index and transforming into numpy array 

features_imp = features.copy()
features.shape
# features_imp.loc[:,'is_na_age'] =features_imp.age.isnull().astype(int)
# imp = NaImputer(features) # creating our imputer instance
# features_imp = imp.basic_naimputation(columns_to_process=['age'])
features_imp = features.fillna(-1)
features_imp_a, target_imp_a = features_imp.values, target.values
rf_cv(features_imp_a, target_imp_a, scoring='accuracy')
features_train_imp, features_test_imp, target_train_imp, target_test_imp = train_test_split(
    features_imp_a, target_imp_a, test_size=0.25, random_state=0)
rf = RandomForestClassifier(n_estimators=200)
rf.fit(features_train_imp, target_train_imp)
target_predicted_proba = rf.predict_proba(features_test_imp)
plot_roc_curve(target_test_imp, target_predicted_proba)
rf_cv(features_imp_a, target_imp_a, scoring='roc_auc')
rf.feature_importances_<jupyter_output><empty_output><jupyter_text>Simulating missing values in a column and see imputation performance   The purpose of this section is to simulate missing values in a important variable like .. and see the decrease of performance and the ability to correcly impute the missing value * We start using the features_full dataset<jupyter_code># constructing features 
features_imp = pd.concat([titanic[['pclass']],
                      pd.get_dummies(titanic['sex'], prefix='sex'),
                      pd.get_dummies(titanic['who'], prefix='who')],axis=1)
features_imp.pclass.value_counts()<jupyter_output><empty_output><jupyter_text>Comparing simulated model and raw model<jupyter_code>#scores_imp = logreg_cv(features_imp.drop('pclass',1), target)
def insert_na(features_full=features_imp,target=target,index=False,
              col_to_simulate='pclass', pct_na_toinsert=0.2, verbose=False):
    """ Returns dataset with a certain pct of na injected in one colum """
    nb_na_toinsert = int(pct_na_toinsert * len(features_full))
    index_na_toinsert = np.random.choice(range(len(features_full)),nb_na_toinsert, replace=False)
    if verbose:
        print("We are inserting {} missing values".format(len(index_na_toinsert)))
    features_full_imp = features_full.copy()
    if index :
        return index_na_toinsert
    else:
        features_full_imp.loc[index_na_toinsert, col_to_simulate] = np.nan
        return features_full_imp

def score_rf_sim(features_full=features_imp,target=target,
                 col_to_simulate='pclass', pct_na_toinsert=0.2, n_repeat=10,verbose=False, *args, **kwargs):
    """ Inserting a percentage of missing values on a variable and look influence on performance 
    with a random forest model """
    features_full_imp = insert_na(features_full,target=target,
                                  col_to_simulate=col_to_simulate, pct_na_toinsert=pct_na_toinsert,verbose=verbose)
    imp_f = NaImputer(features_full_imp)
    features_full_imp.loc[:,col_to_simulate] = imp_f.fillna_serie(colname=col_to_simulate)
    # repeated cross validation 
#     score_rcv = 0
#     for i in range(n_repeat):
#         score_rcv += logreg_cv(features_full_imp, target,*args, **kwargs).mean()
    return logreg_cv(features_full_imp, target, verbose=False).mean()
score_rf_sim(col_to_simulate='pclass', verbose=True)
accuracy_mean_pct_na = np.array([score_rf_sim(
            pct_na_toinsert=i,col_to_simulate='pclass',verbose=True) for i in np.linspace(0,0.98,10)])
def sim_nmc(nmc=60,n_interval=5, *args, **kwargs):
    
    res = np.zeros(n_interval)
    for i in range(nmc):
        res +=  np.array([score_rf_sim(
            pct_na_toinsert=i, *args, **kwargs) for i in np.linspace(0,0.98,n_interval)])
    return res/nmc

test = sim_nmc(nmc=30, n_interval=5)
test
np.linspace(0,0.98,5)
plt.plot(np.linspace(0,0.98,5), test)
plt.title('Accuracy function of percentage of missing values inserted')<jupyter_output><empty_output><jupyter_text>Trying to predict missing values We start with all features to have better prediciton power<jupyter_code>features_pred = features_full.copy().drop_duplicates()
features_pred = features_pred.drop('age', axis = 1)
index_na = insert_na(col_to_simulate='pclass', pct_na_toinsert=0.2, index=True)
index_na = features_pred.index.isin(index_na)
features_pred.head()
target = features_pred.pclass
features_pred = features_pred.drop('pclass', axis = 1)
features_pred_train, target_pred_target = features_pred.loc[~index_na,:], target[~index_na]
features_pred_test, target_pred_test = features_pred.loc[index_na,:], target[index_na]

features_pred_train.shape
features_pred_test.head()
rf = RandomForestClassifier(n_estimators=200)
rf.fit(features_pred_train, target_pred_target)
target_predicted = rf.predict(features_pred_test)
target_predicted_proba = rf.predict_proba(features_pred_test)

print(classification_report(target_pred_test, target_predicted))<jupyter_output>precision    recall  f1-score   support

          1       0.97      0.92      0.94        37
          2       0.85      0.82      0.84        28
          3       0.93      0.96      0.94        79

avg / total       0.92      0.92      0.92       144
{'fraction_non_alphanumeric': 0.10476764329291255, 'fraction_numerical': 0.014461098139639979, 'mean_word_length': 5.386724386724387, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '204759', 'license': 'MIT', 'n_tokens_mistral': 4585, 'n_tokens_neox': 4419, 'n_words': 915, 'path': 'examples/Influence of imputation of missing values on prediction.ipynb', 'repo_name': 'ericfourrier/auto-clean', 'size': '13277'}