<jupyter_start><jupyter_text>Setup<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
from IPython.display import display, Markdown

from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier, RandomForestRegressor
from sklearn.model_selection import cross_val_score, GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix
from sklearn import svm

import scipy.cluster.hierarchy as hac

%matplotlib inline
%config IPCompleter.greedy=True
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Files path `../input/` import osos.listdir('../input')  Setup hyperparameters<jupyter_code># Set normalization
enable_normalization = True
normalization_type = 'minmax' # 'minmax' or 'standard'

# Exploratory analysis

# Set correlation
enable_correlation = False
enable_dendrogram = False
enable_heatmap = False

# Features Selection

# Set features selection with correlation criteria
enable_correlation_selec = False
factor = 0.95 # number close to 1

# Set features selection with univariate statitics test criteria
enable_univariate_selec = True
method_selec = 'selectkbest' # 'selectkbest', 'pca', ...
pca_variance = 0.95 
criteria_k_best = mutual_info_classif # chi2, mutual_info_classif
k_best = 84 # number of best features to select on select k best.

# Balancing

# Train balancing
enable_balancing = True
number_samples = 2500

# Machine learning method

ml_method = 'randomforestreg' # 'gradientboosting', 'svm', ...
gbc_loss = 'deviance' # gradient boosting loss
rfc_criterion = 'gini' # random forest criterion
enable_cv = True # enable cross-validation<jupyter_output><empty_output><jupyter_text>Winning setup Score: 9020Set normalization    enable_normalization = True    normalization_type = 'minmax'  'minmax' or 'standard'Exploratory analysisSet correlation    enable_correlation = False    enable_dendrogram = False    enable_heatmap = FalseFeatures SelectionSet features selection with correlation criteria    enable_correlation_selec = False    factor = 0.95  number close to 1Set features selection with univariate statitics test criteria    enable_univariate_selec = True    method_selec = 'selectkbest'  'selectkbest', 'pca', ...    pca_variance = 0.95     criteria_k_best = mutual_info_classif  chi2, mutual_info_classif    k_best = 84  number of best features to select on select k best.BalancingTrain balancing    enable_balancing = True    number_samples = 2500Machine learning method    ml_method = 'randomforestreg'  'gradientboosting', 'svm', 'randomforest'...    gbc_loss = 'deviance'  gradient boosting loss    rfc_criterion = 'gini'  random forest criterion    enable_cv = True  enable cross-validation  Util functions<jupyter_code># Print the bar graph from data
def bar(acumm_data):
    # Do plot
    fig = plt.figure(figsize=(10,7))
    ax = fig.add_subplot(111)
    ax = sns.barplot(x=acumm_data.index, y=acumm_data.values, palette='tab20b', ax=ax)
    ax.set_xticklabels(ax.get_xticklabels(), rotation=90)    
    return ax

def dendrogram(df):    
    # Do correlation matrix
    corr_matrix = df.corr()

    # Do the clustering
    Z = hac.linkage(corr_matrix, 'single')

    # Plot dendogram
    fig, ax = plt.subplots(figsize=(25, 10))
    plt.title('Hierarchical Clustering Dendrogram')
    plt.xlabel('sample index')
    plt.ylabel('distance')
    groups = hac.dendrogram(
        Z,
        leaf_rotation=90.,  # rotates the x axis labels
        leaf_font_size=8., # font size for the x axis labels
        color_threshold = 0#,
        #truncate_mode='lastp',
        #p=30
    )

    labels_dict = pd.DataFrame(df.columns).to_dict()[0]
    actual_labels = [item.get_text() for item in ax.get_xticklabels()]
    new_labels = [labels_dict[int(i)] for i in actual_labels]
    ax.set_xticklabels(new_labels)
    plt.tight_layout()

def corr_drop(corr_m, factor=.9):
    
    global cm
    cm = corr_m
    # Get correlation score, as high as this score, more chances to be dropped.
    cum_corr = cm.applymap(abs).sum()
    def remove_corr():
        global cm
        for col in cm.columns:
            for ind in cm.index:
                if (ind in cm.columns) and (col in cm.index):
                    # Compare if are high correlated.
                    if (cm.loc[ind,col] > factor) and (ind!=col):
                        cum = cum_corr[[ind,col]].sort_values(ascending=False)
                        cm.drop(cum.index[0], axis=0, inplace=True)
                        cm.drop(cum.index[0], axis=1, inplace=True)
                        # Do recursion until the last high correlated.
                        remove_corr()
        return cm
    return remove_corr()<jupyter_output><empty_output><jupyter_text>Read data and sepair features from labels<jupyter_code>train_features = pd.read_csv('../input/aps_failure_training_set_processed_8bit.csv', na_values='na')
test_features =  pd.read_csv('../input/aps_failure_test_set_processed_8bit.csv', na_values='na')

train_labels = train_features['class']
test_labels = test_features['class']
train_features = train_features.drop('class', axis=1)
test_features = test_features.drop('class', axis=1)<jupyter_output><empty_output><jupyter_text>Raw exploratory analysis<jupyter_code>train_features.describe()<jupyter_output><empty_output><jupyter_text>Pre-processing  Flat dataframe and check for non-valid values If doesn't exist, it is considered non-valid.<jupyter_code>flat_data = train_features.values.flatten()
count=0
for value in flat_data:
    if value is not None:
        continue
    count+= 1
pct_nan = round(100*count/len(flat_data))
print(f'{pct_nan}% of data are non-valid.')<jupyter_output><empty_output><jupyter_text>Scalling Apply MinMaxScaler with values between 0 and 1. Ref: http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html<jupyter_code>from sklearn.preprocessing import MinMaxScaler
if enable_normalization and normalization_type=='minmax':
    scaler = MinMaxScaler()
    scaler.fit(train_features)
    train_features = pd.DataFrame(scaler.transform(train_features), columns=train_features.columns)<jupyter_output><empty_output><jupyter_text>Exploratory analysis after scalling.<jupyter_code>train_features.describe()<jupyter_output><empty_output><jupyter_text>Processing train labels Round values and replace -1 with 0 making all labels positives.<jupyter_code>train_labels = train_labels.apply(round)
train_labels = train_labels.replace({-1:0})<jupyter_output><empty_output><jupyter_text>Show labels porportion<jupyter_code>bar(train_labels.value_counts())
plt.show()<jupyter_output><empty_output><jupyter_text>Processing and analysis over data  Correlation analysis Used method: Pearson distance. Ref: https://pt.wikipedia.org/wiki/Correla%C3%A7%C3%A3oCoeficiente_de_correla%C3%A7%C3%A3o_de_Pearson<jupyter_code>if enable_correlation and enable_dendrogram:
    corr_matrix = train_features.corr()
    dendrogram(corr_matrix)
    plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Correlation matrix heatmap.<jupyter_code>if enable_correlation and enable_heatmap:
    fig, ax = plt.subplots(figsize=(10,10))
    ax = sns.heatmap(corr_matrix, square=True, cmap='Purples', ax=ax)
    plt.tight_layout()
    plt.show()<jupyter_output><empty_output><jupyter_text>[](http://)The data has a lot of features, because of that, is very difficulty to visualize hierarchical graphs, One option is to use a pipeline with selection attribute method to reduce number of features and than dropping the high correlated features.  Feature selection by univariate statistics test Ref.:  http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.SelectKBest.html https://en.wikipedia.org/wiki/Chi-squared_distribution<jupyter_code># to enable run correlation selection without univariate selection.
best_train_features = train_features 
new_corr_matrix = best_train_features.corr()
if enable_univariate_selec:
    if method_selec=='selectkbest':
        selectKBest = SelectKBest(chi2, k_best)
        selectKBest.fit(train_features, train_labels)
        best_train_features = selectKBest.transform(train_features)

        idxs_selected = selectKBest.get_support(indices=True)
        best_train_features = train_features.iloc[:,idxs_selected]<jupyter_output><empty_output><jupyter_text>Selected columns.<jupyter_code>if enable_univariate_selec:
    if method_selec=='selectkbest':
        print(best_train_features.columns) # selected columns<jupyter_output><empty_output><jupyter_text>Descritive analysis after feature selection<jupyter_code>if enable_univariate_selec:
    if method_selec=='selectkbest':
        best_train_features.describe()<jupyter_output><empty_output><jupyter_text>Correlation after feature selection<jupyter_code>if enable_univariate_selec:
    if method_selec=='selectkbest':
        new_corr_matrix = best_train_features.corr()
        dendrogram(new_corr_matrix)
        plt.tight_layout()<jupyter_output><empty_output><jupyter_text>![](http://)__Obs: __ You can make a pipeline with feature selction method and feature selection by correlation, just enable both in *Setup hyperparameters*  Only mantain the best feature between the high correlated features<jupyter_code>if enable_correlation_selec:
    new_new_corr_matrix = corr_drop(new_corr_matrix, factor)
    print(f'Number of features selected is {len(new_new_corr_matrix.columns)}.')
if enable_correlation_selec:
    dendrogram(new_new_corr_matrix)
    plt.tight_layout()
if enable_correlation_selec:
    fig, ax = plt.subplots(figsize=(10,10))
    ax = sns.heatmap(new_new_corr_matrix, square=True, cmap='Purples', ax=ax)
    plt.tight_layout()
    plt.show()
if enable_correlation_selec:
    best_train_features = best_train_features.loc[:,new_new_corr_matrix.columns]<jupyter_output><empty_output><jupyter_text>Feature extraction with PCA<jupyter_code>if method_selec=='pca':
    pca = PCA(pca_variance)
    pca.fit(train_features)
    best_train_features = pca.transform(train_features)
    best_train_features = pd.DataFrame(best_train_features)
if method_selec=='pca':
    print('Number of components {pca.n_components_}')<jupyter_output><empty_output><jupyter_text>Modeling  Balancing<jupyter_code># to enable run without balancing
best_train_features_balanced = best_train_features
train_labels_balanced = train_labels
if enable_balancing:
    idxs_pos = train_labels[train_labels==1].index
    idxs_neg = train_labels[train_labels==0].sample(n=number_samples, replace=False, random_state=0).index
    idxs_balanced = np.concatenate((idxs_pos,idxs_neg))
    best_train_features_balanced = best_train_features.loc[idxs_balanced]
    train_labels_balanced = train_labels.loc[idxs_balanced]
    print(f'Proportion balanced: {int(number_samples/1000)}/1')<jupyter_output><empty_output><jupyter_text>Gradient Boosting Ref.  http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html http://scikit-learn.org/stable/modules/ensemble.html<jupyter_code>if ml_method=='gradientboosting':
    gbc = GradientBoostingClassifier(loss=gbc_loss, random_state=0)
    if not enable_cv:
        gbc.fit(best_train_features_balanced, train_labels_balanced)<jupyter_output><empty_output><jupyter_text>Run optimization with cross-validation.Ref.  http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html<jupyter_code>if ml_method=='gradientboosting' and enable_cv:
    #Seleciona os parâmetros do GB que deseja testar
    params = [{'loss': ['deviance', 'exponential']}]
    
    #Executa grid search com cross validation
    gbcc = GridSearchCV(gbc, params, cv=5, scoring='recall', verbose=10, n_jobs=3)
    gbcc.fit(best_train_features_balanced, train_labels_balanced)
    gbc = gbcc<jupyter_output><empty_output><jupyter_text>Chosen method.<jupyter_code>if ml_method=='gradientboosting':
    display(gbc)<jupyter_output><empty_output><jupyter_text>Random Forest Random Forest Regressor<jupyter_code>if ml_method=='randomforestreg':
    rfc = RandomForestRegressor(n_estimators=100, oob_score = True, random_state=0, n_jobs=3)
    rfc.fit(best_train_features_balanced, train_labels_balanced)
    print(rfc)<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code>if ml_method=='randomforest':
    rfc = RandomForestClassifier(criterion=rfc_criterion, random_state=0)
    if not enable_cv:
        rfc.fit(best_train_features_balanced, train_labels_balanced)
    
if ml_method=='randomforest' and enable_cv:
    #Seleciona os parâmetros do GB que deseja testar
    params = [{'criterion': ['gini', 'entropy'], 'max_features': ['sqrt', 'log2'], 'n_estimators': [10, 100]}]
    rfc = RandomForestClassifier(random_state=0)
    #Executa grid search com cross validation
    rfcc = GridSearchCV(rfc, params, cv=5, scoring='recall', verbose=10, n_jobs=3)
    rfcc.fit(best_train_features_balanced, train_labels_balanced)
    rfc = rfcc
if ml_method=='randomforest':
    display(rfcc)<jupyter_output><empty_output><jupyter_text>SVMRef.  http://scikit-learn.org/stable/modules/svm.html Run optimization with cross-validation.<jupyter_code>if ml_method=='svm':
    #Seleciona os parâmetros da SVM que deseja testar
    params = [{'kernel': ['rbf'], 'gamma': [0.01], 'C': [0.001, 0.01, 0.1, 1, 10]}, 
              {'kernel': ['linear'], 'gamma': [0.01],  'C':  [0.001, 0.01, 0.1, 1, 10]}
             ]
    #Executa grid search com cross validation
    svmc = GridSearchCV(svm.SVC(C=1), params, cv=5, scoring='recall', verbose=10, n_jobs=3)
    svmc.fit(best_train_features_balanced, train_labels_balanced)<jupyter_output><empty_output><jupyter_text>Prediction<jupyter_code># to enable change feature selection method
best_test_features = test_features<jupyter_output><empty_output><jupyter_text>Apply scalling over test data<jupyter_code>if enable_normalization:
    scaler.transform(best_test_features)
    best_test_features = pd.DataFrame(scaler.transform(best_test_features), columns=best_test_features.columns)<jupyter_output><empty_output><jupyter_text>Apply feature selection over test data<jupyter_code>if enable_univariate_selec:
    if method_selec=='selectkbest':        
        X = selectKBest.transform(best_test_features)
        idxs_selected = selectKBest.get_support(indices=True)
        best_test_features = best_test_features.iloc[:,idxs_selected]
    if method_selec=='pca':        
        best_test_features = pca.transform(best_test_features)
if enable_correlation_selec:
    best_test_features = best_test_features.loc[:,new_new_corr_matrix.columns]<jupyter_output><empty_output><jupyter_text>Processing labels As same as done with train labels.<jupyter_code>test_labels = test_labels.apply(round)
test_labels = test_labels.replace({-1:0})<jupyter_output><empty_output><jupyter_text>Make prediction with trainned model choosen and show report Predict with Gradient boosting.<jupyter_code>if ml_method=='gradientboosting':
    y_pred = gbc.predict(best_test_features)
    report = classification_report(test_labels, y_pred)
    print(report)
if ml_method=='randomforestreg':
    y_pred = rfc.predict(best_test_features)
    y_pred = np.round(y_pred)
    report = classification_report(test_labels, y_pred)
    print(report)
if ml_method=='randomforest':
    y_pred = rfc.predict(best_test_features)
    report = classification_report(test_labels, y_pred)
    print(report)<jupyter_output><empty_output><jupyter_text>Predict with SVM.<jupyter_code>if ml_method=='svm':
    y_pred = svmc.predict(best_test_features)
    report = classification_report(test_labels, y_pred)
    print(report)<jupyter_output><empty_output><jupyter_text>Show confusion matrixRef.http://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html<jupyter_code>cm = confusion_matrix(test_labels, y_pred).ravel()
cm = pd.DataFrame(cm.reshape((1,4)), columns=['tn', 'fp', 'fn', 'tp'])
display(cm)<jupyter_output><empty_output><jupyter_text>Show total cost obtained<jupyter_code>total_cost = 10*cm.fp + 500*cm.fn
def printmd(string):
    display(Markdown(string))
printmd(f'Total cost is: \n# <p><span style="color:purple">${float(total_cost.values[0])}</span></p>')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10050465130418922, 'fraction_numerical': 0.010093026083784276, 'mean_word_length': 5.555599840573934, 'pattern_counts': {'":': 0, '<': 137, '<?xml version=': 0, '>': 138, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '33', 'dir': 'jupyter-structured-clean-dedup', 'id': '287022', 'license': 'MIT', 'n_tokens_mistral': 5678, 'n_tokens_neox': 5360, 'n_words': 1049, 'path': 'Raw Codes/best.ipynb', 'repo_name': 'masudurHimel/ML---APS-Failure-at-Scania-Trucks-Data-Set', 'size': '16447'}