<jupyter_start><jupyter_text>Model training and evaluation<jupyter_code># The modules we will need


# NumPy might be useful in some cases with data manipulating 
import numpy as np


# pandas will be useful for data handling
import pandas as pd


# an option for displaying all the columns of our main dataframe
pd.options.display.max_columns = 30


# With matplotlib we will do the plotting
import matplotlib.pyplot as plt
%matplotlib inline


# with this option we control the look and feel of the plots
from matplotlib import style
style.use((['ggplot',  'fast']))


# This will be our classifier
from sklearn.ensemble import RandomForestClassifier

# This helps with automatically splitting the data into train and test parts
from sklearn.model_selection import train_test_split


# Grid search looks at many combinations of parameters of the classifier and finds the optimal combination
from sklearn.model_selection import GridSearchCV

# These are some of the standard metrics for evaluating out models
from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc, f1_score

# We will use this module to 'pickle' and preserve the trained model, so that we can save it for later use
import pickle

model1_df = pd.read_csv('output/model1_data.csv', dtype='float16', index_col='Unnamed: 0')
model1_df.head()
model1_df['LateAircraftDelay'] = model1_df['LateAircraftDelay'].apply(lambda x: True if x > 0 else False)
model1_df.replace([np.inf, -np.inf], 0, inplace=True)
model1_df.dropna(inplace=True)
model1_df.reset_index(inplace=True)
model1_df.head()
X = model1_df[['previous_ArrDelay_percent', 'avg_flight_time_percent', 'utilisation_percent']]
y = model1_df[['LateAircraftDelay']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)
## random True column with the same proportion as the target feature
class_proportion = y_test.mean()[0]

y_test['random'] = y_test.apply( lambda x : 1 if np.random.random() < class_proportion else 0 , axis=1)
y_test.head()
### Do not try this at home, as it will take too looong.
### Uncomment to use
'''
classifier = RandomForestClassifier()

parameters = { 'n_estimators' : [100]
              , 'criterion' : ('gini', 'entropy')
              , 'max_depth' : [3,6,9] 
              , 'min_samples_leaf' : [100, 500, 1000]
              , 'bootstrap' : [True]
              , 'oob_score' : [True]
              , 'class_weight' : ['balanced']
             }

grid = GridSearchCV(classifier, parameters)

grid.fit(X_train, y_train)

print(grid.best_estimator_)
'''
## saving the grid search for later inspection
grid_filepath = 'output/grid_search.p'
#pickle.dump(grid, open(grid_filepath, 'wb'))
## let us load the grid, so one does not have to do the computation again
grid = pickle.load( open( grid_filepath, 'rb' ) )
## Let us use the best parameters to train our model
best_classifier = grid.best_estimator_
print (grid.best_estimator_)
## classifying with the best parameters
## consider loading trained model instead of training it again
best_classifier.fit(X_train, y_train)
## saving the trained model for later use
model_filepath = 'output/model1.p'
#pickle.dump(best_classifier, open(model_filepath, 'wb'))
## let us load the model, so one does not have to do the computation again
best_classifier = pickle.load( open( model_filepath, 'rb' ) )
## using our trained model to make predictions
y_pred = best_classifier.predict(X_test)
## The importance of the features

feature_imp = pd.Series(best_classifier.feature_importances_,index=X.columns)
feature_imp
import matplotlib.pyplot as plt
import seaborn as sns
#%matplotlib inline
# Creating a bar plot
sns.barplot(x=feature_imp, y=feature_imp.index)
# Add labels to your graph
plt.xlabel('Feature Importance Score')
plt.ylabel('Features')
plt.title("Visualizing Important Features")
plt.legend()
plt.show()
### model1 ROC

false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test['LateAircraftDelay'], y_pred)
roc_auc = auc(false_positive_rate, true_positive_rate)

plt.title('Receiver Operating Characteristic')
plt.plot(false_positive_rate, true_positive_rate, 'b', label="model's_AUC = %0.2f"% roc_auc)
plt.legend(loc='lower right')
plt.plot([0,1],[0,1],'r:', linewidth=7)
plt.xlim([-0.1,1.1])
plt.ylim([-0.1,1.1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')



### random guess ROC

false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test['random'], y_pred)
roc_auc = auc(false_positive_rate, true_positive_rate)

plt.title('Receiver Operating Characteristic')
plt.plot(false_positive_rate, true_positive_rate, 'green', label='random_AUC = %0.2f'% roc_auc)
plt.legend(loc='lower right')


plt.show()
## classification report for model1
print (classification_report(y_test['LateAircraftDelay'], y_pred) )
## for comparison we compute a classification report for a random guess
print (classification_report(y_test['random'], y_pred) )
## the confusion matrix for the model ...
print (confusion_matrix(y_test['LateAircraftDelay'], y_pred) )
## ... and the confusion matrix for the random guess
print (confusion_matrix(y_test['random'], y_pred) )<jupyter_output>[[1629477  362654]
 [ 182081   40442]]
{'fraction_non_alphanumeric': 0.1070947894635647, 'fraction_numerical': 0.016535281676600653, 'mean_word_length': 5.446096654275093, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '573180', 'license': 'MIT', 'n_tokens_mistral': 1763, 'n_tokens_neox': 1645, 'n_words': 506, 'path': 'phase_4/phase_4_model1_codebase.ipynb', 'repo_name': 'andreas-masaoutis/dscp_code-documentation', 'size': '5201'}