<jupyter_start><jupyter_text>Random Forest<jupyter_code>import pickle
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from pprint import pprint
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from sklearn.model_selection import ShuffleSplit
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd<jupyter_output><empty_output><jupyter_text>First, we load the data:<jupyter_code># Dataframe
path_df = "/home/lnc/0. Latest News Classifier/03. Feature Engineering/Pickles/df.pickle"
with open(path_df, 'rb') as data:
    df = pickle.load(data)

# features_train
path_features_train = "/home/lnc/0. Latest News Classifier/03. Feature Engineering/Pickles/features_train.pickle"
with open(path_features_train, 'rb') as data:
    features_train = pickle.load(data)

# labels_train
path_labels_train = "/home/lnc/0. Latest News Classifier/03. Feature Engineering/Pickles/labels_train.pickle"
with open(path_labels_train, 'rb') as data:
    labels_train = pickle.load(data)

# features_test
path_features_test = "/home/lnc/0. Latest News Classifier/03. Feature Engineering/Pickles/features_test.pickle"
with open(path_features_test, 'rb') as data:
    features_test = pickle.load(data)

# labels_test
path_labels_test = "/home/lnc/0. Latest News Classifier/03. Feature Engineering/Pickles/labels_test.pickle"
with open(path_labels_test, 'rb') as data:
    labels_test = pickle.load(data)<jupyter_output><empty_output><jupyter_text>Let's check the dimension of our feature vectors:<jupyter_code>print(features_train.shape)
print(features_test.shape)<jupyter_output>(1891, 300)
(334, 300)<jupyter_text>Cross-Validation for Hyperparameter tuning First, we can see what hyperparameters the model has:<jupyter_code>rf_0 = RandomForestClassifier(random_state = 8)

print('Parameters currently in use:\n')
pprint(rf_0.get_params())<jupyter_output>Parameters currently in use:

{'bootstrap': True,
 'class_weight': None,
 'criterion': 'gini',
 'max_depth': None,
 'max_features': 'auto',
 'max_leaf_nodes': None,
 'min_impurity_decrease': 0.0,
 'min_impurity_split': None,
 'min_samples_leaf': 1,
 'min_samples_split': 2,
 'min_weight_fraction_leaf': 0.0,
 'n_estimators': 10,
 'n_jobs': 1,
 'oob_score': False,
 'random_state': 8,
 'verbose': 0,
 'warm_start': False}<jupyter_text>We'll tune the following ones:* `n_estimators` = number of trees in the forest.* `max_features` = max number of features considered for splitting a node* `max_depth` = max number of levels in each decision tree* `min_samples_split` = min number of data points placed in a node before the node is split* `min_samples_leaf` = min number of data points allowed in a leaf node* `bootstrap` = method for sampling data points (with or without replacement)  Randomized Search Cross Validation We first need to define the grid:<jupyter_code># n_estimators
n_estimators = [int(x) for x in np.linspace(start = 200, stop = 1000, num = 5)]

# max_features
max_features = ['auto', 'sqrt']

# max_depth
max_depth = [int(x) for x in np.linspace(20, 100, num = 5)]
max_depth.append(None)

# min_samples_split
min_samples_split = [2, 5, 10]

# min_samples_leaf
min_samples_leaf = [1, 2, 4]

# bootstrap
bootstrap = [True, False]

# Create the random grid
random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}

pprint(random_grid)<jupyter_output>{'bootstrap': [True, False],
 'max_depth': [20, 40, 60, 80, 100, None],
 'max_features': ['auto', 'sqrt'],
 'min_samples_leaf': [1, 2, 4],
 'min_samples_split': [2, 5, 10],
 'n_estimators': [200, 400, 600, 800, 1000]}<jupyter_text>Then, we'll perform the Random Search:<jupyter_code># First create the base model to tune
rfc = RandomForestClassifier(random_state=8)

# Definition of the random search
random_search = RandomizedSearchCV(estimator=rfc,
                                   param_distributions=random_grid,
                                   n_iter=50,
                                   scoring='accuracy',
                                   cv=3, 
                                   verbose=1, 
                                   random_state=8)

# Fit the random search model
random_search.fit(features_train, labels_train)<jupyter_output>Fitting 3 folds for each of 50 candidates, totalling 150 fits<jupyter_text>We can see the best hyperparameters resulting from the Random Search:<jupyter_code>print("The best hyperparameters from Random Search are:")
print(random_search.best_params_)
print("")
print("The mean accuracy of a model with these hyperparameters is:")
print(random_search.best_score_)<jupyter_output>The best hyperparameters from Random Search are:
{'n_estimators': 800, 'min_samples_split': 10, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'max_depth': 40, 'bootstrap': False}

The mean accuracy of a model with these hyperparameters is:
0.9423585404547858<jupyter_text>After that, we can do a more exhaustive search centered in those values:  Grid Search Cross Validation<jupyter_code># Create the parameter grid based on the results of random search 
bootstrap = [False]
max_depth = [30, 40, 50]
max_features = ['sqrt']
min_samples_leaf = [1, 2, 4]
min_samples_split = [5, 10, 15]
n_estimators = [800]

param_grid = {
    'bootstrap': bootstrap,
    'max_depth': max_depth,
    'max_features': max_features,
    'min_samples_leaf': min_samples_leaf,
    'min_samples_split': min_samples_split,
    'n_estimators': n_estimators
}

# Create a base model
rfc = RandomForestClassifier(random_state=8)

# Manually create the splits in CV in order to be able to fix a random_state (GridSearchCV doesn't have that argument)
cv_sets = ShuffleSplit(n_splits = 3, test_size = .33, random_state = 8)

# Instantiate the grid search model
grid_search = GridSearchCV(estimator=rfc, 
                           param_grid=param_grid,
                           scoring='accuracy',
                           cv=cv_sets,
                           verbose=1)

# Fit the grid search to the data
grid_search.fit(features_train, labels_train)<jupyter_output>Fitting 3 folds for each of 27 candidates, totalling 81 fits<jupyter_text>The best hyperparameters turn out to be:<jupyter_code>print("The best hyperparameters from Grid Search are:")
print(grid_search.best_params_)
print("")
print("The mean accuracy of a model with these hyperparameters is:")
print(grid_search.best_score_)<jupyter_output>The best hyperparameters from Grid Search are:
{'bootstrap': False, 'max_depth': 40, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 800}

The mean accuracy of a model with these hyperparameters is:
0.9450666666666667<jupyter_text>Let's save the model in `best_rfc`:<jupyter_code>best_rfc = grid_search.best_estimator_
best_rfc<jupyter_output><empty_output><jupyter_text>We now know the best random forest model. Let's fit it and see how it performs:  Model fit and performance Now, we can fit the model to our training data:<jupyter_code>best_rfc.fit(features_train, labels_train)<jupyter_output><empty_output><jupyter_text>And get the predictions:<jupyter_code>rfc_pred = best_rfc.predict(features_test)<jupyter_output><empty_output><jupyter_text>The conditional class probabilities can be obtained by typing:`rfc_pred = best_rfc.predict_proba(features_test)` For performance analysis, we will use the confusion matrix, the classification report and the accuracy on both training and test data:  Training accuracy<jupyter_code># Training accuracy
print("The training accuracy is: ")
print(accuracy_score(labels_train, best_rfc.predict(features_train)))<jupyter_output>The training accuracy is: 
1.0<jupyter_text>Test accuracy<jupyter_code># Test accuracy
print("The test accuracy is: ")
print(accuracy_score(labels_test, rfc_pred))<jupyter_output>The test accuracy is: 
0.9281437125748503<jupyter_text>Classification report<jupyter_code># Classification report
print("Classification report")
print(classification_report(labels_test,rfc_pred))<jupyter_output>Classification report
             precision    recall  f1-score   support

          0       0.87      0.95      0.91        81
          1       0.90      0.94      0.92        49
          2       0.97      0.86      0.91        72
          3       0.97      0.96      0.97        72
          4       0.95      0.93      0.94        60

avg / total       0.93      0.93      0.93       334<jupyter_text>Confusion matrix<jupyter_code>aux_df = df[['Category', 'Category_Code']].drop_duplicates().sort_values('Category_Code')
conf_matrix = confusion_matrix(labels_test, rfc_pred)
plt.figure(figsize=(12.8,6))
sns.heatmap(conf_matrix, 
            annot=True,
            xticklabels=aux_df['Category'].values, 
            yticklabels=aux_df['Category'].values,
            cmap="Blues")
plt.ylabel('Predicted')
plt.xlabel('Actual')
plt.title('Confusion matrix')
plt.show()<jupyter_output><empty_output><jupyter_text>At this point, we could get the average time the model takes to get predictions. We want the algorithm to be fast since we are creating an app which will gather data from the internet and get the predicted categories. However, since the difference when predicting 10-20 observations will be very little, we won't take this into account.However, the code below could do this task:```pythonfeatures_time = features_trainelapsed_list = []for i in range(0,10):        start = time.time()    predictions = best_lrc.predict(features_time)    end = time.time()    elapsed = end - start    elapsed_list.append(elapsed)mean_time_elapsed = np.mean(elapsed_list)``` Let's see if the hyperparameter tuning process has returned a better model:<jupyter_code>base_model = RandomForestClassifier(random_state = 8)
base_model.fit(features_train, labels_train)
accuracy_score(labels_test, base_model.predict(features_test))
best_rfc.fit(features_train, labels_train)
accuracy_score(labels_test, best_rfc.predict(features_test))<jupyter_output><empty_output><jupyter_text>We'll create a dataset with a model summary to compare models:<jupyter_code>d = {
     'Model': 'Random Forest',
     'Training Set Accuracy': accuracy_score(labels_train, best_rfc.predict(features_train)),
     'Test Set Accuracy': accuracy_score(labels_test, rfc_pred)
}

df_models_rfc = pd.DataFrame(d, index=[0])
df_models_rfc<jupyter_output><empty_output><jupyter_text>Let's save the model and this dataset:<jupyter_code>with open('Models/best_rfc.pickle', 'wb') as output:
    pickle.dump(best_rfc, output)
    
with open('Models/df_models_rfc.pickle', 'wb') as output:
    pickle.dump(df_models_rfc, output)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09684172203513243, 'fraction_numerical': 0.026485846909984526, 'mean_word_length': 4.611848825331972, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '163514', 'license': 'MIT', 'n_tokens_mistral': 3731, 'n_tokens_neox': 3482, 'n_words': 939, 'path': '04.Model-Training/06. MT - Random Forest.ipynb', 'repo_name': 'abbasmalekpour/News-Classifier', 'size': '10987'}