<jupyter_start><jupyter_text>Topics0. An end-to-end Scikit-Learn workflow1. Getting the data ready2. Choose the right estimator/algorithm for our problems3. Fit the model/algorithm and use it to make predictions on our data4. Evaluating a model5. Improve a model6. Save and load a trained model7. Putting it all together!  0. An end-to-end Scikit-Learn workflow<jupyter_code>import pandas as pd
import numpy as np

heart_disease = pd.read_csv("./data/heart-disease.csv")
heart_disease.head()
# create features matrix
X = heart_disease.drop("target", axis=1)

# create labels
y = heart_disease["target"]
from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier()

# we will keep the default hyperparameters
clf.get_params()
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
clf.fit(X_train, y_train)
y_preds = clf.predict(X_test)
y_preds
clf.score(X_train, y_train) # accuracy
clf.score(X_test, y_test) # accuracy
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

print(classification_report(y_test, y_preds))
confusion_matrix(y_test, y_preds)
accuracy_score(y_test, y_preds)
# 5. Imporve our model
# Try different amount of n_estimators

np.random.seed(42)
for i in range(10, 110, 10):
    print(f"Trying model with {i} estimators ...")
    clf = RandomForestClassifier(n_estimators=i)
    clf.fit(X_train, y_train)
    print(f"Model accuracy on test set: {clf.score(X_test,y_test) * 100:.2f}% ")
# 6. Save a model and load it
import pickle

# wb = write binary
pickle.dump(clf, open("./models/random_forest_model_1.pkl", "wb"))
loaded_model = pickle.load(open("./models/random_forest_model_1.pkl", "rb"))
loaded_model.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>1.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
heart_disease = pd.read_csv("./data/heart-disease.csv")
# create features matrix
X = heart_disease.drop("target", axis=1)

# create labels
y = heart_disease["target"]
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)<jupyter_output><empty_output><jupyter_text>1.1 Make sure its all numerical<jupyter_code>car_sales = pd.read_csv("./data/car-sales-extended.csv")
car_sales.head()
len(car_sales)
car_sales.dtypes
X = car_sales.drop("Price", axis=1)
y = car_sales["Price"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
#from sklearn.ensemble import RandomForestRegressor

#model = RandomForestRegressor()
#model.fit(X_train, y_train)
#model.score(X_test, y_test)

# ValueError: could not convert string to float: 'Toyota'
# turn categories into numbers
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer

categorical_features = ["Make", "Colour", "Doors"]
one_hot = OneHotEncoder()
transformer = ColumnTransformer([("one_hot",
                                 one_hot,
                                 categorical_features)],
                               remainder="passthrough")

transformed_X = transformer.fit_transform(X)
pd.DataFrame(transformed_X).head()
type(transformed_X)
X.head()

# Make => 
# Colour =>
# Doors => 9(3) 10(4) 11(5)
# Another way to do it with pd.dummies...
dummies = pd.get_dummies(car_sales[["Make", "Colour", "Doors"]])
dummies
from sklearn.ensemble import RandomForestRegressor

# Let's refit the model
np.random.seed(42)
X_train, X_test, y_train, y_test = train_test_split(transformed_X,
                                                    y, 
                                                    test_size=0.2)
model = RandomForestRegressor()
model.fit(X_train, y_train)
model.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>1.2 What if there were missing values?1. Fill them with some value (also known as imputation).2. Remove the samples with missing data altogether.<jupyter_code># Import car sales missing data
car_sales_missing = pd.read_csv("./data/car-sales-extended-missing-data.csv")
car_sales_missing.head()
car_sales_missing.isna().sum() # missing data
# Create X & y
X = car_sales_missing.drop("Price", axis=1)
y = car_sales_missing["Price"]
# Let's try and convert our data to numbers
# Turn the categories into numbers

# from sklearn.preprocessing import OneHotEncoder
# from sklearn.compose import ColumnTransformer

# categorical_features = ["Make", "Colour", "Doors"]
# one_hot = OneHotEncoder()
# transformer = ColumnTransformer([("one_hot",
#                                    one_hot,
#                                    categorical_features)],
#                                    remainder="passthrough")

# transformed_X = transformer.fit_transform(X)
# transformed_X

# ValueError: Input contains NaN
car_sales_missing<jupyter_output><empty_output><jupyter_text>Option 1: Fill missing data with Pandas<jupyter_code># Fill the "Make" column
car_sales_missing["Make"].fillna("missing", inplace=True)

# Fill the "Colour" column
car_sales_missing["Colour"].fillna("missing", inplace=True)

# Fill the "Odometer (KM)" column
car_sales_missing["Odometer (KM)"].fillna(car_sales_missing["Odometer (KM)"].mean(), inplace=True)

# Fill the "Doors" column
car_sales_missing["Doors"].fillna(4, inplace=True)
# Check our dataframe again
car_sales_missing.isna().sum()
# Remove rows with missing Price value
car_sales_missing.dropna(inplace=True)
 car_sales_missing.isna().sum()
len(car_sales_missing)
X = car_sales_missing.drop("Price", axis=1)
y = car_sales_missing["Price"]

# Let's try and convert our data to numbers
# Turn the categories into numbers
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer

categorical_features = ["Make", "Colour", "Doors"]
one_hot = OneHotEncoder()
transformer = ColumnTransformer([("one_hot",
                                   one_hot,
                                   categorical_features)],
                                   remainder="passthrough")

transformed_X = transformer.fit_transform(car_sales_missing)
transformed_X<jupyter_output><empty_output><jupyter_text>Option 2: Filling missing data and transforming categorical data with Scikit-Learn<jupyter_code># Import car sales missing data
car_sales_missing = pd.read_csv("./data/car-sales-extended-missing-data.csv")
car_sales_missing.head()
car_sales_missing.isna().sum()
# Drop the rows with no labels
car_sales_missing.dropna(subset=["Price"], inplace=True)
car_sales_missing.isna().sum()
# Split into X & y
X = car_sales_missing.drop("Price", axis=1)
y = car_sales_missing["Price"]

# Split data into train and test
np.random.seed(42)
X_train, X_test, y_train, y_test = train_test_split(X,
                                                    y,
                                                    test_size=0.2)
# Check missing values
X.isna().sum()
# Fill missing values with Scikit-Learn
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer

# Fill categorical values with 'missing' & numerical values with mean
cat_imputer = SimpleImputer(strategy="constant", fill_value="missing")
door_imputer = SimpleImputer(strategy="constant", fill_value=4)
num_imputer = SimpleImputer(strategy="mean")

# Define columns
cat_features = ["Make", "Colour"]
door_feature = ["Doors"]
num_features = ["Odometer (KM)"]

# Create an imputer (something that fills missing data)
imputer = ColumnTransformer([
    ("cat_imputer", cat_imputer, cat_features),
    ("door_imputer", door_imputer, door_feature),
    ("num_imputer", num_imputer, num_features)
])

# Fill train and test values separately
filled_X_train = imputer.fit_transform(X_train)
filled_X_test = imputer.transform(X_test)

# Check filled X_train
filled_X_train
# Get our transformed data array's back into DataFrame's
car_sales_filled_train = pd.DataFrame(filled_X_train, 
                                      columns=["Make", "Colour", "Doors", "Odometer (KM)"])

car_sales_filled_test = pd.DataFrame(filled_X_test, 
                                     columns=["Make", "Colour", "Doors", "Odometer (KM)"])

# Check missing data in training set
car_sales_filled_train.isna().sum()
# Check to see the original... still missing values
car_sales_missing.isna().sum()
# Now let's one hot encode the features with the same code as before 
categorical_features = ["Make", "Colour", "Doors"]
one_hot = OneHotEncoder()
transformer = ColumnTransformer([("one_hot", 
                                 one_hot, 
                                 categorical_features)],
                                 remainder="passthrough")

# Fill train and test values separately
transformed_X_train = transformer.fit_transform(car_sales_filled_train)
transformed_X_test = transformer.transform(car_sales_filled_test)

# Check transformed and filled X_train
transformed_X_train.toarray()
# Now we've transformed X, let's see if we can fit a model
np.random.seed(42)
from sklearn.ensemble import RandomForestRegressor

model = RandomForestRegressor()

# Make sure to use transformed (filled and one-hot encoded X data)
model.fit(transformed_X_train, y_train)
model.score(transformed_X_test, y_test)
# Check length of transformed data (filled and one-hot encoded)
# vs. length of original data
len(transformed_X_train.toarray())+len(transformed_X_test.toarray()), len(car_sales)<jupyter_output><empty_output><jupyter_text>2. Choosing the right estimator/algorithm for our problem  2.1 Picking a machine learning model for a regression problem<jupyter_code>from sklearn.datasets import load_boston
boston = load_boston()

boston_df = pd.DataFrame(boston["data"], columns=boston["feature_names"])
boston_df["target"] = pd.Series(boston["target"])
boston_df.head()
# How many samples?
len(boston_df)
# Let's try the Ridge Regression model
from sklearn.linear_model import Ridge

# Setup random seed
np.random.seed(42)

# Create the data
X = boston_df.drop("target", axis=1)
y = boston_df["target"]

# Split into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instantiate Ridge model
model = Ridge()
model.fit(X_train, y_train)

# Check the score of the Ridge model on test data
model.score(X_test, y_test)
# Let's try the Random Forst Regressor
from sklearn.ensemble import RandomForestRegressor

# Setup random seed
np.random.seed(42)

# Create the data
X = boston_df.drop("target", axis=1)
y = boston_df["target"]

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instatiate Random Forest Regressor
rf = RandomForestRegressor()
rf.fit(X_train, y_train)

# Evaluate the Random Forest Regressor
rf.score(X_test, y_test)
# Check the Ridge model again
model.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>2.2 Choosing an estimator for a classification problem<jupyter_code>heart_disease = pd.read_csv("./data/heart-disease.csv")
heart_disease.head()
# Import the LinearSVC estimator class
from sklearn.svm import LinearSVC

# Setup random seed
np.random.seed(42)

# Make the data
X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instantiate LinearSVC
clf = LinearSVC(max_iter=10000)
clf.fit(X_train, y_train)

# Evaluate the LinearSVC
clf.score(X_test, y_test)
heart_disease["target"].value_counts()
# Import the RandomForestClassifier estimator class
from sklearn.ensemble import RandomForestClassifier

# Setup random seed
np.random.seed(42)

# Make the data
X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instantiate Random Forest Classifier
clf = RandomForestClassifier()
clf.fit(X_train, y_train)

# Evaluate the Random Forest Classifier
clf.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Tidbit:1. If you have structured data, used ensemble methods2. If you have unstructured data, use deep learning or transfer learning  3. Fit the model/algorithm on our data and use it to make predictions  3.1 Fitting the model to the dataDifferent names for:- X = features, features variables, data- y = labels, targets, target variables<jupyter_code># Import the RandomForestClassifier estimator class
from sklearn.ensemble import RandomForestClassifier

# Setup random seed
np.random.seed(42)

# Make the data
X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instantiate Random Forest Classifier
clf = RandomForestClassifier(n_estimators=100)

# Fit the model to the data (training the machine learning model)
clf.fit(X_train, y_train)

# Evaluate the Random Forest Classifier (use the patterns the model has learned)
clf.score(X_test, y_test)
clf.predict(X_test)
np.array([y_test])
# Compare predictions to truth labels to evaluate the model
y_preds = clf.predict(X_test)
np.mean(y_preds == y_test)
clf.score(X_test, y_test)
from sklearn.metrics import accuracy_score
accuracy_score(y_test, y_preds)<jupyter_output><empty_output><jupyter_text>Make predictions with predict_proba() - use this if someone asks you "what's the probability your model is assigning to each prediction?"<jupyter_code>clf.predict_proba(X_test[:5]) # returns array([[prob for class1, prob for class2]])
clf.predict(X_test[:5]) 
# predicting first row as class 0 with probability 0.89
from sklearn.ensemble import RandomForestRegressor

np.random.seed(42)

# Create the data
X = boston_df.drop("target", axis=1)
y = boston_df["target"]

# Split into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instantiate and fit model
model = RandomForestRegressor()
model.fit(X_train, y_train)

# Make predictions
y_preds = model.predict(X_test)
y_preds[:5]
np.array(y_test[:5]) # compare with above
# Compare the predictions to the truth and find the avg error
from sklearn.metrics import mean_absolute_error
mean_absolute_error(y_test, y_preds)<jupyter_output><empty_output><jupyter_text>4. Evaluating a machine learning modelThree ways to evaluate Scikit-Learn models/esitmators:- Estimator score method- The scoring parameter- Problem-specific metric functions.  4.1 Evaluating a model with the score method<jupyter_code>from sklearn.ensemble import RandomForestClassifier

np.random.seed(42)

X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

clf = RandomForestClassifier()

clf.fit(X_train, y_train)
clf.score(X_train, y_train)
clf.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Let's do the same but for regression...<jupyter_code>from sklearn.ensemble import RandomForestRegressor

np.random.seed(42)

# Create the data
X = boston_df.drop("target", axis=1)
y = boston_df["target"]

# Split into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instantiate and fit model
model = RandomForestRegressor(n_estimators=100).fit(X_train, y_train)
model.score(X_train, y_train)
model.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>4.2 Evaluating a model using the scoring parameter<jupyter_code>from sklearn.model_selection import cross_val_score

from sklearn.ensemble import RandomForestClassifier

np.random.seed(42)

X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

clf = RandomForestClassifier(n_estimators=100)

clf.fit(X_train, y_train);
clf.score(X_test, y_test)
cross_val_score(clf, X, y, cv=5)
cross_val_score(clf, X, y, cv=10)
np.random.seed(42)

# Single training and test split score
clf_single_score = clf.score(X_test, y_test)

# Take the mean of 5-fold cross-validation score
clf_cross_val_score = np.mean(cross_val_score(clf, X, y, cv=5))

# Compare the two
clf_single_score, clf_cross_val_score
# Scoring parameter set to None by default
cross_val_score(clf, X, y, cv=5, scoring=None)
                              
# Default scoring parameter of classifier = mean accuracy
# clf.score()<jupyter_output><empty_output><jupyter_text>4.2.1 Classification model evaluation metrics- Accuracy- Area under ROC curve- Confusion matrix- Classification report Accuracy<jupyter_code>from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier

np.random.seed(42)

X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

clf = RandomForestClassifier(n_estimators=100)
cross_val_score = cross_val_score(clf, X, y, cv=5)
np.mean(cross_val_score)
print(f"Heart Disease Classifier Cross-Validated Accuracy: {np.mean(cross_val_score) *100:.2f}%")<jupyter_output>Heart Disease Classifier Cross-Validated Accuracy: 82.48%<jupyter_text>Area under the receiver operating characteristic curve (AUC/ROC)* Area under curve (AUC)* ROC curveROC curves are a comparison of a model's true postive rate (tpr) versus a models false positive rate (fpr).* True positive = model predicts 1 when truth is 1* False positive = model predicts 1 when truth is 0* True negative = model predicts 0 when truth is 0* False negative = model predicts 0 when truth is<jupyter_code># Create X_test... etc
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
from sklearn.metrics import roc_curve

# Fit the classifier
clf.fit(X_train, y_train)

# Make predictions with probabilities
y_probs = clf.predict_proba(X_test)

y_probs[:5], len(y_probs)
y_probs_positive = y_probs[:, 1]
y_probs_positive[:5]
fpr, tpr, thresholds = roc_curve(y_test, y_probs_positive)

# Check the false positive rates
fpr
# Create a function for plotting ROC curves
import matplotlib.pyplot as plt

def plot_roc_curve(fpr, tpr):
    """
    Plots a ROC curve given the false positive rate (fpr)
    and true positive rate (tpr) of a model.
    """
    # Plot roc curve
    plt.plot(fpr, tpr, color="orange", label="ROC")
    # Plot line with no predictive power (baseline)
    plt.plot([0, 1], [0, 1], color="darkblue", linestyle="--", label="Guessing")
    
    # Customize the plot
    plt.xlabel("False positive rate (fpr)")
    plt.ylabel("True positive rate (tpr)")
    plt.title("Receiver Operating Characteristic (ROC) Curve")
    plt.legend()
    plt.show()

plot_roc_curve(fpr, tpr)
from sklearn.metrics import roc_auc_score

roc_auc_score(y_test, y_probs_positive)
# area under the yellow curve and x-axis
# Plot perfect ROC curve and AUC score
fpr, tpr, thresholds = roc_curve(y_test, y_test)
plot_roc_curve(fpr, tpr)
roc_auc_score(y_test, y_test) # perfect roc<jupyter_output><empty_output><jupyter_text>Confusion MatrixA confusion matrix is a quick way to compare the labels a model predicts and the actual labels it was supposed to predict.In essence, giving you an idea of where the model is getting confused.<jupyter_code>from sklearn.metrics import confusion_matrix

y_preds = clf.predict(X_test)

confusion_matrix(y_test, y_preds)
# Visualize confusion matrix with pd.crosstab()
pd.crosstab(y_test,
            y_preds,
            rownames=["Actual"],
            colnames=["Predicted"])<jupyter_output><empty_output><jupyter_text>Classification Report To summarize classification metrics:* Accuracy is a good measure to start with if all classes are balanced (e.g. same amount of samples which are labelled with 0 or 1).* Precision and recall become more important when classes are imbalanced.* If false positive predictions are worse than false negatives, aim for higher precision.* If false negative predictions are worse than false positives, aim for higher recall.* F1-score is a combination of precision and recall. Precision = True Positive / (True Positive + False Positive)The measure of patients that we correctly identify having a heart disease out of all the patients actually having it.What is the Precision for our model? Yes, it is 0.83 or, when it predicts that a patient has heart disease, it is correct around 83% of the time. Recall = True Positive / (True Positive + False Negative)The recall is the measure of our model correctly identifying True Positives. Thus, for all the patients who actually have heart disease, recall tells us how many we correctly identified as having a heart disease.For our model, Recall  = 0.86. Recall also gives a measure of how accurately our model is able to identify the relevant data. We refer to it as Sensitivity or True Positive Rate. What if a patient has heart disease, but there is no treatment given to him/her because our model predicted so? That is a situation we would like to avoid! Accuracy = (True Positive + True Negative) / (True Positive + False Positive + True Positive + True Negative)Accuracy is the ratio of the total number of correct predictions and the total number of predictions.<jupyter_code>from sklearn.metrics import classification_report

print(classification_report(y_test, y_preds))
# Where precision and recall become valuable
disease_true = np.zeros(10000)
disease_true[0] = 1 # only one positive case

disease_preds = np.zeros(10000) # model predicts every case as 0

print(classification_report(disease_true,
                                   disease_preds))<jupyter_output>precision    recall  f1-score   support

         0.0       1.00      1.00      1.00      9999
         1.0       0.00      0.00      0.00         1

    accuracy                           1.00     10000
   macro avg       0.50      0.50      0.50     10000
weighted avg       1.00      1.00      1.00     10000<jupyter_text>4.2.2 Regression model evaluation metricsModel evaluation metrics documentation - https://scikit-learn.org/stable/modules/model_evaluation.html* R^2 (pronounced r-squared) or coefficient of determination.* Mean absolute error (MAE)* Mean squared error (MSE) R^2What R-squared does: Compares your models predictions to the mean of the targets. Values can range from negative infinity (a very poor model) to 1. For example, if all your model does is predict the mean of the targets, it's R^2 value would be 0. And if your model perfectly predicts a range of numbers it's R^2 value would be 1.<jupyter_code>from sklearn.ensemble import RandomForestRegressor

np.random.seed(42)

X = boston_df.drop("target", axis=1)
y = boston_df["target"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

model = RandomForestRegressor(n_estimators=100)
model.fit(X_train, y_train);

model.score(X_test, y_test)
from sklearn.metrics import r2_score

# Fill an array with y_test mean
y_test_mean = np.full(len(y_test), y_test.mean())
y_test.mean()
# Model only predicting the mean gets an R^2 score of 0
# Model predicting perfectly the correct values gets an R^2 score of 1

r2_score(y_test, y_test_mean), r2_score(y_test, y_test)
r2_score(y_test, np.zeros(len(y_test)))<jupyter_output><empty_output><jupyter_text>Mean absolute error (MAE)MAE is the average of the absolute differences between predictions and actual values. It gives you an idea of how wrong your models predictions are.<jupyter_code># Mean absolute error
from sklearn.metrics import mean_absolute_error

y_preds = model.predict(X_test)
mae = mean_absolute_error(y_test, y_preds)
mae
df = pd.DataFrame(data={"actual values": y_test,
                        "predicted values": y_preds})
df["differences"] = df["predicted values"] - df["actual values"]

df["abs differences"] = abs(df["predicted values"] - df["actual values"])
df
np.mean(df["abs differences"])<jupyter_output><empty_output><jupyter_text>Mean squared error (MSE)<jupyter_code># Mean squared error
from sklearn.metrics import mean_squared_error

y_preds = model.predict(X_test)
mse = mean_squared_error(y_test, y_preds)
mse
# Calculate MSE by hand
squared = np.square(df["differences"])
squared.mean()<jupyter_output><empty_output><jupyter_text>4.2.3 Finally using the scoring parameter<jupyter_code>from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier

np.random.seed(42)

X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

clf = RandomForestClassifier(n_estimators=100)

np.random.seed(42)
cv_acc = cross_val_score(clf, X, y, cv=5, scoring=None)
cv_acc
# Cross-validated accuracy
print(f'The cross-validated accuracy is: {np.mean(cv_acc)*100:.2f}%')
np.random.seed(42)
cv_acc = cross_val_score(clf, X, y, cv=5, scoring="accuracy")
print(f'The cross-validated accuracy is: {np.mean(cv_acc)*100:.2f}%')
# Precision
cv_precision = cross_val_score(clf, X, y, cv=5, scoring="precision")
np.mean(cv_precision)
# Recall
cv_recall = cross_val_score(clf, X, y, cv=5, scoring="recall")
np.mean(cv_recall)
# F1-Score
cv_f1 = cross_val_score(clf, X, y, cv=5, scoring="f1")
np.mean(cv_f1)<jupyter_output><empty_output><jupyter_text>How about our regression model?<jupyter_code>from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestRegressor

np.random.seed(42)

X = boston_df.drop("target", axis=1)
y = boston_df["target"]

model = RandomForestRegressor(n_estimators=100)
np.random.seed(42)
cv_r2 = cross_val_score(model, X, y, cv=5, scoring=None)
np.mean(cv_r2) # default is R^2
np.random.seed(42)
cv_r2 = cross_val_score(model, X, y, cv=5, scoring="r2")
np.mean(cv_r2)
# Mean absolute error
cv_mae = cross_val_score(model, X, y, cv=5, scoring="neg_mean_absolute_error")
np.mean(cv_mae)
# Mean squared error
cv_mse = cross_val_score(model, X, y, cv=5, scoring="neg_mean_squared_error")
np.mean(cv_mse)<jupyter_output><empty_output><jupyter_text>4.3 Using different evalution metrics as Scikit-Learn functionsClassification evaluation functions<jupyter_code>from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split

np.random.seed(42)

X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

clf = RandomForestClassifier(n_estimators=100)
clf.fit(X_train, y_train)

# Make some predictions
y_preds = clf.predict(X_test)

# Evaluate the classifier
print("Classifier metrics on the test set")
print(f"Accuracy: {accuracy_score(y_test, y_preds)*100:.2f}%")
print(f"Precision: {precision_score(y_test, y_preds)}")
print(f"Recall: {recall_score(y_test, y_preds)}")
print(f"F1: {f1_score(y_test, y_preds)}")<jupyter_output>Classifier metrics on the test set
Accuracy: 85.25%
Precision: 0.8484848484848485
Recall: 0.875
F1: 0.8615384615384615<jupyter_text>Regression evaluation functions<jupyter_code>from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split

np.random.seed(42)

X = boston_df.drop("target", axis=1)
y = boston_df["target"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

model = RandomForestRegressor(n_estimators=100)
model.fit(X_train, y_train)

# Make predictions using our regression model
y_preds = model.predict(X_test)

# Evaluate the regression model
print("Regression model metrics on the test set")
print(f"R^2: {r2_score(y_test, y_preds)}")
print(f"MAE: {mean_absolute_error(y_test, y_preds)}")
print(f"MSE: {mean_squared_error(y_test, y_preds)}")<jupyter_output>Regression model metrics on the test set
R^2: 0.8654448653350507
MAE: 2.136382352941176
MSE: 9.867437068627442<jupyter_text>5. Improving a model First predictions = baseline predictions.First model = baseline model.From a data perspective:- Could we collect more data? (generally, the more data, the better)- Could we improve our data?From a model perspective:- Is there a better model we could use?- Could we improve the current model?Hyperparameters vs. Parameters- Parameters = model find these patterns in data- Hyperparameters = settings on a model you can adjust to (potentially) improve its ability to find patternsThree ways to adjust hyperparameters:- By hand- Randomly with RandomSearchCV- Exhaustively with GridSearchCV<jupyter_code>from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier()
clf.get_params()<jupyter_output><empty_output><jupyter_text>5.1 Tuning hyperparameters by handLet's make 3 sets, training, validation and test.We're going to try and adjust:* max_depth* max_features* min_samples_leaf* min_samples_split* n_estimators<jupyter_code>def evaluate_preds(y_true, y_preds):
    """
    Performs evaluation comparison on y_true labels vs. y_pred labels
    on a classification.
    """
    accuracy = accuracy_score(y_true, y_preds)
    precision = precision_score(y_true, y_preds)
    recall = recall_score(y_true, y_preds)
    f1 = f1_score(y_true, y_preds)
    metric_dict = {"accuracy": round(accuracy, 2),
                   "precision": round(precision, 2),
                   "recall": round(recall, 2),
                   "f1": round(f1, 2)}
    print(f"Acc: {accuracy * 100:.2f}%")
    print(f"Precision: {precision:.2f}")
    print(f"Recall: {recall:.2f}")
    print(f"F1 score: {f1:.2f}")
    
    return metric_dict
from sklearn.ensemble import RandomForestClassifier

np.random.seed(42)

# Shuffle the data
heart_disease_shuffled = heart_disease.sample(frac=1)

# Split into X & y
X = heart_disease_shuffled.drop("target", axis=1)
y = heart_disease_shuffled["target"]

# Split the data into train, validation & test sets
train_split = round(0.7 * len(heart_disease_shuffled)) # 70% of data
valid_split = round(train_split + 0.15 * len(heart_disease_shuffled)) # 15% of data
X_train, y_train = X[:train_split], y[:train_split]
X_valid, y_valid = X[train_split:valid_split], y[train_split:valid_split]
X_test, y_test = X[valid_split:], y[:valid_split]

clf = RandomForestClassifier()
clf.fit(X_train, y_train)

# Make baseline predictions
y_preds = clf.predict(X_valid)

# Evaluate the classifier on validation set
baseline_metrics = evaluate_preds(y_valid, y_preds)
baseline_metrics
np.random.seed(42)

# Create a second classifier with different hyperparameters
clf_2 = RandomForestClassifier(n_estimators=10)
clf_2.fit(X_train, y_train)

# Make predictions with different hyperparameters
y_preds_2 = clf_2.predict(X_valid)

# Evalute the 2nd classsifier
clf_2_metrics = evaluate_preds(y_valid, y_preds_2)
np.random.seed(42)

# Create a third classifier with different hyperparameters
clf_3 = RandomForestClassifier(n_estimators=10, max_depth=5)
clf_3.fit(X_train, y_train)

# Make predictions with different hyperparameters
y_preds_3 = clf_3.predict(X_valid)

# Evalute the 3rd classsifier
clf_3_metrics = evaluate_preds(y_valid, y_preds_3)<jupyter_output>Acc: 84.44%
Precision: 0.82
Recall: 0.92
F1 score: 0.87<jupyter_text>5.2 Hyperparameter tuning with RandomizedSearchCV<jupyter_code>from sklearn.model_selection import RandomizedSearchCV

grid = {"n_estimators": [10, 100, 200, 500, 1000, 1200],
        "max_depth": [None, 5, 10, 20, 30],
        "max_features": ["auto", "sqrt"],
        "min_samples_split": [2, 4, 6],
        "min_samples_leaf": [1, 2, 4]}

np.random.seed(42)

# Split into X & y
X = heart_disease_shuffled.drop("target", axis=1)
y = heart_disease_shuffled["target"]

# Split into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instantiate RandomForestClassifier
clf = RandomForestClassifier(n_jobs=1)

# Setup RandomizedSearchCV
rs_clf = RandomizedSearchCV(estimator=clf,
                            param_distributions=grid, 
                            n_iter=10, # number of models to try
                            cv=5,
                            verbose=2)

# Fit the RandomizedSearchCV version of clf
rs_clf.fit(X_train, y_train);

rs_clf.best_params_
# Make predictions with the best hyperparameters
rs_y_preds = rs_clf.predict(X_test)

# Evaluate the predictions
rs_metrics = evaluate_preds(y_test, rs_y_preds)<jupyter_output>Acc: 81.97%
Precision: 0.77
Recall: 0.86
F1 score: 0.81<jupyter_text>5.3 Hyperparameter tuning with GridSearchCV<jupyter_code>grid_2 = {'n_estimators': [100, 200, 500],
          'max_depth': [None],
          'max_features': ['auto', 'sqrt'],
          'min_samples_split': [6],
          'min_samples_leaf': [1, 2]}
from sklearn.model_selection import GridSearchCV, train_test_split

np.random.seed(42)

# Split into X & y
X = heart_disease_shuffled.drop("target", axis=1)
y = heart_disease_shuffled["target"]

# Split into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Instantiate RandomForestClassifier
clf = RandomForestClassifier(n_jobs=1)

# Setup GridSearchCV
gs_clf = GridSearchCV(estimator=clf,
                      param_grid=grid_2, 
                      cv=5,
                      verbose=2)

# Fit the GridSearchCV version of clf
gs_clf.fit(X_train, y_train);
gs_clf.best_params_
gs_y_preds = gs_clf.predict(X_test)

# evaluate the predictions
gs_metrics = evaluate_preds(y_test, gs_y_preds)<jupyter_output>Acc: 78.69%
Precision: 0.74
Recall: 0.82
F1 score: 0.78<jupyter_text>Let's compare our different models metrics.<jupyter_code>compare_metrics = pd.DataFrame({"baseline": baseline_metrics,
                                "clf_2": clf_2_metrics,
                                "random search": rs_metrics,
                                "grid search": gs_metrics})

compare_metrics.plot.bar(figsize=(10, 8));<jupyter_output><empty_output><jupyter_text>6. Saving and loading trained machine learning modelsTwo ways to save and load machine learning models:* With Python's pickle module* With the joblib module  Pickle<jupyter_code>import pickle

# Save an extisting model to file
pickle.dump(gs_clf, open("models/gs_random_random_forest_model_1.pkl", "wb"))
# Load a saved model
loaded_pickle_model = pickle.load(open("models/gs_random_random_forest_model_1.pkl", "rb"))
# Make some predictions
pickle_y_preds = loaded_pickle_model.predict(X_test)
evaluate_preds(y_test, pickle_y_preds)<jupyter_output>Acc: 78.69%
Precision: 0.74
Recall: 0.82
F1 score: 0.78<jupyter_text>Joblib<jupyter_code>from joblib import dump, load

# Save model to file
dump(gs_clf, filename="models/gs_random_forest_model_1.joblib")
# Import a saved joblib model
loaded_joblib_model = load(filename="models/gs_random_forest_model_1.joblib")
# Make and evaluate joblib predictions
joblib_y_preds = loaded_joblib_model.predict(X_test)
evaluate_preds(y_test, joblib_y_preds)<jupyter_output>Acc: 78.69%
Precision: 0.74
Recall: 0.82
F1 score: 0.78<jupyter_text>7. Putting it all together!<jupyter_code>data = pd.read_csv("data/car-sales-extended-missing-data.csv")
data.head()
data.dtypes
data.isna().sum()<jupyter_output><empty_output><jupyter_text>Steps we want to do (all in one cell):1. Fill missing data2. Convert data to numbers3. Build a model on the data<jupyter_code># Getting data ready
import pandas as pd
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder

# Modelling
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split, GridSearchCV

# Setup random seed
import numpy as np
np.random.seed(42)

# Import data and drop rows with missing labels
data = pd.read_csv("data/car-sales-extended-missing-data.csv")
data.dropna(subset=["Price"], inplace=True)

# Define different features and transformer pipeline
categorical_features = ["Make", "Colour"]
categorical_transformer = Pipeline(steps=[
    ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
    ("onehot", OneHotEncoder(handle_unknown="ignore"))])

door_feature = ["Doors"]
door_transformer = Pipeline(steps=[
    ("imputer", SimpleImputer(strategy="constant", fill_value=4))
])

numeric_features = ["Odometer (KM)"]
numeric_transformer = Pipeline(steps=[
    ("imputer", SimpleImputer(strategy="mean"))
])

# Setup preprocessing steps (fill missing values, then convert to numbers)
preprocessor = ColumnTransformer(
                    transformers=[
                        ("cat", categorical_transformer, categorical_features),
                        ("door", door_transformer, door_feature),
                        ("num", numeric_transformer, numeric_features)
                    ])

# Creating a preprocessing and modelling pipeline
model = Pipeline(steps=[("preprocessor", preprocessor),
                        ("model", RandomForestRegressor())])

# Split data
X = data.drop("Price", axis=1)
y = data["Price"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Fit and score the model
model.fit(X_train, y_train)
model.score(X_test, y_test)
# It's also possible to use GridSearchCV or RandomizedSesrchCV with our Pipeline.
# Use GridSearchCV with our regression Pipeline
from sklearn.model_selection import GridSearchCV

# preprocessor -> num -> imputer -> strategy

pipe_grid = {
    "preprocessor__num__imputer__strategy": ["mean", "median"],
    "model__n_estimators": [100, 1000],
    "model__max_depth": [None, 5],
    "model__max_features": ["auto"],
    "model__min_samples_split": [2, 4]    
}

gs_model = GridSearchCV(model, pipe_grid, cv=5, verbose=2)
gs_model.fit(X_train, y_train)
gs_model.score(X_test, y_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10056, 'fraction_numerical': 0.0188, 'mean_word_length': 4.94876269035533, 'pattern_counts': {'":': 20, '<': 124, '<?xml version=': 0, '>': 130, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '33', 'dir': 'jupyter-structured-clean-dedup', 'id': '334166', 'license': 'MIT', 'n_tokens_mistral': 13242, 'n_tokens_neox': 12462, 'n_words': 3328, 'path': 'intro-to-scikit-learn.ipynb', 'repo_name': 'DeekshithShetty/my-hands-on-ml', 'size': '37500'}