<jupyter_start><jupyter_text>Exercises chapter 3<jupyter_code>import sklearn 
import numpy as np
import pandas as pd
import os

np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "classification"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)
os.makedirs(IMAGES_PATH, exist_ok=True)

# Helper function to save the images
def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)<jupyter_output><empty_output><jupyter_text>1. Build a MINST classifier with 97% accuracy on test set Hint: use `KNeighborsClassifier` and do grid search over hyperparameters (`weights` and `n_neighbors`)<jupyter_code># Importing the famous MINST dataset
from sklearn.datasets import fetch_openml
mnist = fetch_openml('mnist_784', version=1, as_frame=False)
# Pre-processing + splitting into test and train
X, y = mnist["data"], mnist["target"]
y = y.astype(np.uint8) #converting response variable to integer
X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]
# Instantiating the KNeighborsClassifier
from sklearn.neighbors import KNeighborsClassifier
my_knn = KNeighborsClassifier()
# Grid Search over hyperparameters (weights and n_neighbors)
from sklearn.model_selection import GridSearchCV

param_grid_knn = {
    'weights': ('uniform', 'distance'),
    'n_neighbors': list(range(1, 10))
}
grid = GridSearchCV(my_knn, param_grid_knn, cv = 5, scoring = 'accuracy')
grid.fit(X_train, y_train)
# examine the best model
# Source of the code: https://www.ritchieng.com/machine-learning-efficiently-search-tuning-param/ 
# Single best score achieved across all params (k)
print(grid.best_score_)

# Dictionary containing the parameters (k) used to generate that score
print(grid.best_params_)

# Actual model object fit with those best parameters
# Shows default parameters that we did not specify
print(grid.best_estimator_)
# Instantiating and fitting the model with the best parameters
optimal_knn = KNeighborsClassifier(
    n_neighbors=4, 
    weights='distance'
)

optimal_knn.fit(X_train, y_train)
# Evaluate *accuracy* on test data
y_knn_pred = optimal_knn.predict(X_test)

from sklearn.metrics import accuracy_score
accuracy_score(y_test, y_knn_pred)<jupyter_output><empty_output><jupyter_text>2. Writing function that shifts images, then using it to augment the dataset and train the model on the augmented data<jupyter_code>from scipy.ndimage import shift

#Example:
#shift(image, [2, 1], cval=0) shifts the image 2 pixels down and 1 pixel to the right

# Creating the function
def shift_image(image, direction):    
    if direction == 'left':
        shifting_arr = [0, -1]
    elif direction == 'right':
        shifting_arr = [0, 1]
    elif direction == 'up':
        shifting_arr = [-1, 0]
    elif direction == 'down':
        shifting_arr = [1, 0]
        
    return(shift(image.reshape(28, 28), shifting_arr, cval=0))

# How can I access an image?? Look for an example
# X_train[0]
# Augmenting the dataset
x_up = np.array([shift_image(x, direction='up') for x in X_train])
x_down = np.array([shift_image(x, direction='down') for x in X_train])
x_right = np.array([shift_image(x, direction='right') for x in X_train])
x_left = np.array([shift_image(x, direction='left') for x in X_train])
X_enlarged = np.concatenate(
    (X_train,
    x_up.reshape(60000, 784),
    x_down.reshape(60000, 784),
    x_right.reshape(60000, 784),
    x_left.reshape(60000, 784))
)
# Enlarge y too!
# np.tile(np.array([1,2]), 2)
y_enlarged = np.tile(y_train, 5)
# Now train the model!
from sklearn.neighbors import KNeighborsClassifier
optimal_knn2 = KNeighborsClassifier(
    n_neighbors=4, 
    weights='distance'
)

optimal_knn2.fit(X_enlarged, y_enlarged)
# Evaluate *accuracy* on test data
y_knn2_pred = optimal_knn2.predict(X_test)

from sklearn.metrics import accuracy_score
accuracy_score(y_test, y_knn2_pred)<jupyter_output><empty_output><jupyter_text>The accuracy improved, but just a little bit. BTW, it's literally the same metric value that appears on the solutions, so my solution is fine. 3. Tackle the Titatnic dataset. The goal is to predict whether or not a passenger survived based on attributes such as their age, sex, passenger class, where they embarked and so on.<jupyter_code># loading the data
import urllib.request

TITANIC_PATH = os.path.join("datasets", "titanic")
DOWNLOAD_URL = "https://raw.githubusercontent.com/ageron/handson-ml2/master/datasets/titanic/"

def fetch_titanic_data(url=DOWNLOAD_URL, path=TITANIC_PATH):
    if not os.path.isdir(path):
        os.makedirs(path)
    for filename in ("train.csv", "test.csv"):
        filepath = os.path.join(path, filename)
        if not os.path.isfile(filepath):
            print("Downloading", filename)
            urllib.request.urlretrieve(url + filename, filepath)

fetch_titanic_data()    
import pandas as pd

def load_titanic_data(filename, titanic_path=TITANIC_PATH):
    csv_path = os.path.join(titanic_path, filename)
    return pd.read_csv(csv_path)
train_data = load_titanic_data("train.csv")
test_data = load_titanic_data("test.csv")
train_data.head()<jupyter_output><empty_output><jupyter_text>The attributes have the following meaning:* **PassengerId**: a unique identifier for each passenger* **Survived**: that's the target, 0 means the passenger did not survive, while 1 means he/she survived.* **Pclass**: passenger class.* **Name**, **Sex**, **Age**: self-explanatory* **SibSp**: how many siblings & spouses of the passenger aboard the Titanic.* **Parch**: how many children & parents of the passenger aboard the Titanic.* **Ticket**: ticket id* **Fare**: price paid (in pounds)* **Cabin**: passenger's cabin number* **Embarked**: where the passenger embarked the Titanic Let's explicitly set the `PassengerId` column as the index column:<jupyter_code>train_data = train_data.set_index("PassengerId")
test_data = test_data.set_index("PassengerId")
train_data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 891 entries, 1 to 891
Data columns (total 11 columns):
 #   Column    Non-Null Count  Dtype  
---  ------    --------------  -----  
 0   Survived  891 non-null    int64  
 1   Pclass    891 non-null    int64  
 2   Name      891 non-null    object 
 3   Sex       891 non-null    object 
 4   Age       714 non-null    float64
 5   SibSp     891 non-null    int64  
 6   Parch     891 non-null    int64  
 7   Ticket    891 non-null    object 
 8   Fare      891 non-null    float64
 9   Cabin     204 non-null    object 
 10  Embarked  889 non-null    object 
dtypes: float64(2), int64(4), object(5)
memory usage: 83.5+ KB<jupyter_text>It's better to ignore Name and Ticket because it's difficult to convert them into useful information. It's necessary to implemennt an imputation strategy for Age missing values. Idea: input the median according to Sex and Pclass.<jupyter_code>X_train = train_data.drop("Survived" ,axis= 1)
y_train = train_data.Survived
X_train.info()
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer

num_attribs = ['Age', 'SibSp', 'Parch', 'Fare']
num_indexes = X_train.columns.get_indexer(num_attribs)
cat_attribs = ['Sex', 'Embarked', 'Pclass']
cat_indexes = X_train.columns.get_indexer(cat_attribs)

num_pipeline = Pipeline([
    #('imputer', GroupImputer(group_cols = ['Sex', 'Pclass'], target = "Age", metric="median")),
    #('imputer2', GroupImputer(group_cols = ['Sex', 'Pclass'], target = "Fare", metric="median")),
    ("selector", ColumnTransformer([
        ("selector", "passthrough", num_indexes)
    ], remainder="drop")),
    ('imputer', SimpleImputer()),
    ('std_scaler', StandardScaler())
])

cat_pipeline = Pipeline([("selector", ColumnTransformer([
        ("selector", "passthrough", cat_indexes)
    ], remainder="drop")),
                        ('oneHot',OneHotEncoder(categories='auto'))])
# Combine pipeline for numeric and pipeline for categorical
from sklearn.pipeline import FeatureUnion

full_pipeline = FeatureUnion(transformer_list=[
    ('num_pipeline', num_pipeline),
    ('cat_pipeline', cat_pipeline)])

X_train2 = full_pipeline.fit_transform(X_train)<jupyter_output><empty_output><jupyter_text>At this point, I have succesfully:- Imputed missing values based on group means- Scaled numeric features- One-hot-encoded categorical variablesNext: do `RandomSearchCV` with Random Forest model :)<jupyter_code>from sklearn.ensemble import RandomForestClassifier

forest_clas = RandomForestClassifier(random_state=42)<jupyter_output><empty_output><jupyter_text>Looking at default parameters<jupyter_code>print(forest_clas.get_params())
from sklearn.model_selection import RandomizedSearchCV
# Tutorial I followed here: 
# https://towardsdatascience.com/hyperparameter-tuning-the-random-forest-in-python-using-scikit-learn-28d2aa77dd74

# Number of trees in random forest
n_estimators = [int(x) for x in np.linspace(start = 100, stop = 2000, num = 15)]

# Number of features to consider at every split
max_features = ['auto', 'sqrt', 'log2']

# Maximum number of levels in tree
max_depth = [int(x) for x in np.linspace(6, 110, num = 11)]
max_depth.append(None)

# Minimum number of samples required to split a node
min_samples_split = [2, 5, 10]

# Minimum number of samples required at each leaf node
min_samples_leaf = [1, 2, 4]

# Method of selecting samples for training each tree
bootstrap = [True, False]

# Create the random grid
random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}
# Use the random grid to search for best hyperparameters

# Random search of parameters, using 3 fold cross validation, 
# search across 100 different combinations, and use all available cores
rf_random = RandomizedSearchCV(estimator = forest_clas,
                               param_distributions = random_grid,
                               n_iter = 100,
                               cv = 5,
                               verbose=1,
                               random_state=42,
                               n_jobs = -1)

# Fit the random search model
rf_random.fit(X_train2, y_train)
rf_random.best_params_<jupyter_output><empty_output><jupyter_text>Now I can narrow down the hyperparameter search around the parameters found with `RandomizedSearchCV`<jupyter_code>rf_random.best_score_
from sklearn.model_selection import GridSearchCV

# Create the parameter grid based on the results of random search 
param_grid = {
    'bootstrap': [True],
    'max_depth': [56, 57, 58, 59, 60],
    'max_features': ['auto'],
    'min_samples_leaf': [1, 2, 3],
    'min_samples_split': [3, 4, 5, 6, 7],
    'n_estimators': [760, 770, 780, 790, 800]
}

# Instantiate the grid search model
grid_search = GridSearchCV(estimator = forest_clas, param_grid = param_grid, 
                          cv = 5, n_jobs = -1, verbose = 1)

grid_search.fit(X_train2, y_train)
grid_search.best_params_
grid_search.best_score_
# Now I should train the model on the whole training dataset
final_rf = RandomForestClassifier(random_state=42)
final_rf.set_params(**grid_search.best_params_)

final_rf.fit(X_train2, y_train)
test_data.info()
X_test = full_pipeline.transform(test_data)
y_pred = final_rf.predict(X_test)
y_pred[:5]
# Extract PassengerId
passengerid = test_data.reset_index().PassengerId
type(passengerid[:5])
data = {'Survived':y_pred}
 
# Creates pandas DataFrame.
df = pd.DataFrame(data, index = passengerid)
df.to_csv('outputs/titanic_pred.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10700261780104713, 'fraction_numerical': 0.022823952879581152, 'mean_word_length': 4.44543429844098, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '351478', 'license': 'Apache-2.0', 'n_tokens_mistral': 4241, 'n_tokens_neox': 3980, 'n_words': 1111, 'path': '03a_exercises.ipynb', 'repo_name': 'franciscoyira/handson-ml2', 'size': '12224'}