<jupyter_start><jupyter_text>CZ4042 Neural Networks & Deep Learning Assignment - 1: Part B, Question 3> <NAME>  > U1822549K  > School of Computer Science and Engineering  > Nanyang Technological University, Singapore    Imports<jupyter_code># Setting the seed here is sufficient. 
# If you don't plan to use these starter code, make sure you add this cell.

SEED = 42

import os
os.environ['TF_CUDNN_DETERMINISTIC'] = '1'

import random 
random.seed(SEED)

import numpy as np
np.random.seed(SEED)

import tensorflow as tf
tf.random.set_seed(SEED)
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers import IntegerLookup
from tensorflow.keras.layers import Normalization
from tensorflow.keras.layers import StringLookup
from tensorflow.keras.layers import Embedding, Reshape, Concatenate
from tensorflow.keras.metrics import RootMeanSquaredError
from tensorflow.keras.callbacks import EarlyStopping
import pandas as pd
from tensorflow.keras import backend as K
import sys
import operator
print(tf.__version__)
# Supress any warnings for clean presentation
import warnings
warnings.filterwarnings("ignore")
df = pd.read_csv('hdb_price_prediction.csv')
df.head()
def dataframe_to_dataset(dataframe):
    dataframe = dataframe.copy()
    labels = dataframe.pop("resale_price")
    ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))
    ds = ds.shuffle(buffer_size=len(dataframe))
    return ds


def encode_numerical_feature(feature, name, dataset):
    # Create a Normalization layer for our feature
    normalizer = Normalization()

    # Prepare a Dataset that only yields our feature
    feature_ds = dataset.map(lambda x, y: x[name])
    feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1))

    # Learn the statistics of the data
    normalizer.adapt(feature_ds)

    # Normalize the input feature
    encoded_feature = normalizer(feature)
    return encoded_feature


def encode_categorical_feature(feature, name, dataset, is_string):
    lookup_class = StringLookup if is_string else IntegerLookup
    # Create a lookup layer which will turn strings into integer indices
    lookup = lookup_class(output_mode="int") # NOTE: as mentioned in the question paper, this actually does one-hot encoding. You could replace 'binary' with 'one_hot' if you wish to.

    # Prepare a Dataset that only yields our feature
    feature_ds = dataset.map(lambda x, y: x[name])
    feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1))

    # Learn the set of possible string values and assign them a fixed integer index
    lookup.adapt(feature_ds)

    # Turn the string input into integer indices
    encoded_feature = lookup(feature)
    return encoded_feature

# Split data
train_dataframe = df[df['year'] <= 2020]
val_dataframe = df[df['year'] > 2020]

# Generate tf.data.Dataset objects for each dataframe
train_ds = dataframe_to_dataset(train_dataframe)
val_ds = dataframe_to_dataset(val_dataframe)

# Batch the datasets
train_ds = train_ds.batch(128)
val_ds = val_ds.batch(128)
# Categorical features (integers)
month = keras.Input(shape=(1,), name="month", dtype="int64")

# Categorical features encoded (integers)
month_encoded = encode_categorical_feature(month, "month", train_ds, False)

# Categorical feature (String)
flat_model_type = keras.Input(shape=(1,), name="flat_model_type", dtype="string")
storey_range = keras.Input(shape=(1,), name="storey_range", dtype="string")

# Categorical feature encoded (String)
flat_model_type_encoded = encode_categorical_feature(flat_model_type, "flat_model_type", train_ds, True)
storey_range_encoded = encode_categorical_feature(storey_range, "storey_range", train_ds, True)

# Numerical features
dist_to_nearest_stn = keras.Input(shape=(1,), name="dist_to_nearest_stn")
dist_to_dhoby = keras.Input(shape=(1,), name="dist_to_dhoby")
degree_centrality = keras.Input(shape=(1,), name="degree_centrality")
eigenvector_centrality = keras.Input(shape=(1,), name="eigenvector_centrality")
remaining_lease_years = keras.Input(shape=(1,), name="remaining_lease_years")
floor_area_sqm = keras.Input(shape=(1,), name="floor_area_sqm")

# Numerical Features Encoded
dist_to_nearest_stn_encoded = encode_numerical_feature(dist_to_nearest_stn, "dist_to_nearest_stn", train_ds)
dist_to_dhoby_encoded = encode_numerical_feature(dist_to_dhoby, "dist_to_dhoby", train_ds)
degree_centrality_encoded = encode_numerical_feature(degree_centrality, "degree_centrality", train_ds)
eigenvector_centrality_encoded = encode_numerical_feature(eigenvector_centrality, "eigenvector_centrality", train_ds)
remaining_lease_years_encoded = encode_numerical_feature(remaining_lease_years, "remaining_lease_years", train_ds)
floor_area_sqm_encoded = encode_numerical_feature(floor_area_sqm, "floor_area_sqm", train_ds)

# Combining all inputs
all_inputs = [
    month,
    flat_model_type,
    storey_range,
    dist_to_nearest_stn,
    dist_to_dhoby,
    degree_centrality,
    eigenvector_centrality,
    remaining_lease_years,
    floor_area_sqm
]
embedding_layers = []

# All Categorial Variables
categorial_variables = ["month", "flat_model_type", "storey_range"]

# All Categorial Variables (encoded)
categorial_variables_encoded = [month_encoded, flat_model_type_encoded, storey_range_encoded]

# Generate an embedding for each categorial variable and store it
for categorial_variable, categorial_variable_encoded in zip(categorial_variables, categorial_variables_encoded):
    # Output dimensions specified in the assignment manual
    unique_categories  = train_dataframe[categorial_variable].nunique()
    embedding_size = np.floor((unique_categories)/2)
    embedding_size = int(embedding_size)

    # Input Dimensions
    vocab  = unique_categories + 1
    
    # Embedding Layer
    embedding = Embedding(input_dim = vocab, output_dim = embedding_size)(categorial_variable_encoded)

    # Reshaping such that it can be concatenated
    embedding = Reshape((-1,))(embedding)
    embedding_layers.append(embedding)
# Combining all Encoded inputs
all_features = layers.concatenate(
    [
        embedding_layers[0],
        embedding_layers[1],
        embedding_layers[2],
        dist_to_nearest_stn_encoded,
        dist_to_dhoby_encoded,
        degree_centrality_encoded,
        eigenvector_centrality_encoded,
        remaining_lease_years_encoded,
        floor_area_sqm_encoded,
    ]
)<jupyter_output><empty_output><jupyter_text>R2 Value Metric<jupyter_code>def coeff_determination(y_true, y_pred):
    # Custom Metric to calculate R2 Value
    # Source: https://jmlb.github.io/ml/2017/03/20/CoeffDetermination_CustomMetric4Keras/
    SS_res =  K.sum(K.square(y_true-y_pred)) 
    SS_tot = K.sum(K.square(y_true - K.mean(y_true)))
    return (1 - SS_res/(SS_tot + K.epsilon()))<jupyter_output><empty_output><jupyter_text>Part A: Introducing Early Stopping<jupyter_code>def early_stopping_callback(metric, mode, patience, verbose):
    # Early Stopping if the model does not improve
    return EarlyStopping(monitor=metric, mode=mode, verbose=verbose, patience=patience)<jupyter_output><empty_output><jupyter_text>2-layer Feedforward Neural Network<jupyter_code>def getModel(features, first_layer_activation, second_layer_activation):
    # Inputs -> Encoded Features (Concatenated) -> Dense -> Dense (Output)
    x = layers.Dense(10, activation=first_layer_activation)(features)
    output = layers.Dense(1, activation=second_layer_activation)(x)
    model = keras.Model(all_inputs, output)
    return model

def getOptimizer(lr):
    # Adam Optimizer (Default Parameters) -> Stochastic Gradient Descent
    optimizer = keras.optimizers.Adam(learning_rate=lr)
    return optimizer<jupyter_output><empty_output><jupyter_text>Model Training* **Loss Function**: Mean Squared Error* **Learning Rate**: 0.05* **Epochs**: 100* **Batch Size**: 128<jupyter_code>loss = "mse"
metrics = [RootMeanSquaredError(), coeff_determination]
optimizer = getOptimizer(0.05)

model = getModel(all_features, "relu", "linear")
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
keras.utils.plot_model(model, show_shapes=True, rankdir="LR")
epochs = 100
verbose = 1
callbacks = [early_stopping_callback('val_loss', 'min', 10, verbose)]
batch_size = 128

history = model.fit(train_ds, epochs=epochs, validation_data=val_ds, verbose=verbose, callbacks = callbacks, batch_size=batch_size)<jupyter_output>Epoch 1/100
683/683 [==============================] - 4s 3ms/step - loss: 48933765120.0000 - root_mean_squared_error: 221209.7812 - coeff_determination: -1.0547 - val_loss: 12630631424.0000 - val_root_mean_squared_error: 112386.0781 - val_coeff_determination: 0.5132
Epoch 2/100
683/683 [==============================] - 3s 3ms/step - loss: 8245698560.0000 - root_mean_squared_error: 90805.8281 - coeff_determination: 0.6486 - val_loss: 10499447808.0000 - val_root_mean_squared_error: 102466.8125 - val_coeff_determination: 0.5936
Epoch 3/100
683/683 [==============================] - 3s 3ms/step - loss: 7232543744.0000 - root_mean_squared_error: 85044.3672 - coeff_determination: 0.6911 - val_loss: 9285679104.0000 - val_root_mean_squared_error: 96362.2266 - val_coeff_determination: 0.6393
Epoch 4/100
683/683 [==============================] - 3s 3ms/step - loss: 6520332288.0000 - root_mean_squared_error: 80748.5781 - coeff_determination: 0.7214 - val_loss: 8084582400.0000 - val_root_mean_s[...]<jupyter_text>Part B: Recursive Feature Elimination  Helper Functions<jupyter_code>def get_model_features(binary_vector_mask):
    # Removing the features not required by using the Binary Vector Mask
    features = all_features.copy()

    # Getting Indices where value is 0 i.e. feature is not required
    indices = [i for i, x in enumerate(binary_vector_mask) if x == 0]

    # Reverse the list to preserve index locations
    for index in sorted(indices, reverse=True):
        del features[index]
        
    return features

experiment_df = pd.DataFrame(columns=['Feature List', 'RMSE', 'Val Loss', 'Epoch Num.', 'R2'])
best_features_df = pd.DataFrame(columns=['Features'])

# Recreating all features for reference
all_features = []
all_features.extend(embedding_layers)
all_features.extend(
    [dist_to_nearest_stn_encoded,
    dist_to_dhoby_encoded,
    degree_centrality_encoded,
    eigenvector_centrality_encoded,
    remaining_lease_years_encoded,
    floor_area_sqm_encoded])

feature_name_converter = {
    "reshape/Reshape:0": "month",
    "reshape_1/Reshape:0": "flat_model_type",
    "reshape_2/Reshape:0": "storey_range",
    "normalization/truediv:0": "distance_to_nearest_stn",
    "normalization_1/truediv:0": "dist_to_dhoby",
    "normalization_2/truediv:0": "degree_centrality",
    "normalization_3/truediv:0": "eigenvector_centrality",
    "normalization_4/truediv:0": "remaining_lease_years",
    "normalization_5/truediv:0": "floor_area_sqm"
}

def record_experiment(features, rmse, val_loss, r2, epoch_num):
    all_missing_features = []

    full_features =[]
    for x in all_features:
        full_features.append(x.name)

    reduced_features =[]
    for x in features:
        reduced_features.append(x.name)

    missing_features = [item for item in full_features if item not in reduced_features]
    for missing_feature in missing_features:
        all_missing_features.append(feature_name_converter[missing_feature])
    
    feature_list = 'Feature(s) Removed: ' + ' , '.join(all_missing_features)
    new_entry = {
        'Feature List': feature_list,
        'RMSE': rmse,
        'Val Loss': val_loss,
        'Epoch Num.': epoch_num,
        'R2': r2
        }
    return new_entry<jupyter_output><empty_output><jupyter_text>2-layer Feedforward Neural Network<jupyter_code>def get_model_with_custom_features(features, first_layer_activation, second_layer_activation):
    # Inputs -> Encoded Features -> Concatenated -> Dense -> Dense (Output)
    x = Concatenate()(features)
    x = layers.Dense(10, activation=first_layer_activation)(x)
    output = layers.Dense(1, activation=second_layer_activation)(x)
    model = keras.Model(all_inputs, output)
    return model<jupyter_output><empty_output><jupyter_text>RFE Algorithm<jupyter_code># Each index represents a feature. 1 means feature is present. O means feature is not present.
# Initally all features are present. 
binary_vector_mask = [1, 1, 1, 1, 1, 1, 1, 1, 1]

# Reference Val Loss is the minimum of the previous model with all 9 features. 
lastLoss = min(history.history['val_loss'])

histories = {}
iterationCount = 0
layer_with_min_loss = None

# Model Metrics
loss = "mse"
metrics = [RootMeanSquaredError(), coeff_determination]
optimizer = getOptimizer(0.05)
epochs = 100
verbose = 0
callbacks = [early_stopping_callback('val_loss', 'min', 10, verbose)]

# Algorithm
while(sum(binary_vector_mask) > 0):
    # While there are still features available to remove
    iterationCount += 1

    if(layer_with_min_loss is not None):
        # Set a very high value to the loss of removal of the last layer such that we do not remove it again
        histories[layer_with_min_loss] = sys.maxsize

    print("Iteration: %0.0f" % iterationCount)

    for index, vector in enumerate(binary_vector_mask):
        # Iterate through all features
        if(vector == 1):
            # Only train model if feature is present
            
            # Get filtered features, i.e. only those features who are represented by 1 in the binary mask
            binary_vector_mask[index] = 0
            print("Binary Mask: ", binary_vector_mask)
            features = get_model_features(binary_vector_mask)
            binary_vector_mask[index] = 1

            # Compile & Train Model
            model_ = get_model_with_custom_features(features, "relu", "linear")
            model_.compile(loss=loss, optimizer=optimizer, metrics=metrics)
            history_ = model_.fit(train_ds, epochs=epochs, validation_data=val_ds, callbacks=callbacks, verbose=verbose)

            # Store trained model metrics
            epoch_num = np.argmin(history_.history['val_loss'])
            min_loss = history_.history['val_loss'][epoch_num]
            min_rmse = history_.history['val_root_mean_squared_error'][epoch_num]
            r2 = history_.history['val_coeff_determination'][epoch_num]

            # Store in a Pandas Dataframe for future Reference
            experiment_df = experiment_df.append(record_experiment(features, min_rmse, min_loss, r2, epoch_num), ignore_index = True)
            histories[index] = min_loss

            print("Validation Loss: %0.2f" % min_loss)
            print()
            
            K.clear_session()

    # Get Reference Loss
    base_model_val_loss = lastLoss

    # Get Layer Index whose removal leads to minimum val loss value
    layer_with_min_loss = min(histories.items(), key=operator.itemgetter(1))[0]
 
    if(base_model_val_loss > min(histories.values())):
        # Remove feature if new loss is lower
        binary_vector_mask[layer_with_min_loss] = 0
        lastLoss = histories[layer_with_min_loss]
    else:
        # We have obtained the best model
        # Break out of the loop
        print("Best Model Achieved.")
        break<jupyter_output>Iteration: 1
Binary Mask:  [0, 1, 1, 1, 1, 1, 1, 1, 1]
Validation Loss: 5292457472.00

Binary Mask:  [1, 0, 1, 1, 1, 1, 1, 1, 1]
Validation Loss: 5548882432.00

Binary Mask:  [1, 1, 0, 1, 1, 1, 1, 1, 1]
Validation Loss: 6061371392.00

Binary Mask:  [1, 1, 1, 0, 1, 1, 1, 1, 1]
Validation Loss: 5918700544.00

Binary Mask:  [1, 1, 1, 1, 0, 1, 1, 1, 1]
Validation Loss: 8208735744.00

Binary Mask:  [1, 1, 1, 1, 1, 0, 1, 1, 1]
Validation Loss: 4786109952.00

Binary Mask:  [1, 1, 1, 1, 1, 1, 0, 1, 1]
Validation Loss: 5528730112.00

Binary Mask:  [1, 1, 1, 1, 1, 1, 1, 0, 1]
Validation Loss: 5854488576.00

Binary Mask:  [1, 1, 1, 1, 1, 1, 1, 1, 0]
Validation Loss: 5517011456.00

Iteration: 2
Binary Mask:  [0, 1, 1, 1, 1, 0, 1, 1, 1]
Validation Loss: 4240435456.00

Binary Mask:  [1, 0, 1, 1, 1, 0, 1, 1, 1]
Validation Loss: 5790810624.00

Binary Mask:  [1, 1, 0, 1, 1, 0, 1, 1, 1]
Validation Loss: 6347803136.00

Binary Mask:  [1, 1, 1, 0, 1, 0, 1, 1, 1]
Validation Loss: 5914918912.00

Binary Mask:[...]<jupyter_text>Experiment Results<jupyter_code>pd.options.display.max_colwidth = 100
experiment_df[['Feature List', 'RMSE']]<jupyter_output><empty_output><jupyter_text>Part C: Model Performance Comparison<jupyter_code>val_rmse = history.history['val_root_mean_squared_error']
val_r2 = history.history['val_coeff_determination']
index_min = np.argmin(val_rmse) 

# Printing
print("--------- Model with all 9 input features ------------")
print("Epoch: " + str(index_min+1) + "nd Epoch")
print("Root Mean Squared Error: %0.3f" % val_rmse[index_min])
print("R2 (Coefficient of Determination): %0.3f" % val_r2[index_min])
print("------------------------------------------------------")

print()

min_val_rmse = experiment_df['RMSE'].min()
feature_removed_row = experiment_df.loc[experiment_df['RMSE'] == min_val_rmse]

print("--------- Model with", binary_vector_mask.count(1), "input features ----------------")
print("Epoch: " + str(feature_removed_row["Epoch Num."].item()) + "th Epoch")
print("Root Mean Squared Error: %0.3f" % min_val_rmse)
print("R2 (Coefficient of Determination): %0.3f" % feature_removed_row["R2"].item())
print(feature_removed_row["Feature List"].item())
print("------------------------------------------------------")<jupyter_output>--------- Model with all 9 input features ------------
Epoch: 32nd Epoch
Root Mean Squared Error: 73813.875
R2 (Coefficient of Determination): 0.789
------------------------------------------------------

--------- Model with 7 input features ----------------
Epoch: 20th Epoch
Root Mean Squared Error: 64971.500
R2 (Coefficient of Determination): 0.836
Feature(s) Removed: degree_centrality , eigenvector_centrality
------------------------------------------------------<jupyter_text>Part D: Examine the change in Model Performance By using Recursive Feature Elimination (RFE), we are able to identiy features whose elimination leads to a better model performance (lower validation loss). Removal of features `degree_centrality` and `eigenvector_centrality` leads to lower `Root Mean Sequared Error` and higher `R2 (Coefficient of Determination)`.       Ranking of features based on experiments<jupyter_code>experiments_eval = experiment_df.iloc[:9]
experiments_eval = experiments_eval.sort_values("RMSE")
experiments_eval = experiments_eval.drop(experiments_eval.columns[[2, 3, 4]], 1)
experiments_eval.reset_index(drop=True, inplace=True)
experiments_eval['Feature List'] = experiments_eval['Feature List'].map(lambda x: x.lstrip('Feature(s) Removed'))
experiments_eval.index = np.arange(1, len(experiments_eval) + 1)
experiments_eval<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12197195070123247, 'fraction_numerical': 0.039630259243518914, 'mean_word_length': 4.4707933740191805, 'pattern_counts': {'":': 9, '<': 44, '<?xml version=': 0, '>': 58, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '136538', 'license': 'MIT', 'n_tokens_mistral': 6729, 'n_tokens_neox': 6112, 'n_words': 1492, 'path': 'PartB_Q3.ipynb', 'repo_name': 'guptajay/CZ4042-Assignment-1', 'size': '18827'}