<jupyter_start><jupyter_text>Trying to normalize the data through log transformation<jupyter_code>#Removing outliers 
# streams_features = streams_features[streams_features["playCount"]<5]
# streams_features = streams_features[streams_features["playCount"]>=0.5]
# streams_features.count()
# sns.histplot(data = streams_features, x = "playCount", stat = "count", kde = True)
#Cap most played songs at 10
streams_features["playCount"].where(streams_features["playCount"] <= 10, 10, inplace=True)
import numpy as np
no_zeroes = streams_features[streams_features["playCount"]>0]
log_dist = np.log(no_zeroes["playCount"])
sns.histplot(data = log_dist, stat = "count", kde = True)
# fig, axs = plt.subplots(1,3,figsize=(12,5))
# sns.histplot(data = no_zeroes["playCount"], stat = "count", kde = True, ax = axs[0])
# sns.histplot(data = log_dist2, stat = "count", kde = True, ax = axs[1])
# sns.histplot(data = sqrt_dist, stat = "count", kde = True, ax = axs[2])
streams_features = streams_features[streams_features["playCount"]>0]
streams_features["playCount"] = np.log(no_zeroes["playCount"])
streams_features["playCount"].skew() #After  transformation
streams_features["playCount"].count()
corr_matrix = streams_features.corr()
corr_matrix["playCount"].sort_values(ascending=False)
import numpy as np
mask = np.triu(np.ones_like(corr_matrix, dtype=bool))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(corr_matrix, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})
X = streams_features.drop(["artistName", "trackName", "minutesTotal", "trackId", "playCount"], axis = 1)
y = streams_features["playCount"]
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import PowerTransformer
pt = PowerTransformer(method='yeo-johnson')
standard_scalar  = StandardScaler()
min_max_scaler = MinMaxScaler()
X = min_max_scaler.fit_transform(X)
#X = pt.fit_transform(X)
# pt = PowerTransformer(method='yeo-johnson')
# data = pt.fit_transform(X)
# # convert the array back to a dataframe
# dataset = pd.DataFrame(data)
# # histograms of the variables
# dataset.hist()
# plt.show()
len(X)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=42)
X_train, X_valid, y_train, y_valid = train_test_split(X_train,y_train, test_size=0.1, random_state=42)
X_lengths = {"X_train" : len(X_train), "X_test" : len(X_test), "X_valid" : len(X_valid)}
y_lengths = {"y_train" : len(y_train), "y_test" : len(y_test), "y_valid" : len(y_valid)}
X_lengths, y_lengths
import tensorflow as tf
from tensorflow import keras
model_params = {
    "optimizer":keras.optimizers.Adam(
    learning_rate=0.00001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=True,
    name='Adam'), 
    "loss": tf.keras.losses.MeanAbsoluteError(),
    "hidden_activation":"relu",
    "output_activation":"relu",  
    #"loss":keras.losses.Huber(), 
    "initializer" : tf.keras.initializers.HeNormal()
}
reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.75,
                              patience=2, min_lr=0.00001)

early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=8,min_delta = 0.0001)
model = keras.models.Sequential([
    keras.layers.Dense(50, activation=model_params["hidden_activation"], input_shape=X_train.shape[1:], 
                       kernel_initializer=model_params["initializer"]),
    #keras.layers.Dense(50, activation=model_params["hidden_activation"]),
    #keras.layers.Dense(30, activation=model_params["hidden_activation"]),
    #keras.layers.Dense(25, activation=model_params["hidden_activation"],kernel_initializer=model_params["initializer"]),
    keras.layers.Dense(20, activation=model_params["hidden_activation"],kernel_initializer=model_params["initializer"]),
    keras.layers.Dense(1, activation=model_params["output_activation"]),
])

model.compile(loss=model_params["loss"], 
              optimizer=model_params["optimizer"])
loss_min = {}
val_loss_min = {}
for i in [250]:
    history = model.fit(X_train, 
                    y_train, 
                    epochs=i, 
                    verbose = 1, 
                    validation_data=(X_valid, y_valid),
                    callbacks = [early_stopping, reduce_lr],
                    batch_size = 32)
    loss_min[i] = min(history.history["loss"])
    val_loss_min[i] = min(history.history["val_loss"])
plot_loss(history, exp = True)
np.exp(model.evaluate(x = X_test,  y = y_test, verbose=1, batch_size = 16))<jupyter_output>104/104 [==============================] - 0s 2ms/step - loss: 1.7015
{'fraction_non_alphanumeric': 0.14710778193924262, 'fraction_numerical': 0.02184769038701623, 'mean_word_length': 5.341688654353562, 'pattern_counts': {'":': 6, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '464169', 'license': 'MIT', 'n_tokens_mistral': 1755, 'n_tokens_neox': 1649, 'n_words': 292, 'path': 'ML/Training/Exploration/ml_exploration_log_transform.ipynb', 'repo_name': 'jmoro0408/Spotify_Analysis_2.0', 'size': '4806'}