<jupyter_start><jupyter_text>Fake Banknote detection<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from IPython.display import display<jupyter_output><empty_output><jupyter_text>Data Exploratory<jupyter_code>df = pd.read_csv('banknotes.csv')

display(df.info())
display(df.head())
display(df.describe())

display(df['class'].value_counts())
sns.pairplot(df, hue="class")<jupyter_output>/home/dangkhoa/.conda/envs/py36/lib/python3.6/site-packages/scipy/stats/stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval
/home/dangkhoa/.conda/envs/py36/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:488: RuntimeWarning: invalid value encountered in true_divide
  binned = fast_linbin(X, a, b, gridsize) / (delta * nobs)
/home/dangkhoa/.conda/envs/py36/lib/python3.6/site-packages/statsmodels/nonparametric/kdetools.py:34: RuntimeWarning: invalid value encountered in double_scalars
  FAC1 = 2*(np.pi*bw/RANGE)**2<jupyter_text>Training data<jupyter_code>from sklearn.preprocessing import scale

X = scale(df.drop('class', axis=1).values)
y = df['class'].values

display(X.shape)
display(y.shape)<jupyter_output><empty_output><jupyter_text>Baseline model  - Random Forest<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

model = RandomForestClassifier()
cross_val_score(model, X, y)<jupyter_output>/home/dangkhoa/.conda/envs/py36/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.
  from numpy.core.umath_tests import inner1d<jupyter_text>Logistic Regression Model<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X, y,
    test_size=0.3, random_state=42)
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD

K.clear_session()

model = Sequential()

model.add(Dense(1, input_shape=(4,), activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='SGD',
              metrics=['accuracy'])

model.summary()
history = model.fit(
    X_train, y_train,
    epochs=10, validation_split=0.2, verbose=2)
historydf = pd.DataFrame(history.history, index=history.epoch)
historydf.plot();
result = model.evaluate(X_test, y_test)

print("Test accuracy: {:3.1f} %".format(result[1]*100))<jupyter_output>412/412 [==============================] - 0s 28us/step
Test accuracy: 77.7 %<jupyter_text>Logistic Regression Model - Customize learning rate$$w' = w - \alpha \frac{\partial f}{\partial w}$$- $\alpha$ = learning rate    + small: tiny step toward optimal point        + large: overshoot<jupyter_code>dflist = []
test_acc = {}

learning_rates = [0.01, 0.05, 0.1, 0.5]

for lr in learning_rates:
    # Model
    K.clear_session()

    model = Sequential()
    
    model.add(Dense(1, input_shape=(4,), activation='sigmoid'))
    
    model.compile(loss='binary_crossentropy',
                  optimizer=SGD(lr=lr),
                  metrics=['accuracy'])

    # Train
    h = model.fit(
        X_train, y_train,
        epochs=10, validation_split=0.2,
        batch_size=16, verbose=0)
    
    dflist.append(pd.DataFrame(h.history, index=h.epoch))
    
    # Test
    res = model.evaluate(X_test, y_test)
    test_acc[str(lr)] = res[1]*100
historydf = pd.concat(dflist, axis=1)
display(historydf.head())

metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product(
    [learning_rates, metrics_reported],
    names=['learning_rate', 'metric'])

historydf.columns = idx
display(historydf.head())
ax = plt.subplot(411)
historydf.xs('loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax, figsize=(20,20))
plt.title("Loss")

ax = plt.subplot(412)
historydf.xs('acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Accuracy")
plt.xlabel("Epochs")

ax = plt.subplot(413)
historydf.xs('val_acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Validation Accuracy")
plt.xlabel("Epochs")

ax = plt.subplot(414)
historydf.xs('val_loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Validation Loss")
plt.xlabel("Epochs")


plt.tight_layout()
for key, val in test_acc.items():
    print('Learning rate %s - Test accuracy = %0.2f' % (key, val))<jupyter_output>Learning rate 0.01 - Test accuracy = 80.10
Learning rate 0.05 - Test accuracy = 96.36
Learning rate 0.1 - Test accuracy = 97.82
Learning rate 0.5 - Test accuracy = 98.06<jupyter_text>Logistic Regression Model - Customize batch size- Gradient Descent    + 1 epoch train the whole dataset    - Stochastic Gradient Descent (SGD)   + 1 epoch train the 1 datapoint   - Mini-batch Gradient Descent   + 1 epoch train the a subset of dataset (16, 32, 128, 256... datapoints)   + small batch size: speedup training, noisy gradient, slow convergence   + large batch size: slow training, less noisy<jupyter_code>dflist = []
test_acc = {}

batch_sizes = [16, 32, 64, 128]

for batch_size in batch_sizes:
    # model
    K.clear_session()

    model = Sequential()

    model.add(Dense(1, input_shape=(4,), activation='sigmoid'))

    model.compile(loss='binary_crossentropy',
                  optimizer='sgd',
                  metrics=['accuracy'])
    
    # Train
    h = model.fit(
        X_train, y_train,
        epochs=10, validation_split=0.2,
        batch_size=batch_size,
        verbose=0)
    
    dflist.append(pd.DataFrame(h.history, index=h.epoch))
    
    # Test
    res = model.evaluate(X_test, y_test)
    test_acc[str(batch_size)] = res[1]*100
historydf = pd.concat(dflist, axis=1)
display(historydf.head())

metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product(
    [batch_sizes, metrics_reported],
    names=['batch_size', 'metric'])

historydf.columns = idx
display(historydf.head())
ax = plt.subplot(411)
historydf.xs('loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax, figsize=(20,20))
plt.title("Loss")

ax = plt.subplot(412)
historydf.xs('acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Accuracy")
plt.xlabel("Epochs")

ax = plt.subplot(413)
historydf.xs('val_acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Validation Accuracy")
plt.xlabel("Epochs")

ax = plt.subplot(414)
historydf.xs('val_loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Validation Loss")
plt.xlabel("Epochs")


plt.tight_layout()
for key, val in test_acc.items():
    print('Batch size %s - Test accuracy = %0.2f' % (key, val))<jupyter_output>Batch size 16 - Test accuracy = 94.90
Batch size 32 - Test accuracy = 83.74
Batch size 64 - Test accuracy = 95.39
Batch size 128 - Test accuracy = 49.76<jupyter_text>Logistic Regression Model - Customize optimizer EWMA- Exponentially weighted moving average- Applied in Signal Processing, Time-series, Neural Network...    + remove noises    + smoothing signal    + Apply to Optimizer in Neural network$$s_t = ax_t + (1-a)s_{t-1}$$    + a in range [0,1]        - a small: prefer past(t-1) value        - a large: prefer current value(t) value        - Example Optimizer- Find the minimum cost value of a cost function by doing back propagation SGD- Hyperparameter:     + learning rate $\epsilon$- Notations:    + $\widehat{g}$: Gradient estimate    + $\theta$: weights SGD + Momentum- Hyperparameter:     + learning rate $\epsilon$    + momentum velocity v: Try to smoothen the gradient update, if moving toward the old direction -> move slowly- Notations:    + $\widehat{g}$: Gradient estimate    + $\theta$: weights SGD + Nesterov Momentum- Hyperparameter:     + learning rate $\epsilon$    + momentum velocity v: Try to smoothen the gradient update, if moving toward the old direction -> move slowly- Notations:    + $\widehat{g}$: Gradient estimate    + $\theta$: weights- Different from SGD + Momentum    + Update the gradient based on temporal momentum     AdaGrad (Adaptive Algorithm - learning rate can change)- Adaptive correction using Squared Gradien RMSprop (Adaptive Algorithm - learning rate can change)- Apply EWNA to squared gradient AdaGrad Adam (Adaptive Algorithm - learning rate can change)- Adaptive, Apply EWNA on 1st and 2nd moments Optimizers Summary<jupyter_code>from keras.optimizers import SGD, Adam, Adagrad, RMSprop

dflist = []
test_acc = {}

optimizers = ['SGD(lr=0.01)',
              'SGD(lr=0.01, momentum=0.3)',
              'SGD(lr=0.01, momentum=0.3, nesterov=True)',  
              'Adam(lr=0.01)',
              'Adagrad(lr=0.01)',
              'RMSprop(lr=0.01)']

for opt_name in optimizers:
    # model
    K.clear_session()
    
    model = Sequential()
    
    model.add(Dense(1, input_shape=(4,), activation='sigmoid'))
    
    model.compile(loss='binary_crossentropy',
                  optimizer=eval(opt_name),
                  metrics=['accuracy'])
    
    # Train
    h = model.fit(
        X_train, y_train,
        epochs=10, validation_split=0.2,
        batch_size=16, verbose=0)
    
    dflist.append(pd.DataFrame(h.history, index=h.epoch))
    
    # Test
    res = model.evaluate(X_test, y_test)
    test_acc[str(opt_name)] = res[1]*100
historydf = pd.concat(dflist, axis=1)
display(historydf.head())

metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product(
    [optimizers, metrics_reported],
    names=['optimizer', 'metric'])

historydf.columns = idx
display(historydf.head())
ax = plt.subplot(411)
historydf.xs('loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax, figsize=(20,20))
plt.title("Loss")

ax = plt.subplot(412)
historydf.xs('acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Accuracy")
plt.xlabel("Epochs")

ax = plt.subplot(413)
historydf.xs('val_acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Validation Accuracy")
plt.xlabel("Epochs")

ax = plt.subplot(414)
historydf.xs('val_loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Validation Loss")
plt.xlabel("Epochs")


plt.tight_layout()
for key, val in test_acc.items():
    print('Optimizer %s - Test accuracy = %0.2f' % (key, val))<jupyter_output>Optimizer SGD(lr=0.01) - Test accuracy = 74.51
Optimizer SGD(lr=0.01, momentum=0.3) - Test accuracy = 76.46
Optimizer SGD(lr=0.01, momentum=0.3, nesterov=True) - Test accuracy = 92.96
Optimizer Adam(lr=0.01) - Test accuracy = 97.82
Optimizer Adagrad(lr=0.01) - Test accuracy = 87.14
Optimizer RMSprop(lr=0.01) - Test accuracy = 98.06<jupyter_text>Weight Initialization- Neural Network work best when input normalized (mean=0, std=1)- Initialize weights non-zero => better training<jupyter_code>dflist = []

initializers = ['zeros', 'uniform', 'normal',
                'he_normal', 'lecun_uniform']
test_acc = {}

for init in initializers:
    # model
    K.clear_session()

    model = Sequential()
    
    model.add(Dense(1, input_shape=(4,),
                    kernel_initializer=init,
                    activation='sigmoid'))

    model.compile(loss='binary_crossentropy',
                  optimizer='rmsprop',
                  metrics=['accuracy'])

    # Train
    h = model.fit(
        X_train, y_train,
        validation_split=0.2,
        batch_size=16, epochs=5, verbose=0)
    
    dflist.append(pd.DataFrame(h.history, index=h.epoch))
    
    # Test
    res = model.evaluate(X_test, y_test)
    test_acc[str(init)] = res[1]*100
historydf = pd.concat(dflist, axis=1)
display(historydf.head())

metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product(
    [initializers, metrics_reported],
    names=['initializers', 'metric'])

historydf.columns = idx
display(historydf.head())
ax = plt.subplot(411)
historydf.xs('loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax, figsize=(20,20))
plt.title("Loss")

ax = plt.subplot(412)
historydf.xs('acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Accuracy")
plt.xlabel("Epochs")

ax = plt.subplot(413)
historydf.xs('val_acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Validation Accuracy")
plt.xlabel("Epochs")

ax = plt.subplot(414)
historydf.xs('val_loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Validation Loss")
plt.xlabel("Epochs")


plt.tight_layout()
for key, val in test_acc.items():
    print('Initiator %s - Test accuracy = %0.2f' % (key, val))<jupyter_output>Initiator zeros - Test accuracy = 80.10
Initiator uniform - Test accuracy = 80.58
Initiator normal - Test accuracy = 81.07
Initiator he_normal - Test accuracy = 20.87
Initiator lecun_uniform - Test accuracy = 88.35
{'fraction_non_alphanumeric': 0.1462038039276326, 'fraction_numerical': 0.03239523735889903, 'mean_word_length': 4.217829770068576, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '205051', 'license': 'Unlicense', 'n_tokens_mistral': 4814, 'n_tokens_neox': 4478, 'n_words': 997, 'path': 'CourseUdemy_Keras/6_Miscellaneous/01_Customize_learning_rate_&_batch_size_&_optimizer_&_Weight_Initialization.ipynb', 'repo_name': 'dangkhoa1992/Advanced-Machine-Learning-Specialization', 'size': '12934'}