<jupyter_start><jupyter_text>1. Import Packages<jupyter_code>import numpy as np
from numpy.random import seed
seed(7)
import math
import h5py
from numpy import array
from keras import initializers
from keras.models import Sequential
from keras.models import Model, load_model
from keras.layers import LSTM, BatchNormalization, Activation
from keras.layers import Conv1D, MaxPooling1D, Dense, Dropout, Flatten
from keras.layers import RepeatVector
from keras.layers import TimeDistributed
from keras.utils import to_categorical
from keras.optimizers import Adam, SGD
from keras.losses import categorical_crossentropy
from keras.callbacks import LearningRateScheduler, ReduceLROnPlateau, ModelCheckpoint
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.preprocessing import MinMaxScaler, StandardScaler
import matplotlib.pyplot as plt
import os<jupyter_output>Using TensorFlow backend.<jupyter_text>2. Data Normalization<jupyter_code>def scale_data(X):
    #scaler = MinMaxScaler()
    scaler = StandardScaler()
    #scaler.fit(X)
    #ret = scaler.transform(X)
    ret = np.empty(shape = X.shape)
    for idx,r in enumerate(X):
        norm = np.linalg.norm(r)
        scaled = r/norm
        ret[idx] = scaled
    return ret<jupyter_output><empty_output><jupyter_text>3. Label Smoothing -- only available with Keras 2.3.1+<jupyter_code>def cce_function(y_true, y_pred):
    return categorical_crossentropy(y_true, y_pred, label_smoothing=0.1)<jupyter_output><empty_output><jupyter_text>4. Read Data in Batches<jupyter_code>def batch_generator(X, Y, batch_size, shuffle=False):
    indices = np.arange(len(X)) 
    batch=[]
    while True:
        # it might be a good idea to shuffle your data before each epoch
        if shuffle:
            np.random.shuffle(indices) 
        for i in indices:
            batch.append(i)
            if len(batch)==batch_size:
                yield X[batch], Y[batch]
                batch=[]<jupyter_output><empty_output><jupyter_text>5. Pre-processing Train/Val/Test Data<jupyter_code>train_fname = 'vector-data/train.csv'
raw_train = open(train_fname, 'rt')
train_data = np.loadtxt(raw_train, delimiter=",")
train_X = train_data[:,1:8193]
train_Y = train_data[:,8193]
train_X = scale_data(train_X)
train_Y = to_categorical(train_Y, num_classes=5)
plt.plot(train_X[1])
val_fname = 'vector-data/val.csv'
raw_val = open(val_fname, 'rt')
val_data = np.loadtxt(raw_val, delimiter=",")
val_X = val_data[:,1:8193]
val_Y = val_data[:,8193]
val_X = scale_data(val_X)
val_Y = to_categorical(val_Y, num_classes=5)
plt.plot(val_X[1287])
test_fname = 'vector-data/test.csv'
raw_test = open(test_fname, 'rt')
test_data = np.loadtxt(raw_test, delimiter=",")
test_X = test_data[:,1:8193]
test_Y = test_data[:,8193]
test_X = scale_data(test_X)
test_Y = to_categorical(test_Y, num_classes=5)<jupyter_output><empty_output><jupyter_text>6. Model definition -- VGG-16 like<jupyter_code># define model
k = 15
train_X = train_X.reshape((len(train_X), 8192, 1))
val_X = val_X.reshape((len(val_X), 8192, 1))
test_X = test_X.reshape((len(test_X), 8192, 1))

model = Sequential()
model.add(Conv1D(filters=64, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu', input_shape=(8192,1)))
model.add(Conv1D(filters=64, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=4, padding='same'))

model.add(Conv1D(filters=128, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(Conv1D(filters=128, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=4, strides=4))

model.add(Conv1D(filters=256, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(Conv1D(filters=256, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(Conv1D(filters=256, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=4, strides=4))

model.add(Conv1D(filters=512, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(Conv1D(filters=512, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(Conv1D(filters=512, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=4, strides=4))

model.add(Conv1D(filters=512, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(Conv1D(filters=512, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(Conv1D(filters=512, kernel_size=k, kernel_initializer='he_normal', padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=4, strides=4))

model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.7))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.7))
model.add(Dense(5, activation='softmax'))<jupyter_output>WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.

WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.

WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4185: The name tf.truncated_normal is deprecated. Please use tf.random.truncated_normal instead.

WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.

WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Pleas[...]<jupyter_text>7. Optimizer and Compiling Model<jupyter_code>opt = Adam(lr=1e-5, clipvalue=5)
#model.compile(optimizer=opt, loss=cce_function, metrics=['accuracy'])
model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['accuracy'])
print(model.summary())<jupyter_output>WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.

WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.

_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv1d_1 (Conv1D)            (None, 8192, 64)          1024      
_________________________________________________________________
conv1d_2 (Conv1D)            (None, 8192, 64)          61504     
_________________________________________________________________
max_pooling1d_1 (MaxPooling1 (None, 2048, 64)          0         
_________________________________________________________________
conv1d_3 (Conv1D)       [...]<jupyter_text>8. Iteraters, Batch Size, and Callbacks<jupyter_code>batch_size = 16
train_it = batch_generator(train_X, train_Y, batch_size, shuffle=True)
val_it = batch_generator(val_X, val_Y, batch_size)

reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=math.sqrt(0.1),
                              patience=5, min_lr=1e-8, verbose=1)
# Prepare model model saving directory.
save_dir = os.path.join(os.getcwd(), 'saved_models')
#model_name = 'spectra_model.{epoch:03d}.h5'
model_name = 'spectra_model.h5'
if not os.path.isdir(save_dir):
    os.makedirs(save_dir)
filepath = os.path.join(save_dir, model_name)

# Prepare callbacks for model saving and for learning rate adjustment.
checkpoint = ModelCheckpoint(filepath=filepath,
                             monitor='val_acc',
                             verbose=1,
                             save_best_only=True)<jupyter_output><empty_output><jupyter_text>9. Train<jupyter_code>model.fit_generator(train_it,
                    steps_per_epoch=len(train_X)//batch_size,
                    validation_data=val_it,
                    validation_steps=len(val_X)//batch_size,
                    epochs=50,
                    callbacks=[reduce_lr, checkpoint],
                    verbose=1)<jupyter_output>WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:986: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.

WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:973: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.

WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:2741: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.

Epoch 1/50
WARNING:tensorflow:From /home1/00946/zzhang/.local/lib/python3.7/site-pack[...]<jupyter_text>10. Validate<jupyter_code>from sklearn.utils.multiclass import unique_labels

def plot_confusion_matrix(y_true, y_pred, classes,
                          normalize=False,
                          title=None,
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if not title:
        if normalize:
            title = 'Normalized confusion matrix'
        else:
            title = 'Confusion matrix, without normalization'
    # Compute confusion matrix
    cm = confusion_matrix(y_true, y_pred)
    # Only use the labels that appear in the data
    classes = classes[unique_labels(y_true, y_pred)]
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')
    fig, ax = plt.subplots()
    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)
    # We want to show all ticks...
    ax.set(xticks=np.arange(cm.shape[1]),
           yticks=np.arange(cm.shape[0]),
           # ... and label them with the respective list entries
           xticklabels=classes, yticklabels=classes,
           title=title,
           ylabel='Measured Performance',
           xlabel='Predicted Performance')
    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")
    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            ax.text(j, i, format(cm[i, j], fmt),
                    ha="center", va="center",
                    color="white" if cm[i, j] > thresh else "black")
    fig.tight_layout()
    return ax
val_it = batch_generator(val_X, val_Y, 1)
Y_pred = model.predict_generator(val_it, 1288)
y_pred = np.argmax(Y_pred, axis=1)

print('Confusion Matrix')
print(confusion_matrix(val_data[:,8193], y_pred))
plot_confusion_matrix(array(np.argmax(val_Y, axis=1)), array(y_pred), classes=array(['Ia', 'Ib', 'Ic', 'IIn', 'IIP']), title='Confusion matrix, without normalization')<jupyter_output>Confusion Matrix
[[260  12  15   8   5]
 [ 18 186  34   3   9]
 [ 25  32 209   7   4]
 [  5   9  10 165  15]
 [  5  13   4  15 220]]
Confusion matrix, without normalization<jupyter_text>11. Test<jupyter_code>test_it = batch_generator(test_X, test_Y, 1)
Y_pred = model.predict_generator(test_it, len(test_X))
y_pred = np.argmax(Y_pred, axis=1)

print('Confusion Matrix')
print(confusion_matrix(test_data[:,8193], y_pred))<jupyter_output>Confusion Matrix
[[128  10   7   3   2]
 [  4  94  21   1   5]
 [  7  21 106   4   1]
 [  1   3   4  91   4]
 [  3  12   3   4 107]]<jupyter_text>12. Visualizing Test Result in Confusion Matrix<jupyter_code>plot_confusion_matrix(array(np.argmax(test_Y, axis=1)), array(y_pred), classes=array(['Ia', 'Ib', 'Ic', 'IIn', 'IIP']), title='Confusion matrix, without normalization')<jupyter_output>Confusion matrix, without normalization<jupyter_text>13. Load Model<jupyter_code>l_model = load_model("saved_models/spectra_model.h5")<jupyter_output>WARNING:tensorflow:Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.
WARNING:tensorflow:Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.<jupyter_text>14. Single Prediction<jupyter_code>idx = np.random.randint(0, len(test_X))
pred = l_model.predict(test_X[idx].reshape(1,8192,1))
print(f"predict: {np.argmax(pred)}, label: {np.argmax(test_Y[idx])}")<jupyter_output>predict: 2, label: 2<jupyter_text>15. Batch Prediction -- Test<jupyter_code>test_it = batch_generator(test_X, test_Y, 1)
Y_pred = l_model.predict_generator(test_it, len(test_X))
y_pred = np.argmax(Y_pred, axis=1)

print('Confusion Matrix')
print(confusion_matrix(test_data[:,8193], y_pred))
plot_confusion_matrix(array(np.argmax(test_Y, axis=1)), array(y_pred), classes=array(['Ia', 'Ib', 'Ic', 'IIn', 'IIP']), title='Confusion matrix, without normalization')<jupyter_output>Confusion matrix, without normalization
{'fraction_non_alphanumeric': 0.12942351598173515, 'fraction_numerical': 0.03603025114155251, 'mean_word_length': 4.761200164406083, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '576018', 'license': 'MIT', 'n_tokens_mistral': 5038, 'n_tokens_neox': 4761, 'n_words': 934, 'path': 'classifier-vector.ipynb', 'repo_name': 'zhaozhang/spectra', 'size': '14016'}