<jupyter_start><jupyter_text>Test the full connected layer model  Import necessary modules<jupyter_code># python built-in libraries
import datetime
# choose the tensorflow log level
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # or any {'0', '1', '2'}
# import tensorflow
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
# ohter packages
import xarray as xr
import numpy as np<jupyter_output><empty_output><jupyter_text>Global Variables<jupyter_code># sampling
DATA_FILE = "./data/all_data.nc"  # path to the file of the feature tensor
FRACTIONS = (0.8, 0.1, 0.1)  # train, validation, test
BATCH_SIZE = 128  # size of the batch
BUFFER_SIZE = BATCH_SIZE * 4  # size of the shuffle buffer
RANDOM_SEED = 42  # a seed for the random sampling
THRESHOLD = 0.95
# training
LEARNING_RATE = 0.001  # starting learning rate
BETA1 = 0.9  # decay 1
BETA2 = 0.999  # decay 2
EPOCHS = 200  # number of epochs
# saving
TIME_STAMP = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # time stamp
MODEL_NAME = "C4D4"  # name of the model
SAVE_LOC = "./data/models/{}_{}".format(MODEL_NAME, TIME_STAMP)  # path to the folder to save the model
LOG_LOC = "./data/logs/fit/{}_{}".format(MODEL_NAME, TIME_STAMP)  # path to the log, if you change this, you also need to change it in the run_tensorboard<jupyter_output><empty_output><jupyter_text>Obtain dataThe data is a 209 data point PDF. The label is a one-hot 2 dim vector. `10` means major phase >= threshold, `01` means major phase <= threshold.<jupyter_code>def load_data():
    # load
    ds = xr.open_dataset(DATA_FILE)
    features = ds["G"].values
    # label the data by threshold
    single = np.array([1, 0])
    mulitple = np.array([0, 1])
    labels = np.apply_along_axis(lambda tup: single if np.max(tup) >= THRESHOLD else mulitple, 1,  ds["fraction"].values)
    # shuffle
    n = features.shape[0]
    np.random.seed(RANDOM_SEED)
    shuffled_idx = np.random.permutation(n)
    features, labels = features[shuffled_idx], labels[shuffled_idx]
    # split
    f0, f1, f2 = FRACTIONS
    i, j, k = round(f0 * n), round((f0 + f1) * n), round((f0 + f1 + f2) * n)
    train_data = tf.data.Dataset.from_tensor_slices((features[:i], labels[:i])).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
    valid_data = tf.data.Dataset.from_tensor_slices((features[i:j], labels[i:j])).batch(BATCH_SIZE)
    test_data = tf.data.Dataset.from_tensor_slices((features[j:k], labels[j:k])).batch(BATCH_SIZE)
    # close dataset
    ds.close()
    return train_data, valid_data, test_data
train_data, valid_data, test_data = load_data()<jupyter_output><empty_output><jupyter_text>Create the modelWe use the logistric regression. It is a single layer with a softmax function.<jupyter_code>def create_model():
    model = keras.Sequential()
    model.add(keras.Input(shape=(209,)))
    model.add(keras.layers.Reshape((209, 1)))
    model.add(keras.layers.Conv1D(32, 3, strides=1, activation='relu', padding="same"))
    model.add(keras.layers.Conv1D(32, 3, strides=1, activation='relu', padding="same"))
    model.add(keras.layers.BatchNormalization())
    model.add(keras.layers.MaxPooling1D(2))
    model.add(keras.layers.Conv1D(64, 3, strides=1, activation='relu', padding="same"))
    model.add(keras.layers.Conv1D(64, 3, strides=1, activation='relu', padding="same"))
    model.add(keras.layers.BatchNormalization())
    model.add(keras.layers.MaxPooling1D(2))
    model.add(keras.layers.Dropout(0.2))
    model.add(keras.layers.Flatten())
    model.add(keras.layers.Dense(512, activation='relu'))
    model.add(keras.layers.Dense(128, activation='relu'))
    model.add(keras.layers.Dense(32, activation='relu'))
    model.add(keras.layers.Dense(2, activation='softmax'))
    return model
model = create_model()
model.summary()<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
reshape (Reshape)            (None, 209, 1)            0         
_________________________________________________________________
conv1d (Conv1D)              (None, 209, 32)           128       
_________________________________________________________________
conv1d_1 (Conv1D)            (None, 209, 32)           3104      
_________________________________________________________________
batch_normalization (BatchNo (None, 209, 32)           128       
_________________________________________________________________
max_pooling1d (MaxPooling1D) (None, 104, 32)           0         
_________________________________________________________________
conv1d_2 (Conv1D)            (None, 104, 64)           6208      
________________________________________________________[...]<jupyter_text>Choose optimization method<jupyter_code>model.compile(
    # Optimizer
    optimizer=keras.optimizers.Adam(LEARNING_RATE, BETA1, BETA2),
    # Loss function to minimize
    loss=keras.losses.CategoricalCrossentropy(),
    # List of metrics to monitor
    metrics=[
        keras.metrics.CosineSimilarity(),
        keras.metrics.CategoricalAccuracy()
    ],
)<jupyter_output><empty_output><jupyter_text>Train the model<jupyter_code># tensor board
tensorboard_callback = keras.callbacks.TensorBoard(
    log_dir=LOG_LOC, 
    histogram_freq=1
)
# early stopping to avoid over fitting
earlystopping_callback = keras.callbacks.EarlyStopping(
    monitor='val_loss',
    patience=30, 
    restore_best_weights=True
)
history = model.fit(
    x=train_data,
    epochs=EPOCHS,
    callbacks=[
        tensorboard_callback,
        earlystopping_callback
    ],
    validation_data=valid_data
)<jupyter_output>Epoch 1/200
688/688 [==============================] - 26s 17ms/step - loss: 0.6426 - cosine_similarity: 0.7519 - categorical_accuracy: 0.6636 - val_loss: 0.5519 - val_cosine_similarity: 0.7901 - val_categorical_accuracy: 0.7235
Epoch 2/200
688/688 [==============================] - 11s 16ms/step - loss: 0.5265 - cosine_similarity: 0.8005 - categorical_accuracy: 0.7359 - val_loss: 0.4902 - val_cosine_similarity: 0.8180 - val_categorical_accuracy: 0.7618
Epoch 3/200
688/688 [==============================] - 11s 16ms/step - loss: 0.4619 - cosine_similarity: 0.8288 - categorical_accuracy: 0.7740 - val_loss: 0.4306 - val_cosine_similarity: 0.8445 - val_categorical_accuracy: 0.8014
Epoch 4/200
688/688 [==============================] - 11s 16ms/step - loss: 0.3994 - cosine_similarity: 0.8552 - categorical_accuracy: 0.8125 - val_loss: 0.4073 - val_cosine_similarity: 0.8543 - val_categorical_accuracy: 0.8108
Epoch 5/200
688/688 [==============================] - 12s 17ms/step - loss: 0.3573 [...]<jupyter_text>Evaluate the model<jupyter_code>result = model.evaluate(
    x=test_data,
    return_dict=True
)
for name, val in result.items():
    print("{:20s} {:.4f}".format(name, val))<jupyter_output>loss                 0.3279
cosine_similarity    0.9092
categorical_accuracy 0.8921<jupyter_text>Save the model<jupyter_code>model.save(SAVE_LOC)<jupyter_output>INFO:tensorflow:Assets written to: ./data/models/C4D4_20210928-190611/assets
{'fraction_non_alphanumeric': 0.14367335562987738, 'fraction_numerical': 0.054626532887402456, 'mean_word_length': 4.396240601503759, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '26539', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2655, 'n_tokens_neox': 2371, 'n_words': 506, 'path': '19_cnn4_fc4_run_xarray_data.ipynb', 'repo_name': 'st3107/tfhelper', 'size': '7176'}