<jupyter_start><jupyter_text>Pendulum dataset. $$ \dot{x_{1}} = x_{2}$$$$ \dot{x_{2}} = -\sin(x_{1})$$ Autoencoder Loss:$ X_{input}$ - input data. $Y$- latent space data.  $\tilde{Y}$ - latent space advanced. $g^{-1}(\tilde{Y})$ - latent space advanced decoded. $Loss_{ae} = MSE(g^{-1}(\tilde{Y})- X_{input})$ DMD Loss: $DMD_{loss}=\arg \min _{A}\left\| {X}^{+}- A\cdot X^{-} \right\| _{F}^{2}$$ X^{+} = A \cdot X^{-}$By the singular value decomposition, $ X^{-} = U \cdot \Sigma \cdot V^{T}$Then, $ A = X^{+} \cdot V \cdot \Sigma^{-1} \cdot U^{*}$,Hence, $ \left\| {X}^{+}- A\cdot X^{-} \right\| _{F}^{2} = \left\| {X}^{+}- (X^{+} \cdot V \cdot \Sigma^{-1} \cdot U^{*})\cdot X^{-} \right\| _{F}^{2} =\left\| {X}^{+}- (X^{+} \cdot V \cdot \Sigma^{-1} \cdot U^{*})\cdot ( U \cdot \Sigma \cdot V^{T}) \right\| _{F}^{2} = \left\| {X}^{+}( I - V \cdot V^{T}) \right\| _{F}^{2}$$ Loss_{dmd} = \left\| {X}^{+}( I - V \cdot V^{T}) \right\| _{F}^{2}$ Future state prediction Loss: $ X^{+} = A \cdot X^{-}$By the singular value decomposition, $ X^{-} = U \cdot \Sigma \cdot V^{T}$Then, $ A = X^{+} \cdot V \cdot \Sigma^{-1} \cdot U^{*}$,$ X_{pred} = [x_{0}, x_{0}\cdot A, x_{0} \cdot A^2, x_{0} \cdot A^{3} ... , x_{0}\cdot A^{m-1}]$$Loss_{pred} = MSE(X - X_{pred})$<jupyter_code>%load_ext autoreload
%autoreload 2
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
from dmd_machine.dmd_ae_machine import DMDMachine
from dmd_machine.loss_function import LossFunction
from data.Data import DataMaker
from datetime import date 
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import model_from_json
from return_stats import *
from create_plots import *
from datetime import date  
import pickle
import time

plt.rcParams['figure.figsize'] = [15, 8]
plt.rcParams['figure.facecolor'] = 'white'
%matplotlib inline<jupyter_output><empty_output><jupyter_text>$X_{1} \in [-3.1, 3.1]$$X_{2} \in [-2, 2]$The potential function is under 0.99.The potential function for the pendulum is $\frac{1}{2} x_{2}^{2} - cos(x_{1})$. These ranges arechosen to sample the pendulum in the full phase space where the pendulumapproaches having an infinite period.<jupyter_code>training_data = pickle.load(open('./data/dataset_pendulum.pkl', 'rb'))
data = training_data.data_val
data.shape
_ = plt.figure(figsize=(15,11))
for ii in np.arange(0, 10000, 60):
    plt.plot(data[ii, 0, :], data[ii, 1, :])
plt.xlabel("$x_{1} = \Theta $", fontsize=20)
plt.ylabel("$x_{2}= \partial \Theta /\partial t $", fontsize=20)
plt.title("Pendulum dataset", fontsize=20)

plt.show()
def dmd_loss(x_data):
    x_minus = x_data[:, :-1]
    x_plus = x_data[:, 1:]

    U, S, Vt = np.linalg.svd(x_minus, full_matrices=True)

    I = np.identity(np.shape(Vt)[0])

    VVt = np.matmul(np.transpose(Vt), Vt)

    for_norm = np.linalg.norm(np.matmul(x_plus, (I - VVt)), ord='fro')
    
    return for_norm
for_vec_x = np.zeros(np.shape(training_data.data_val)[0])
for ii in range(0, np.shape(training_data.data_val)[0]):
    loss = dmd_loss(data[ii, :, :])
    for_vec_x[ii] = loss
_ = plt.figure(figsize=(15,8))
_ = plt.plot(np.log10(for_vec_x))
_ = plt.ylabel("log(dmd_loss)")
_ = plt.xlabel("ic index")
_ = plt.title("$\| {X}^{+}( I - V \cdot V^{T})\| _{F}^{2}$")<jupyter_output><empty_output><jupyter_text>Train DMD Autoencoder.<jupyter_code># Network Hyper Parameters.
hyp_params = dict()
hyp_params['num_t_steps'] = training_data.params['num_time_steps']
hyp_params['phys_dim'] = training_data.params["num_physical_dim"]
hyp_params['num_init_conds'] = training_data.params['num_initial_conditions']
hyp_params['batch_size'] = 256
hyp_params['num_epochs'] = 200

# Encoding/Decoding Layer Parameters.
hyp_params['num_en_layers'] = 2
hyp_params['num_en_neurons'] = 80
hyp_params['latent_dim'] = 2
hyp_params['window_size'] = 256

hyp_params['activation'] = 'elu'
hyp_params['weight_initializer'] = 'he_uniform'
hyp_params['bias_initializer'] = 'he_uniform'

hyp_params['c1'] = 10  # coefficient auto-encoder loss.
hyp_params['c2'] = 1  # coefficient of dmd loss.
hyp_params['c3'] = 1  # coefficient of pred loss.

# save results in the folder " Results/save_folder"- including loss curves and plot latent data.
save_folder = "AeEx2_jup" + str(date.today().isoformat())  

# number of initial conditions in training and testing dataset.
hyp_params['num_init_conds_training'] = int(0.8 * hyp_params['num_init_conds'])
hyp_params['num_init_conds_test'] = hyp_params['num_init_conds'] - hyp_params['num_init_conds_training']

# initialize machine and loss objects.
myMachine = DMDMachine(hyp_params)
myLoss = LossFunction(hyp_params)

# Learning rate initialization.
hyp_params["initial_learning_rate"] = 3e-3  # MAJOR PARAMETER CHOICE
hyp_params["esteps"] = 30  # MAJOR PARAMETER CHOICE
count = 0

# clear previous run session.
tf.keras.backend.clear_session()
# ======================================================================================================================
# Prepare dataset. 
# ======================================================================================================================
# shuffle the dataset and then divide to training vs testing data sets. 80% training .20% testing.
data_train, data_test= train_test_split(data, test_size=0.2, random_state=42)

print("dimensions of training dataset (ic x phys_dim x timesteps) = ", np.shape(data_train))
print("dimensions of testing dataset (ic x phys_dim x timesteps) = ", np.shape(data_test))
# ======================================================================================================================
# Unit test to verify that testing and training datasets are disjoint.
# ======================================================================================================================
for ic_train in data_train:
    for ic_test in data_test:
        if ic_test[:, 0][0] == ic_train[:, 0][0] and ic_test[:, 0][1] == ic_train[:, 0][1]\
        and ic_test[:, 0][2] == ic_train[:, 0][2]:
            print("Testing and Training datasets intersect!")
            print(ic_test[:, 0])
# convert datasets from numpy to tensorflow.
data_train =  tf.data.Dataset.from_tensor_slices(data_train)
data_test =  tf.data.Dataset.from_tensor_slices(data_test)
# create folder to save results.
create_new_folders(save_folder)

# save hyperparams in a json file.
save_hyp_params_in_json(hyp_params=hyp_params, json_file_path=os.path.join("results", save_folder, "hyp_params.txt"))
# initialize loss results (lists) as a function of epoch (iteration).
train_loss_results = []
test_loss_results = []

train_dmd_loss = []
test_dmd_loss = []

train_ae_loss = []
test_ae_loss = []

train_pred_loss = []
test_pred_loss = []

epoch = 0

while epoch < (hyp_params['num_epochs']):
    # start timer. 
    start_time = time.process_time()
    # save the total loss of the training data and testing data.
    epoch_loss_avg_train = tf.keras.metrics.Mean()
    epoch_loss_avg_test = tf.keras.metrics.Mean()

    # keep track of individual losses as well, aka dmd loss and ae loss.
    epoch_loss_dmd_train = tf.keras.metrics.Mean()
    epoch_loss_dmd_test = tf.keras.metrics.Mean()

    epoch_loss_ae_train = tf.keras.metrics.Mean()
    epoch_loss_ae_test = tf.keras.metrics.Mean()

    epoch_loss_pred_train = tf.keras.metrics.Mean()
    epoch_loss_pred_test = tf.keras.metrics.Mean()

    # Build out the batches within a given epoch.
    train_batch = data_train.shuffle(hyp_params['num_init_conds_training'], seed=42).batch(hyp_params["batch_size"],
                                                                                           drop_remainder=True)

    test_batch = data_test.shuffle(hyp_params['num_init_conds_test'], seed=42).batch(hyp_params["batch_size"],
                                                                                     drop_remainder=True)

    # Learning rate scheduling plan.  See Ch. 11 of O'Reilly.
    if epoch % hyp_params["esteps"] == 0:
        hyp_params['lr'] = (.2 ** count) * hyp_params["initial_learning_rate"]
        adam_optimizer = tf.keras.optimizers.Adam(hyp_params['lr'])
        count += 1

    # Iterate through all the batches within an epoch.
    for batch_training_data in train_batch:
        # normalize batch

        # Build terms that we differentiate (i.e. loss) and that we differentiate with respect to.
        with tf.GradientTape() as tape:
            # training=True is only needed if there are layers with different
            # behavior during training versus inference (e.g. Dropout).
            predictions_train = myMachine(batch_training_data)
            ae_loss = predictions_train[3]
            dmd_loss = predictions_train[2]
            pred_loss = predictions_train[5]

            loss_train = myLoss(batch_training_data, predictions_train)

            if epoch % 5 == 0:
                if abs((tf.reduce_min(predictions_train[1][:, 0, :]) - tf.reduce_max(
                        predictions_train[1][:, 0, :])).numpy()) < 0.3 \
                        or abs((tf.reduce_min(predictions_train[1][:, 1, :]) - tf.reduce_max(
                    predictions_train[1][:, 1, :])).numpy()) < 0.3 \
                        or abs((tf.reduce_min(predictions_train[0][:, 0, :]) - tf.reduce_max(
                    predictions_train[0][:, 0, :])).numpy()) < 0.3 \
                        or abs((tf.reduce_min(predictions_train[0][:, 1, :]) - tf.reduce_max(
                    predictions_train[0][:, 1, :])).numpy()) < 0.3:
                    print("restart initializers. ")
                    epoch = -1
                    tf.keras.backend.clear_session()
                    myMachine = DMDMachine(hyp_params)
                    myLoss = LossFunction(hyp_params)

        # Compute gradients and then apply them to update weights within the Neural Network
        gradients = tape.gradient(loss_train, myMachine.trainable_variables)
        adam_optimizer.apply_gradients([
            (grad, var)
            for (grad, var) in zip(gradients, myMachine.trainable_variables)
            if grad is not None
        ])

        # Keep track of the loss after each batch.
        epoch_loss_avg_train.update_state(loss_train)
        epoch_loss_ae_train.update_state(ae_loss)
        epoch_loss_dmd_train.update_state(dmd_loss)
        epoch_loss_pred_train.update_state(pred_loss)

    for batch_test_data in test_batch:
        predictions_test = myMachine(batch_test_data)
        dmd_test = predictions_test[2]
        ae_test = predictions_test[3]
        pred_test = predictions_test[5]

        loss_test = myLoss(batch_test_data, predictions_test)

        epoch_loss_avg_test.update_state(loss_test)
        epoch_loss_ae_test.update_state(ae_test)
        epoch_loss_dmd_test.update_state(dmd_test)
        epoch_loss_pred_test.update_state(pred_test)

    train_loss_results.append(epoch_loss_avg_train.result())
    test_loss_results.append(epoch_loss_avg_test.result())

    train_dmd_loss.append(epoch_loss_dmd_train.result())
    train_ae_loss.append(epoch_loss_ae_train.result())
    train_pred_loss.append(epoch_loss_pred_train.result())

    test_dmd_loss.append(epoch_loss_dmd_test.result())
    test_ae_loss.append(epoch_loss_ae_test.result())
    test_pred_loss.append(epoch_loss_pred_test.result())

    if epoch % 15 == 0:
        # save plots in results folder. Plot the latent space, ae_reconstruction, and input_batch.
        create_plots(batch_training_data, predictions_train, hyp_params, epoch, train_loss_results, save_folder, \
                     "train")
        create_plots(batch_test_data, predictions_test, hyp_params, epoch, test_loss_results, save_folder, "test")

    if epoch % 10 == 0:
        # plot latent, input and reconstructed ae latest batch data.
        try:
            print_status_bar(epoch, hyp_params["num_epochs"], epoch_loss_avg_train.result(),
                             epoch_loss_avg_test.result(), time.process_time() - start_time,
                             log_file_path=os.path.join("results", save_folder, "log.txt"))
        except Exception:
            print("print status failed.")

    if epoch % 50 == 0 and epoch != 0:
        # plot loss curves.
        create_plots_of_loss(train_dmd_loss, train_ae_loss, test_dmd_loss, test_ae_loss, train_pred_loss,
                             test_pred_loss, myLoss.c1, myLoss.c2, myLoss.c3, epoch, save_folder)

        # save loss curves in pickle files.
        try:
            save_loss_curves(train_loss_results, test_loss_results, train_dmd_loss, test_dmd_loss, train_ae_loss,
                             test_ae_loss, train_pred_loss, test_pred_loss,
                             file_path=os.path.join("results", save_folder, "Loss"))
        except Exception:
            print("failed to save loss curves in pickle files. ")

        # save current machine.
        try:
            myMachine.autoencoder.save(os.path.join("models", "2021", "pendulum", save_folder), save_format='save_weights')
        except Exception:
            print("failed save machine. ")

    epoch += 1

# final summary of the network, again for diagnostic purposes.
myMachine.summary()<jupyter_output>0/200 ,loss_train: 35.57222 , loss_test: 6.4585347, run time = 636.125 sec.
{'fraction_non_alphanumeric': 0.16733247908344012, 'fraction_numerical': 0.016130247983719002, 'mean_word_length': 3.4915368991198377, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '562215', 'license': 'MIT', 'n_tokens_mistral': 4526, 'n_tokens_neox': 4276, 'n_words': 962, 'path': '.ipynb_checkpoints/dmd_autoencoder_pendulum_train-checkpoint.ipynb', 'repo_name': 'opaliss/dmd_autoencoder', 'size': '13267'}