# Path: util.py
# class LossHistory(Callback):
#     def on_train_begin(self, logs={}):
#         self.losses = []
#         self.acces = []
# 
#     def on_batch_end(self, batch, logs={}):
#         self.losses.append(logs.get('loss'))
#         self.acces.append(logs.get('acc'))


from keras.models import Sequential
from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent
from keras.utils.visualize_util import plot  # Add by Steven Robot
from keras.callbacks import ModelCheckpoint  # Add by Steven Robot
from keras.callbacks import Callback         # Add by Steven Robot
from util import LossHistory                 # Add by Steven Robot
import numpy as np
import dataset                               # Add by Steven Robot
import visualization                         # Add by Steven Robot
import time                                  # Add by Steven Robot
import os                                    # Add by Steven Robot
import lstm2ntm                              # Add by Steven Robot
# For each of step of the output sequence, decide which character should be chosen
model.add(TimeDistributed(Dense(INPUT_DIMENSION_SIZE+2)))
# model.add(Activation('softmax'))
# model.add(Activation('hard_sigmoid'))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              # loss='mse',
              optimizer='adam',
              metrics=['accuracy'])

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print("Model architecture")
plot(model, show_shapes=True, to_file=FOLDER+"lstm_associative_recall.png")
print("Model summary")
print(model.summary())
print("Model parameter count")
print(model.count_params())

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print("Training...")
# Train the model each generation and show predictions against the
# validation dataset
for iteration in range(1, 200):
    print()
    print('-' * 78)
    print(time.strftime('%Y-%m-%d %H:%M:%S'))
    print('Iteration', iteration)

# Path: util.py
# class LossHistory(Callback):
#     def on_train_begin(self, logs={}):
#         self.losses = []
#         self.acces = []
# 
#     def on_batch_end(self, batch, logs={}):
#         self.losses.append(logs.get('loss'))
#         self.acces.append(logs.get('acc'))


# Path: unit_test/lstm2ntm_unit_test.py
from keras.models import Sequential
from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent
from keras.utils.visualize_util import plot  # Add by Steven Robot
from keras.callbacks import ModelCheckpoint  # Add by Steven Robot
from keras.callbacks import Callback         # Add by Steven Robot
from util import LossHistory                 # Add by Steven Robot
import numpy as np
import dataset                               # Add by Steven Robot
import visualization                         # Add by Steven Robot
import time                                  # Add by Steven Robot
import os                                    # Add by Steven Robot
import lstm2ntm                              # Add by Steven Robot

# For each of step of the output sequence, decide which character should be chosen
model.add(TimeDistributed(Dense(INPUT_DIMENSION_SIZE+2)))
# model.add(Activation('softmax'))
# model.add(Activation('hard_sigmoid'))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              # loss='mse',
              optimizer='adam',
              metrics=['accuracy'])

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print("Model architecture")
plot(model, show_shapes=True, to_file=FOLDER+"lstm_associative_recall.png")
print("Model summary")
print(model.summary())
print("Model parameter count")
print(model.count_params())

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print("Training...")
# Train the model each generation and show predictions against the
# validation dataset
for iteration in range(1, 200):
    print()
    print('-' * 78)
    print(time.strftime('%Y-%m-%d %H:%M:%S'))
    print('Iteration', iteration)

    history = LossHistory()