<jupyter_start><jupyter_text>LSTM UNET TEST<jupyter_code>from model import *
from data import *
import os
import sys

ROOTPATH = os.path.abspath("./")
sys.path.append(ROOTPATH)
train_dir = "./TRAIN/"
val_dir = "./VAL/" 

test_dir = "./test/"

all_list = os.listdir(train_dir)
train_dataset = [train_dir+s_dir+'/' for s_dir in all_list]

all_list = os.listdir(val_dir)
val_dataset = [val_dir+s_dir+'/' for s_dir in all_list]<jupyter_output>/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
Using TensorFlow backend.<jupyter_text>Create data generator<jupyter_code>train_data_generator = data_generator(train_dataset,batch_size=1)
val_data_generator = data_generator(val_dataset,batch_size=1)<jupyter_output><empty_output><jupyter_text>Tensorboard records (optional) need install tensorboard<jupyter_code>from keras.callbacks import TensorBoard
class TB(TensorBoard):
    def __init__(self, log_every=1, **kwargs):
        super().__init__(**kwargs)
        self.log_every = log_every
        self.counter = 0
    
    def on_batch_end(self, batch, logs=None):
        self.counter+=1
        if self.counter%self.log_every==0:
            for name, value in logs.items():
                if name in ['batch', 'size']:
                    continue
                summary = tf.Summary()
                summary_value = summary.value.add()
                summary_value.simple_value = value.item()
                summary_value.tag = name
                self.writer.add_summary(summary, self.counter)
            self.writer.flush()
        
        super().on_batch_end(batch, logs)
        
tensorboard_log = TB(log_dir='./Graph', histogram_freq=0, write_graph=True, write_images=True)
print(model.summary())<jupyter_output><empty_output><jupyter_text>Compile model<jupyter_code>model = LSTMUnet()
model_checkpoint = ModelCheckpoint('lstm_unet.hdf5', monitor='loss',verbose=1, save_best_only=True)
print(model.summary())<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>model.fit_generator(
    train_data_generator,
    validation_data=val_data_generator,
    validation_steps =400,
    steps_per_epoch=2325,
    epochs=15,
    callbacks=[model_checkpoint, tensorboard_log])<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code>import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
model_val = LSTMUnet()
model_val.load_weights('lstm_3.hdf5') # load your pretrained model
test_dir = "/cole_driver/dff/PLS3Bx10/"
test_dir2 = "/cole_driver/dff/PLS3Sx50/" #change to your test folder

input_ = data_feed(test_dir)
input_s = np.zeros((1,)+input_.shape,dtype=np.uint8)
input_s[0] = input_

out_d = model_val.predict(input_s)
imd = plt.imshow(out_d[0,::,::,0])
plt.colorbar()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11815754339118825, 'fraction_numerical': 0.013684913217623497, 'mean_word_length': 4.519337016574585, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '106248', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1050, 'n_words': 179, 'path': 'test.ipynb', 'repo_name': 'NerdToMars/Fringe-pattern-denoising-based-on-deep-learning', 'size': '2996'}