<jupyter_start><jupyter_text>Connect Google Drive<jupyter_code>from google.colab import drive
drive.mount('/content/gdrive')<jupyter_output>Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).<jupyter_text>Import<jupyter_code>import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import cv2
import os

from tensorflow import keras
from tensorflow.keras.layers import Input
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import MaxPool2D
from tensorflow.keras.layers import ReLU
from tensorflow.keras.layers import Softmax
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Lambda
from tensorflow.keras.layers import Reshape
from tensorflow.keras.layers import LSTM
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import concatenate
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.metrics import sparse_top_k_categorical_accuracy
from tensorflow.keras.callbacks import CSVLogger
from ast import literal_eval<jupyter_output><empty_output><jupyter_text>Parameters  and  Work-Space Paths<jupyter_code># parameters
BATCH_SIZE = 200
EPOCHS = 20
STEPS_PER_EPOCH = 1000
VALIDATION_STEPS = 50
EVALUATE_STEPS = 850
IMAGE_SIZE = 128
LINE_SIZE = 3


# load path
TRAIN_DATA_PATH = 'gdrive/My Drive/QW/Data/Data_10000/All_classes_10000.csv'
VALID_DATA_PATH = 'gdrive/My Drive/QW/Data/My_test_data/My_test_data.csv'
LABEL_DICT_PATH = 'gdrive/My Drive/QW/Data/labels_dict.npy'

# save path
CKPT_PATH = 'gdrive/My Drive/QW/Notebook/Quick Draw/Thesis_pre_research/(3-2)Stroke_LSTM-MobileNet/best_model_3_2_new.ckpt'
LOSS_PLOT_PATH = 'gdrive/My Drive/QW/Notebook/Quick Draw/Thesis_pre_research/(3-2)Stroke_LSTM-MobileNet/loss_plot_3_2_new.png'
ACC_PLOT_PATH = 'gdrive/My Drive/QW/Notebook/Quick Draw/Thesis_pre_research/(3-2)Stroke_LSTM-MobileNet/acc_plot_3_2_new.png'
LOG_PATH = 'gdrive/My Drive/QW/Notebook/Quick Draw/Thesis_pre_research/(3-2)Stroke_LSTM-MobileNet/Log_3_2_new.log'
print('finish!')<jupyter_output>finish!<jupyter_text>Generator<jupyter_code>def generate_data(data, batch_size, choose_recognized):
    data = data.sample(frac = 1)
    
    while 1:
        
#         get columns' values named 'drawing', 'word' and 'recognized'
        drawings = data["drawing"].values
        drawing_recognized = data["recognized"].values
        drawing_class = data["word"].values
      
#         initialization
        cnt = 0
        data_X =[]
        data_Y =[]
        
#         generate batch
        for i in range(len(drawings)):
            if choose_recognized:
                if drawing_recognized[i] == 'False':    #Choose according to recognized value
                    continue
            draw = drawings[i]
            stroke_vec = literal_eval(draw)
            
            l = len(stroke_vec)
            stroke_set = []
            if l <= 3:
                if l == 1:
                    stroke_set =[[0],[0],[0]]
                if l == 2:
                    stroke_set =[[0],[1],[1]]
                if l == 3:
                    stroke_set =[[0],[1],[2]]
            if l > 3:
                a = l // 3
                b = l % 3
                c = (a + 1) * 3
                d = c - l
                n = 0
                for h in range(0,d):
                    temp = []
                    for k in range(a):
                        temp.append(n)
                        n += 1
                    stroke_set.append(temp)
                for h in range(d,3):
                    temp = []
                    for k in range(a+1):
                        temp.append(n)
                        n += 1
                    stroke_set.append(temp)
            
            img = np.zeros([256, 256])
            x = []
            stroke_num = 0
            for j in range(3):
                stroke_index = stroke_set[j]
                for m in list(stroke_index):
                    line = np.array(stroke_vec[m]).T
                    cv2.polylines(img, [line], False, (255-(13*min(stroke_num,10))), LINE_SIZE)
                    stroke_num += 1
                img_copy = img.copy()
                img_x = cv2.resize(img_copy,(IMAGE_SIZE,IMAGE_SIZE),interpolation = cv2.INTER_NEAREST)
                x.append(img_x)
            x = np.array(x)
            x = x[:,:,:,np.newaxis]
            label = drawing_class[i]
            y = labels2nums_dict[label]
            data_X.append(x)
            data_Y.append(y)
            cnt += 1
            if cnt==batch_size:        #generate a batch when cnt reaches batch_size 
                cnt = 0
                yield (np.array(data_X), np.array(data_Y))
                data_X = []
                data_Y = []
print("finish!")<jupyter_output>finish!<jupyter_text>Callbacks<jupyter_code># define a class named LossHitory 
class LossHistory(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = {'batch':[], 'epoch':[]}
        self.accuracy = {'batch':[], 'epoch':[]}
        self.val_loss = {'batch':[], 'epoch':[]}
        self.val_acc = {'batch':[], 'epoch':[]}

    def on_batch_end(self, batch, logs={}):
        self.losses['batch'].append(logs.get('loss'))
        self.accuracy['batch'].append(logs.get('acc'))
        self.val_loss['batch'].append(logs.get('val_loss'))
        self.val_acc['batch'].append(logs.get('val_acc'))

    def on_epoch_end(self, batch, logs={}):
        self.losses['epoch'].append(logs.get('loss'))
        self.accuracy['epoch'].append(logs.get('acc'))
        self.val_loss['epoch'].append(logs.get('val_loss'))
        self.val_acc['epoch'].append(logs.get('val_acc'))

    def loss_plot(self, loss_type, loss_fig_save_path, acc_fig_save_path):
        iters = range(len(self.losses[loss_type]))
        plt.figure('acc')
        plt.plot(iters, self.accuracy[loss_type], 'r', label='train acc')
        plt.plot(iters, self.val_acc[loss_type], 'b', label='val acc')
        plt.grid(True)
        plt.xlabel(loss_type)
        plt.ylabel('acc')
        plt.legend(loc="upper right")
        plt.savefig(acc_fig_save_path)
        plt.show()
        
        
        plt.figure('loss')
        plt.plot(iters, self.losses[loss_type], 'g', label='train loss')
        plt.plot(iters, self.val_loss[loss_type], 'k', label='val loss')
        plt.grid(True)
        plt.xlabel(loss_type)
        plt.ylabel('loss')
        plt.legend(loc="upper right")
        plt.savefig(loss_fig_save_path)
        plt.show()
        
# create a object from LossHistory class
History = LossHistory()

print("finish!")

cp_callback = tf.keras.callbacks.ModelCheckpoint(
    CKPT_PATH, 
    verbose = 1, 
    monitor='val_acc', 
    mode = 'max', 
    save_best_only=True)

print("finish!")
ReduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=3,
                      min_delta=0.005, mode='max', cooldown=3, verbose=1)
csv_logger = CSVLogger(LOG_PATH, separator=',', append=True)<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code># load train data and valid data
#  labels_dict and data path

# labels convert into nums
labels_dict = np.load(LABEL_DICT_PATH)
labels2nums_dict = {v: k for k, v in enumerate(labels_dict)}

# read csv 
train_data = pd.read_csv(TRAIN_DATA_PATH)
valid_data = pd.read_csv(VALID_DATA_PATH)

print('finish!')<jupyter_output>finish!<jupyter_text>Model<jupyter_code>def _conv_block(inputs, filters, kernel=(3, 3), strides=(1, 1)):
    filters = int(filters)
    x = keras.layers.ZeroPadding2D(padding=(1, 1))(inputs)
    x = keras.layers.Conv2D(
        filters,
        kernel,
        padding='valid',
        use_bias=False,
        strides=strides)(x)
    x = keras.layers.BatchNormalization()(x)
    return keras.layers.ReLU(6)(x)


def _depthwise_conv_block(inputs,
                          pointwise_conv_filters,
                          depth_multiplier=1,
                          strides=(1, 1)):

    x = keras.layers.ZeroPadding2D(padding=(1, 1))(inputs)
    x = keras.layers.DepthwiseConv2D( 
        (3, 3),
        padding='valid',
        depth_multiplier=depth_multiplier,
        strides=strides,
        use_bias=False)(x)
    x = keras.layers.BatchNormalization()(x)
    x = keras.layers.ReLU(6)(x)

    x = keras.layers.Conv2D(
        pointwise_conv_filters, (1, 1),
        padding='same',
        use_bias=False,
        strides=(1, 1))(x)
    x = keras.layers.BatchNormalization()(x)
    return keras.layers.ReLU(6)(x)



def mobile_net(M):
    x_input = keras.layers.Lambda(lambda x: x[:,M])(X_INPUT)
    x = _conv_block(x_input, 32, strides=(2, 2))
    x = _depthwise_conv_block(x, 64,  depth_multiplier=1)
    x = _depthwise_conv_block(x, 128,  depth_multiplier=1, strides=(2, 2))
    x = _depthwise_conv_block(x, 128,  depth_multiplier=1)
    x = _depthwise_conv_block(x, 256,  depth_multiplier=1, strides=(2, 2))
    x = _depthwise_conv_block(x, 256,  depth_multiplier=1)
    x = _depthwise_conv_block(x, 512,  depth_multiplier=1, strides=(2, 2))
    x = _depthwise_conv_block(x, 512,  depth_multiplier=1)
    x = _depthwise_conv_block(x, 512,  depth_multiplier=1)
    x = _depthwise_conv_block(x, 512,  depth_multiplier=1)
    x = _depthwise_conv_block(x, 512,  depth_multiplier=1)
    x = _depthwise_conv_block(x, 512,  depth_multiplier=1)
    x = _depthwise_conv_block(x, 1024, depth_multiplier=1, strides=(2, 2))
    x = _depthwise_conv_block(x, 1024, depth_multiplier=1)
    x = keras.layers.GlobalAveragePooling2D()(x)
    return x

X_INPUT = keras.layers.Input(shape=(3,IMAGE_SIZE,IMAGE_SIZE,1))
x1_output = mobile_net(0)
x2_output = mobile_net(1)
x3_output = mobile_net(2)

x = keras.layers.concatenate([x1_output, x2_output,x3_output],axis = 1)

x = keras.layers.Reshape((3,1024))(x)
x = keras.layers.LSTM(512, return_sequences = True)(x)
x = keras.layers.Dropout(0.5)(x)
x = keras.layers.LSTM(512, return_sequences = False)(x)
x = keras.layers.Dropout(0.5)(x)
xx = keras.layers.concatenate([x,x3_output],axis = 1)
xx = keras.layers.Dense(340)(xx)
X_OUTPUT = keras.layers.Activation('softmax')(xx)

MODEL = keras.Model(inputs=X_INPUT, outputs= X_OUTPUT)
MODEL.summary()<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            (None, 3, 128, 128,  0          [...]<jupyter_text>TPU Complie<jupyter_code>model = MODEL
TPU_model = tf.contrib.tpu.keras_to_tpu_model(
    model,
    strategy=tf.contrib.tpu.TPUDistributionStrategy(
            tf.contrib.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
        )
    )
# model = model.load_weights(CKPT_PATH)
TPU_model.compile(loss=tf.keras.losses.sparse_categorical_crossentropy,
                  optimizer=tf.train.AdamOptimizer(learning_rate=1e-3),
                  metrics=['accuracy'])


print('finish')<jupyter_output>WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.

INFO:tensorflow:Querying Tensorflow master (grpc://10.52.152.90:8470) for TPU system metadata.
INFO:tensorflow:Found TPU system:
INFO:tensorflow:*** Num TPU Cores: 8
INFO:tensorflow:*** Num TPU Workers: 1
INFO:tensorflow:*** Num TPU Cores Per Worker: 8
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 4271535773542931026)
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 17770548513360908256)
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 18414946529386799766)
INFO:tensorflow:*[...]<jupyter_text>Train<jupyter_code>print('start training')
# callbacks = [History, cp_callback]

history = TPU_model.fit_generator(generate_data(train_data, BATCH_SIZE, True),
                              steps_per_epoch = STEPS_PER_EPOCH,
                              epochs = EPOCHS,
                              validation_data = generate_data(valid_data, BATCH_SIZE, False) ,
                              validation_steps = VALIDATION_STEPS,
                              verbose = 1,
                              initial_epoch = 0,
                              callbacks = [History,cp_callback,csv_logger]
                             )
print("finish training")

History.loss_plot('epoch', LOSS_PLOT_PATH, ACC_PLOT_PATH)

print('finish!')<jupyter_output>start training
Epoch 1/20
INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(25,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(25, 3, 128, 128, 1), dtype=tf.float32, name='input_1_10'), TensorSpec(shape=(25, 1), dtype=tf.int32, name='activation_target_10')]
INFO:tensorflow:Overriding default placeholder.
INFO:tensorflow:Remapping placeholder for input_1
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/tpu/python/tpu/keras_support.py:302: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
INFO:tensorflow:Started compiling
INFO:tensorflow:Finished compiling.[...]<jupyter_text>Evaluate<jupyter_code>def top_3_accuracy(X, Y):
        return sparse_top_k_categorical_accuracy(X, Y, 3)
  
def top_5_accuracy(X, Y):
        return sparse_top_k_categorical_accuracy(X, Y, 5)
  
model_E = MODEL  


model_E.compile(loss=tf.keras.losses.sparse_categorical_crossentropy,
                    optimizer=tf.train.AdamOptimizer(learning_rate=1e-4),
                    metrics=['accuracy',top_3_accuracy, top_5_accuracy])

model_weights_path = CKPT_PATH   
model_E.load_weights(model_weights_path)
print('finish')
result = model_E.evaluate_generator(
    generate_data(valid_data, BATCH_SIZE, False),
    steps = EVALUATE_STEPS,
    verbose = 1
)

print('loss:', result[0])
print('top1 accuracy:', result[1])
print('top3 accuracy:', result[2])
print('top5 accuracy:', result[3])<jupyter_output>850/850 [==============================] - 425s 500ms/step
loss: 1.001194847681943
top1 accuracy: 0.748688234721913
top3 accuracy: 0.8958705863531898
top3 accuracy: 0.9259117664309109
{'fraction_non_alphanumeric': 0.12741460918510938, 'fraction_numerical': 0.03281309965459895, 'mean_word_length': 3.152722443559097, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '479125', 'license': 'Apache-2.0', 'n_tokens_mistral': 5612, 'n_tokens_neox': 5223, 'n_words': 970, 'path': 'src/notebook/(3_2)Stroke_LSTM_MobileNet.ipynb', 'repo_name': 'AlsoSprachZarathushtra/Quick-Draw-Recognition', 'size': '15634'}