<jupyter_start><jupyter_text>[View in Colaboratory](https://colab.research.google.com/github/huyhoang17/Colab_Temporary/blob/master/[Overfitting]Training_CTC_Vietnamese_Recognition_40epochs.ipynb)<jupyter_code># Load the Drive helper and mount
from google.colab import drive

# This will prompt for authorization.
drive.mount('/content/drive')
!ls drive/My\ Drive/Japanese_Recognition/datasets
# import os
import pickle
# TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']
# print(TPU_WORKER)
# !rm -rf drive/My\ Drive/Japanese_Recognition/datasets/pp_vn_handwriting_data
# !unzip drive/My\ Drive/Japanese_Recognition/datasets/pp_vn_handwriting_data.zip -d drive/My\ Drive/Japanese_Recognition/datasets
# import zipfile
# zip_ref = zipfile.ZipFile('drive/My Drive/Japanese_Recognition/datasets/pp_vn_handwriting_data.zip', 'r')
# zip_ref.extractall('drive/My Drive/Japanese_Recognition/datasets')
# zip_ref.close()
!ls drive/My\ Drive/Japanese_Recognition/datasets/pp_vn_handwriting_data | wc -l 
import pickle
with open('drive/My Drive/Japanese_Recognition/datasets/transcription.pk', 'rb') as f:  # noqa
    data = pickle.load(f)
no_samples = len(data)
print(no_samples)
pp_dataset = 'drive/My Drive/Japanese_Recognition/datasets/pp_vn_handwriting_data'
img_size = (1150, 32)
no_epochs = 5
import keras
import tensorflow as tf
print('TensorFlow version:', tf.__version__)
print('Keras version:', keras.__version__)
import os
from os.path import join
import json
import random
import itertools
import re
import datetime
import numpy as np
from scipy import ndimage
import pylab
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from keras import backend as K
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.layers import Input, Dense, Activation
from keras.layers import Reshape, Lambda
from keras.layers.merge import add, concatenate
from keras.models import Model, load_model
from keras.layers.recurrent import GRU
from keras.optimizers import SGD
from keras.utils.data_utils import get_file
from keras.preprocessing import image
import keras.callbacks
import cv2
chars = '\ !%"#&\'()*+,-./0123456789:;?AÁẢÀÃẠÂẤẨẦẪẬĂẮẲẰẴẶBCDĐEÉẺÈẼẸÊẾỂỀỄỆFGHIÍỈÌĨỊJKLMNOÓỎÒÕỌÔỐỔỒỖỘƠỚỞỜỠỢPQRSTUÚỦÙŨỤƯỨỬỪỮỰVWXYÝỶỲỸỴZaáảàãạâấẩầẫậăắẳằẵặbcdđeéẻèẽẹêếểềễệfghiíỉìĩịjklmnoóỏòõọôốổồỗộơớởờỡợpqrstuúủùũụưứửừữựvwxyýỷỳỹỵz'
len(chars)
chars_ = []
for char in chars:
  chars_.append(char)
print(chars_)
len(chars_)
import logging
def get_logger(name):
    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)

    # formatter
    fmt = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    )

    # handler
    handler = logging.StreamHandler()
    handler.setLevel(logging.DEBUG)
#     file_handler = logging.FileHandler(cf.LOGGING, mode='a')
#     file_handler.setLevel(logging.DEBUG)
    handler.setFormatter(fmt)
#     file_handler.setFormatter(fmt)

    # add handler to formatter
    logger.addHandler(handler)
#     logger.addHandler(file_handler)

    return logger
logger = get_logger(__name__)
def labels_to_text(letters, labels):
    return ''.join(list(map(lambda x: letters[x] if x < len(letters) else "", labels)))  # noqa


def text_to_labels(letters, text):
    return list(map(lambda x: letters.index(x), text))


def decode_batch(out):
    ret = []
    for j in range(out.shape[0]):
        out_best = list(np.argmax(out[j, 2:], 1))
        out_best = [k for k, g in itertools.groupby(out_best)]
        outstr = labels_to_text(out_best)
        ret.append(outstr)
    return ret


def decode_predict_ctc(out, chars, top_paths=1):
    results = []
    beam_width = 5
    if beam_width < top_paths:
        beam_width = top_paths
    for i in range(top_paths):
        lables = K.get_value(
            K.ctc_decode(
                out, input_length=np.ones(out.shape[0]) * out.shape[1],
                greedy=False, beam_width=beam_width, top_paths=top_paths
            )[0][i]
        )[0]
        text = labels_to_text(chars, lables)
        results.append(text)
    return results


def predit_a_image(model_p, pimg, top_paths=1):
    # c = np.expand_dims(a.T, axis=0)
    net_out_value = model_p.predict(pimg)
    top_pred_texts = decode_predict_ctc(net_out_value, top_paths)
    return top_pred_texts


def is_valid_str(letters, s):
    for ch in s:
        if ch not in letters:
            return False
    return True
def ctc_lambda_func(args):
    y_pred, labels, input_length, label_length = args
    # the 2 is critical here since the first couple outputs of the RNN
    # tend to be garbage:
    y_pred = y_pred[:, 2:, :]
    return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
def CRNN_model():
    act = 'relu'
    input_data = Input(name='the_input', shape=img_size + (1, ), dtype='float32')
    inner = Conv2D(16, (3, 3), padding='same',
                   activation=act, kernel_initializer='he_normal',
                   name='conv1')(input_data)
    inner = MaxPooling2D(pool_size=(2, 2), name='max1')(
        inner)
    inner = Conv2D(32, (3, 3), padding='same',
                   activation=act, kernel_initializer='he_normal',
                   name='conv2')(inner)
    inner = MaxPooling2D(pool_size=(2, 2), name='max2')(
        inner)
    inner = Conv2D(64, (3, 3), padding='same',
                   activation=act, kernel_initializer='he_normal',
                   name='conv3')(input_data)
    inner = MaxPooling2D(pool_size=(2, 2), name='max3')(
        inner)
    inner = Conv2D(128, (3, 3), padding='same',
                   activation=act, kernel_initializer='he_normal',
                   name='conv4')(inner)
    inner = MaxPooling2D(pool_size=(2, 2), name='max4')(
        inner)
    inner = Conv2D(256, (3, 3), padding='same',
                   activation=act, kernel_initializer='he_normal',
                   name='conv5')(inner)
    inner = MaxPooling2D(pool_size=(2, 2), name='max5')(
        inner)

#     conv_to_rnn_dims = (1150 // (2 ** 2),
#                         (32 // (2 ** 2)) * 16)
    conv_to_rnn_dims = (256, 572)
    print(conv_to_rnn_dims)
#     import pdb; pdb.set_trace()
    inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
  
    # cuts down input size going into RNN:
    # TIME_DENSE_SIZE = 256
    inner = Dense(256, activation=act, name='dense1')(inner)

    gru_1 = GRU(256, return_sequences=True,
                kernel_initializer='he_normal', name='gru1')(inner)
    gru_1b = GRU(256, return_sequences=True, go_backwards=True,
                 kernel_initializer='he_normal', name='gru1_b')(inner)
    gru1_merged = add([gru_1, gru_1b])
    gru_2 = GRU(256, return_sequences=True,
                kernel_initializer='he_normal', name='gru2')(gru1_merged)
    gru_2b = GRU(256, return_sequences=True, go_backwards=True,
                 kernel_initializer='he_normal', name='gru2_b')(gru1_merged)

    # transforms RNN output to character activations:
    # no unique labels
    inner = Dense(216, kernel_initializer='he_normal',
                  name='dense2')(concatenate([gru_2, gru_2b]))
    y_pred = Activation('softmax', name='softmax')(inner)

    Model(inputs=input_data, outputs=y_pred).summary()

    labels = Input(name='the_labels', shape=[256], dtype='float32')
    input_length = Input(name='input_length', shape=[1], dtype='int64')
    label_length = Input(name='label_length', shape=[1], dtype='int64')

    # loss function
    loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')(
        [y_pred, labels, input_length, label_length]
    )

    model = Model(inputs=[input_data, labels,
                          input_length, label_length], outputs=loss_out)

    y_func = K.function([input_data], [y_pred])

    return model, y_func
class TextSequenceGenerator(keras.utils.Sequence):
    """Generates data for Keras"""

    def __init__(self, samples, batch_size=16,
                 img_size=img_size, max_text_len=160,
                 downsample_factor=4, shuffle=True):
        # train 95, test 5
        imgs, gt_texts = [], []
        for sample in samples:
            img = list(sample.keys())[0]
            fn_path = os.path.join(pp_dataset, img.split('/')[-1])
            imgs.append(fn_path)
            gt_texts.append(list(sample.values())[0])
        self.imgs = imgs
        self.gt_texts = gt_texts

        self.max_text_len = max_text_len
        self.chars = chars
        self.blank_label = len(self.chars)
        self.ids = range(len(self.imgs))

        self.img_size = img_size
        self.img_w, self.img_h = self.img_size
        self.batch_size = batch_size
        self.downsample_factor = downsample_factor
        self.shuffle = shuffle
        self.on_epoch_end()

    def __len__(self):
        """Denotes the number of batches per epoch"""
        return int(np.floor(len(self.ids) / self.batch_size))

    def __getitem__(self, index):
        """Generate one batch of data"""
        indexes = self.indexes[index *
                               self.batch_size:(index + 1) * self.batch_size]

        ids = [self.ids[k] for k in indexes]

#         for id_ in [1820, 5915]:
#             if id_ in ids:
#                 ids.remove(id_)
        X, y = self.__data_generation(ids)

        return X, y

    def on_epoch_end(self):
        """Updates indexes after each epoch"""
        self.indexes = np.arange(len(self.ids))
        if self.shuffle:
            np.random.shuffle(self.indexes)

    def __data_generation(self, ids):
        """Generates data containing batch_size samples"""
        for i, id_ in enumerate(ids):
            img = cv2.imread(self.imgs[id_], cv2.IMREAD_GRAYSCALE)
            if img is None:
                ids.remove(id_)
                print("\n==> Error id: ", id_)
        size = len(ids)
        
        if K.image_data_format() == 'channels_first':
            X = np.ones([size, 1, self.img_w, self.img_h])
        else:
            X = np.ones([size, self.img_w, self.img_h, 1])
        Y = np.ones([size, self.max_text_len])
#         input_length = np.ones((size, 1), dtype=np.float32) * \
#             (self.img_w // self.downsample_factor - 2)
        input_length = np.ones((size, 1), dtype=np.float32) * 254
        label_length = np.zeros((size, 1), dtype=np.float32)

        # Generate data
        for i, id_ in enumerate(ids):
            
            img = cv2.imread(self.imgs[id_], cv2.IMREAD_GRAYSCALE)  # (h, w)
            if img is None:
                continue
#             img = 255 - img  # bg: black, text: white
            # bg: white, text: black
            ratio = img.shape[0] / self.img_h
            new_w = int(img.shape[1] / ratio) + 1
            resized_image = cv2.resize(img, (new_w, self.img_h))  # (h, w)
            img = cv2.copyMakeBorder(
                resized_image, 0, 0, 0, self.img_w - resized_image.shape[1],
                cv2.BORDER_CONSTANT, value=0
            )  # (h, w)
            img = img / 255  # (h, w)

            if K.image_data_format() == 'channels_first':
                img = np.expand_dims(img, 0)  # (1, h, w)
                img = np.expand_dims((0, 2, 1))  # (1, w, h)
            else:
                img = np.expand_dims(img, -1)  # (h, w, 1)
                img = img.transpose((1, 0, 2))  # (w, h, 1)

            X[i] = img
            text2label = text_to_labels(self.chars, self.gt_texts[id_])
            Y[i] = text2label + \
                [self.blank_label for _ in range(
                    self.max_text_len - len(text2label))]
            label_length[i] = len(self.gt_texts[id_])

        inputs = {
            'the_input': X,
            'the_labels': Y,
            'input_length': input_length,
            'label_length': label_length,
        }
        outputs = {'ctc': np.zeros([size])}

        return (inputs, outputs)
def train(pretrained=False):

    with open('drive/My Drive/Japanese_Recognition/datasets/transcription.pk', 'rb') as f:  # noqa
        data = pickle.load(f)
    no_samples = len(data)
    no_train_set = int(no_samples * 0.95)
    no_val_set = no_samples - no_train_set
    logger.info("No train set: %d", no_train_set)
    logger.info("No val set: %d", no_val_set)

    train_set = TextSequenceGenerator(
        data[:no_train_set],
        img_size=img_size, max_text_len=256,
        downsample_factor=4,
        shuffle=True
    )
#     import pdb; pdb.set_trace()
    test_set = TextSequenceGenerator(
        data[no_train_set:],
        img_size=img_size, max_text_len=256,
        downsample_factor=4,
        shuffle=False
    )

    model, y_func = CRNN_model()
    model.load_weights('drive/My Drive/Japanese_Recognition/models/best_model_CTC_jps_25epochs.h5')
    
    sgd = SGD(lr=0.005, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
    model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)

    ckp = ModelCheckpoint(
        "drive/My Drive/Japanese_Recognition/models/model_cp_CTC_jps.hdf5".format(no_epochs), monitor='val_loss',
        verbose=1, save_best_only=True, save_weights_only=True
    )
    earlystop = EarlyStopping(
        monitor='val_loss', min_delta=0, patience=10, verbose=0, mode='min'
    )

#     model = tf.contrib.tpu.keras_to_tpu_model(
#     model,
#     strategy=tf.contrib.tpu.TPUDistributionStrategy(
#         tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)))
    
    model.fit_generator(generator=train_set,
                        steps_per_epoch=no_train_set // 16,
                        epochs=10,
                        validation_data=test_set,
                        validation_steps=no_val_set // 16,
                        callbacks=[ckp, earlystop])

    return model, y_func
# from keras.models import model_from_json

# with open('drive/My Drive/Japanese_Recognition/models/config_jps_25epochs.json') as f:
#     json_string = f.read()

# model = model_from_json(json_string)
# model.load_weights('drive/My Drive/Japanese_Recognition/models/best_model_CTC_jps_25epochs.h5')
model, y_func = train()

model_json = model.to_json()
# with open('drive/My Drive/Japanese_Recognition/models/config_jps_{}epochs.json'.format(no_epochs), 'w') as f:
with open('drive/My Drive/Japanese_Recognition/models/config_jps.json', 'w') as f:
    f.write(model_json)

# model.save_weights('drive/My Drive/Japanese_Recognition/models/best_model_CTC_jps_{}epochs.h5'.format(no_epochs))
model.save_weights('drive/My Drive/Japanese_Recognition/models/best_model_CTC_jps.h5')
from keras.models import model_from_json

with open('drive/My Drive/Japanese_Recognition/models/config_jps_{}epochs.json'.format(no_epochs)) as f:
    json_string = f.read()

model = model_from_json(json_string)
model.load_weights('drive/My Drive/Japanese_Recognition/models/best_model_CTC_jps_{}epochs.h5'.format(no_epochs))
input_data = model.get_layer('the_input').output
y_pred = model.get_layer('softmax').output
model_p = Model(inputs=input_data, outputs=y_pred)
import pickle
with open('drive/My Drive/Japanese_Recognition/datasets/transcription.pk', 'rb') as f:  # noqa
    data = pickle.load(f)
no_samples = len(data)
no_train_set = int(no_samples * 0.95)
no_val_set = no_samples - no_train_set
test_set = TextSequenceGenerator(
    data[no_train_set:],
    img_size=img_size, max_text_len=256,
    downsample_factor=4,
    shuffle=False
)
train_set = TextSequenceGenerator(
    data[:no_train_set],
    img_size=img_size, max_text_len=256,
    downsample_factor=4,
    shuffle=True
)
texts = []
for sample in data:
  texts.append(list(sample.values())[0])
print(max(map(len, texts)))
# import scipy.misc
# scipy.misc.imsave('outfile.jpg', image_array)
def predict(data_iter, index_batch, index_img):
  samples = data_iter[index_batch]
  img = samples[0]['the_input'][index_img]
  # plt.imshow(np.squeeze(img).T)
  img = np.expand_dims(img, axis=0)
  print(img.shape)
  
  net_out_value = model_p.predict(img)
  print(net_out_value.shape)
  pred_texts = top_pred_texts = decode_predict_ctc(net_out_value, chars_)
  print(pred_texts[0].strip())
  gt_texts = data_iter[index_batch][0]['the_labels'][index_img]
  gt_texts = labels_to_text(chars_, gt_texts.astype(int))
  print(gt_texts)
for i in range(16):
    predict(train_set, 1, i)
for i in range(16):
    predict(test_set, 1, i)
# OVERFITTING<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12194079951174855, 'fraction_numerical': 0.018126335062557217, 'mean_word_length': 3.0660049627791564, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '166377', 'license': 'MIT', 'n_tokens_mistral': 6149, 'n_tokens_neox': 5828, 'n_words': 1079, 'path': '[Overfitting]Training_CTC_Vietnamese_Recognition_40epochs.ipynb', 'repo_name': 'huyhoang17/Colab_Temporary', 'size': '16385'}