<jupyter_start><jupyter_text>Parameter setting and import libraries<jupyter_code>from __future__ import print_function
#
import os
import glob
import re
import argparse
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split

import matplotlib.pyplot as plt

import time
from time import sleep
from tqdm import tqdm # if use notebook

import multiprocessing as mp
from multiprocessing import Event
import queue

from PIL import Image
import cv2
import imgaug as ia
from imgaug import augmenters as iaa
import random

parser = argparse.ArgumentParser()
parser.add_argument('--gpu_id', default=5)
parser.add_argument('--image_dir', default="/data/seanyu/cat_dog/dataset/")
parser.add_argument('--save_dir', default='./result')
parser.add_argument('--batch_size', default=48, type=int)
parser.add_argument('--do_augment', default=True, type = bool)
parser.add_argument('--epochs', default=100, type=int)
parser.add_argument('--lr', default=0.00017, type=float)
parser.add_argument('--image_size', default=(256,256,3), type = int)
parser.add_argument('--n_classes', default=2, type = int)
parser.add_argument('--n_batch', default=100, type = int)
parser.add_argument('--train_ratio', default=0.9, type = float)
parser.add_argument('--model_file_name', default = 'model.h5')
parser.add_argument('--n_threads', default = 4, type = int)
parser.add_argument('--dq_size', default = 6, type = int)

FLAGS = parser.parse_args([])
print(FLAGS)
FLAGS.gpu_id = "6"
FLAGS.image_dir = "/data/seanyu/cat_dog/dataset/"<jupyter_output><empty_output><jupyter_text>Check path and load data<jupyter_code>os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
import tensorflow as tf

if not os.path.exists(FLAGS.save_dir):
    os.makedirs(FLAGS.save_dir)

model_dir = FLAGS.save_dir + '/model'

"""  Get data """
d_train = FLAGS.image_dir + '/train/'
d_test = FLAGS.image_dir + '/test1/'

image_train_list = glob.glob(d_train + '*.jpg')
image_test_list = glob.glob(d_test + '*.jpg')

df_train = pd.DataFrame({'img_path': image_train_list})
df_test = pd.DataFrame({'img_path': image_test_list})

df_train['cate'] = df_train.img_path.apply(os.path.basename)
df_train['cate'] = [i.split(".")[0] for i in list(df_train.cate)]
df_train.cate = df_train.cate.replace({'dog': 0, 'cat': 1})

nb_epoch = FLAGS.epochs

df_train_0, df_val_0 = train_test_split(df_train[df_train['cate'] == 0], test_size = 1-FLAGS.train_ratio)
df_train_1, df_val_1 = train_test_split(df_train[df_train['cate'] == 1], test_size = 1-FLAGS.train_ratio)

df_val = pd.concat((df_val_0, df_val_1)).reset_index(drop = True)

del df_val_0, df_val_1
try:
    import imgaug as ia
    from imgaug import augmenters as iaa
except:
    print("Import Error, Please make sure you have imgaug")
        
try:
    import sys
    sys.path.append("/mnt/deep-learning/usr/seanyu/common_tools/")
    from customized_imgaug_func import keypoint_func, img_channelswap
except:
    print("Warning, if you used customized imgaug function")
    
class Augmentation_Setup(object):  
    sometimes = lambda aug: iaa.Sometimes(0.5, aug)
    lesstimes = lambda aug: iaa.Sometimes(0.2, aug)
    
    augmentation = iaa.Sequential([
        iaa.Fliplr(0.5, name="FlipLR"),
        iaa.Flipud(0.5, name="FlipUD"),
        iaa.ContrastNormalization((0.8, 1.2), name = "Contrast"),
        iaa.Add((-15, 15), per_channel = 0.5),
        iaa.OneOf([iaa.Multiply((0.8, 1.2), per_channel = 0.5, name = "Multiply"),
                   iaa.AddToHueAndSaturation((-15,30),name = "Hue"),
                  ]),
        sometimes(iaa.GaussianBlur((0, 1.0), name="GaussianBlur")),
        iaa.OneOf([iaa.Affine(rotate = 90),
                   iaa.Affine(rotate = 180),
                   iaa.Affine(rotate = 270)]),
        sometimes(iaa.Affine(
                    scale = (0.8,1.2),
                    #translate_percent = (-0.2, 0.2),
                    #rotate = (-15, 15),
                    mode = 'wrap'
                    )),
        iaa.OneOf([iaa.AdditiveGaussianNoise(scale=0.05*255, name="Noise"),
               iaa.CoarseDropout((0.05, 0.15), size_percent=(0.01, 0.05), name = 'Cdrop')
               ]),
    ])<jupyter_output><empty_output><jupyter_text>Data Generator<jupyter_code>class GetDataset():
    def __init__(self, df_list, class_id, n_classes, f_input_preproc, image_size=(256,256,3), onehot=True, augmentation=None):
        
        self.df_list = df_list
        self.class_id = class_id
        self.n_classes = n_classes
        self.preproc = f_input_preproc
        self.image_size = image_size
        self.onehot = onehot
        self.aug = augmentation
        
        ## Init ##
        self.df_list = self.df_list.sample(frac=1.).reset_index(drop=True)
        self.current_index = 0
    
    def __len__(self):
        return len(self.df_list)
    
    def __getitem__(self, idx):
        
        img = self.load_image(img_path=self.df_list.iloc[self.current_index]['img_path'], image_size=self.image_size)
        
        if self.aug is not None:
            img = self.aug.augment_image(img)
            
        img = img.astype(np.float32)
        
        if self.preproc is not None:
            img = self.preproc(img)
        
        label = self.class_id
        if self.onehot:
             label = tf.keras.utils.to_categorical(label, num_classes=self.n_classes)
        
        self.current_index = (self.current_index + 1) % len(self.df_list)
        return img, label
    
    def __next__(self):
        return self.__getitem__(idx=self.current_index)
    
    @staticmethod
    def load_image(img_path, image_size):
        img = cv2.imread(img_path)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        img = cv2.resize(img, (image_size[0], image_size[1]))
        return img
    
class Customized_dataloader():
    """
    1. Compose multiple generators together
    2. Make this composed generator into multi-processing function
    """
    def __init__(self, list_dataset, batch_size_per_dataset=16, queue_size=128, num_workers=0):
        """
        Args:
            - list_dataset: put generator object as list [gen1, gen2, ...]
            - batch_size_per_dataset: bz for each generator (total_batch_size/n_class)
            - queue_size: queue size
            - num_workers: start n workers to get data
        
        Action: Call with next
        """
        self.list_dataset = list_dataset
        self.batch_size_per_dataset = batch_size_per_dataset
        self.sample_queue = mp.Queue(maxsize = queue_size)
        
        self.jobs = num_workers
        self.events = list()
        self.workers = list()
        for i in range(num_workers):
            event = Event()
            work = mp.Process(target = enqueue, args = (self.sample_queue, event, self.compose_data))
            work.daemon = True
            work.start()
            self.events.append(event)
            self.workers.append(work)
        print("workers ready")
        
    def __next__(self):
        return self.sample_queue.get()
    
    def compose_data(self):
        while True:
            imgs, labels = [], []
            for z in range(self.batch_size_per_dataset):
                data = [next(i) for i in self.list_dataset]
                img, label = zip(*data)
                imgs.append(np.array(img))
                labels.append(np.array(label))
            yield np.concatenate(imgs), np.concatenate(labels)
    
    def stop_worker(self):
        for t in self.events:
            t.set()
        for i, t in enumerate(self.workers):
            t.join(timeout = 1)
        print("all_worker_stop")

# ----- #
def enqueue(queue, stop, gen_func):
    gen = gen_func()
    while True:
        if stop.is_set():
            return
        queue.put(next(gen))
def preproc(img):
    return (img - img.min()) / (img.max() - img.min())
USE_RESNET_PREPROC = False
dog_train = GetDataset(df_list=df_train[df_train['cate'] == 0],
                       class_id=0, n_classes=2,
                       f_input_preproc=preproc if not USE_RESNET_PREPROC else tf.keras.applications.resnet50.preprocess_input,
                       augmentation=Augmentation_Setup.augmentation, 
                       onehot= True, 
                       image_size=(256,256,3))

cat_train = GetDataset(df_list=df_train[df_train['cate'] == 1], 
                       class_id=1, n_classes=2, 
                       f_input_preproc=preproc if not USE_RESNET_PREPROC else tf.keras.applications.resnet50.preprocess_input,
                       augmentation=Augmentation_Setup.augmentation, 
                       onehot= True, 
                       image_size=(256,256,3))

dog_valid = GetDataset(df_list=df_val[df_val['cate'] == 0], 
                       class_id=0, n_classes=2,
                       f_input_preproc=preproc if not USE_RESNET_PREPROC else tf.keras.applications.resnet50.preprocess_input,
                       augmentation=None, 
                       onehot= True, 
                       image_size=(256,256,3))

cat_valid = GetDataset(df_list=df_val[df_val['cate'] == 1], 
                       class_id=1, n_classes=2, 
                       f_input_preproc=preproc if not USE_RESNET_PREPROC else tf.keras.applications.resnet50.preprocess_input,
                       augmentation=None, 
                       onehot= True, 
                       image_size=(256,256,3))
valid_gen = Customized_dataloader([dog_valid, cat_valid], batch_size_per_dataset=FLAGS.batch_size//2, num_workers=2, queue_size=10)
x_val, y_val = [], []
for _ in tqdm(range(10)):
    a,b = next(valid_gen)
    x_val.append(a)
    y_val.append(b)
x_val = np.concatenate(x_val)
y_val = np.concatenate(y_val)
valid_gen.stop_worker()

print(x_val.shape)
print(y_val.shape)
print(y_val.sum(axis=0))<jupyter_output>0%|          | 0/10 [00:00<?, ?it/s]<jupyter_text>Build model<jupyter_code>import sys
import tensorflow as tf
import os
import numpy as np

from tensorflow.keras.utils import get_file

sys.path.append("/mnt/deep-learning/usr/seanyu/lab_mldl_tools/models/")
from tf_resnet.model import set_custom_objects, resnet_graph

TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/'\
                         'releases/download/v0.2/'\
                         'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'

"""
class Build_FunctionalModel():
    def __init__(self, input_shape, classes, backbone='resnet50', use_pretrain=False):
        input_layer = tf.keras.layers.Input(shape=input_shape, name="input")
        self.pretrain_modules, stage_layers = resnet_graph(input_tensor=input_layer, 
                                                           architecture=backbone, 
                                                           train_bn=True, norm_use='bn')
        if use_pretrain:
            weight_path = get_file('resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5', 
                                   TF_WEIGHTS_PATH_NO_TOP, 
                                   cache_subdir="models", 
                                   md5_hash='a268eb855778b3df3c7506639542a6af')
            self.pretrain_modules.load_weights(weight_path)

        self.out = tf.keras.layers.Dense(units=classes, name="output")(self.pretrain_modules.output)
    
    def build(self):
        return tf.keras.Model(inputs=[self.pretrain_modules.input], outputs=[self.out])
"""


# Use tf.keras, official resnet
class Build_FunctionalModel():
    def __init__(self, input_shape, classes, backbone='resnet50', use_pretrain=False):
        
        self.input_layer = tf.keras.layers.Input(shape=input_shape, name="input")
        graph_pool = tf.keras.applications.ResNet50(input_tensor=self.input_layer, include_top=False)
        gap = tf.keras.layers.GlobalAveragePooling2D()(graph_pool.output)
        self.logit = tf.keras.layers.Dense(units=classes, name="logit")(gap)
        self.out = tf.keras.layers.Activation("softmax", name="output")(self.logit)
        
        
        #self.graph_pool = tf.keras.applications.resnet50.ResNet50(input_shape=input_shape, include_top=False)#weights='imagenet')
        #gap = tf.keras.layers.GlobalAveragePooling2D()(self.graph_pool.output)
        #self.out = tf.keras.layers.Dense(units=classes, name="output", activation="softmax")(gap)
    def build(self):
        #return tf.keras.models.Model(inputs=self.graph_pool.input, outputs=self.out)
        return tf.keras.models.Model(inputs=self.input_layer, outputs=self.out)


"""
# Use keras, official resnet
import keras
class Build_FunctionalModel():
    def __init__(self, input_shape, classes, backbone='resnet50', use_pretrain=False):
        self.input_layer = keras.layers.Input(shape=input_shape, name="input")
        graph_pool = keras.applications.ResNet50(input_tensor=self.input_layer, include_top=False)
        gap = keras.layers.GlobalAveragePooling2D()(graph_pool.output)
        self.logit = keras.layers.Dense(units=classes, name="logit")(gap)
        self.out = keras.layers.Activation("softmax", name="output")(self.logit)

    def build(self):
        return keras.models.Model(inputs=self.input_layer, outputs=self.out)
"""
tf.keras.backend.clear_session()

"""
pretrain_model = tf.keras.applications.resnet50.ResNet50(include_top=False, input_shape=x_val.shape[1:], weights='imagenet')
gap = tf.keras.layers.GlobalAveragePooling2D()(pretrain_model.output)
model_output = tf.keras.layers.Dense(units=2, activation='softmax', name='output')(gap)
model = tf.keras.models.Model(inputs = [pretrain_model.input], outputs = [model_output])
"""

model = Build_FunctionalModel(input_shape=[256, 256, 3], classes=2, backbone="resnet50", use_pretrain=True)
model = model.build()

optim = tf.train.AdamOptimizer(learning_rate=FLAGS.lr) #tf.keras.optimizers.Adam(lr=FLAGS.lr)
model.compile(loss='categorical_crossentropy', 
              metrics=["accuracy"], 
              optimizer=optim)

"""
model.compile(loss="categorical_crossentropy", 
              metrics=["accuracy"], 
              optimizer=keras.optimizers.Adam(lr=FLAGS.lr))
"""<jupyter_output>/home/seanyu/.conda/envs/tf18_keras/lib/python3.6/site-packages/keras_applications/resnet50.py:265: UserWarning: The output shape of `ResNet50(include_top=False)` has been changed since Keras 2.2.0.
  warnings.warn('The output shape of `ResNet50(include_top=False)` '<jupyter_text>Note Area  tf.keras lab-resnet tf.keras official-resnet* loss use tf.keras.losses.categorical_crossentropy / tf.keras.losses.binary_crossentropy will FAIL* loss use tf.losses.softmax_cross_entropy* loss use 'categorical_crossentropy' will PASS and SOSO keras1. PASS and GOOD  Start training<jupyter_code>train_gen = Customized_dataloader([dog_train, cat_train], 
                                  batch_size_per_dataset=FLAGS.batch_size//2, 
                                  num_workers=4, queue_size=50)
cb_list = [tf.keras.callbacks.ReduceLROnPlateau(factor=0.5,
                                                patience=4,
                                                min_lr=1e-12),
           tf.keras.callbacks.EarlyStopping(min_delta = 1e-4, 
                                            patience= 50)
          ]

model.fit_generator(train_gen,
                    epochs=FLAGS.epochs,
                    steps_per_epoch=FLAGS.n_batch, 
                    validation_data=(x_val, y_val),
                    #callbacks=cb_list
                    )<jupyter_output>100/100 [==============================] - 48s 477ms/step - loss: 0.1683 - acc: 0.9498 - val_loss: 2.6755 - val_acc: 0.5000
Epoch 3/100
100/100 [==============================] - 47s 465ms/step - loss: 0.1667 - acc: 0.9517 - val_loss: 0.8784 - val_acc: 0.5000
Epoch 4/100
100/100 [==============================] - 46s 463ms/step - loss: 0.1357 - acc: 0.9583 - val_loss: 0.9400 - val_acc: 0.5000
Epoch 5/100
100/100 [==============================] - 47s 466ms/step - loss: 0.1176 - acc: 0.9602 - val_loss: 0.7147 - val_acc: 0.4917
Epoch 6/100
100/100 [==============================] - 46s 464ms/step - loss: 0.1450 - acc: 0.9535 - val_loss: 0.8221 - val_acc: 0.5083
Epoch 7/100
100/100 [==============================] - 46s 465ms/step - loss: 0.0930 - acc: 0.9675 - val_loss: 0.5935 - val_acc: 0.6333
Epoch 8/100
100/100 [==============================] - 46s 465ms/step - loss: 0.1361 - acc: 0.9629 - val_loss: 0.3012 - val_acc: 0.8708
Epoch 9/100
100/100 [==============================] - 46s 4[...]
{'fraction_non_alphanumeric': 0.13496969696969696, 'fraction_numerical': 0.03381818181818182, 'mean_word_length': 2.769933744573909, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '566976', 'license': 'Apache-2.0', 'n_tokens_mistral': 5896, 'n_tokens_neox': 5441, 'n_words': 951, 'path': 'cats_and_dogs_playground/.ipynb_checkpoints/TF_catdog-checkpoint.ipynb', 'repo_name': 'vashineyu/deep-learning-experiments', 'size': '16500'}