<jupyter_start><jupyter_text>DATAJAM.AI CATS & DOGS - EASY VERSION -   1) Please never forget this MEME   2) Data preparation  a) tidy up the folders<jupyter_code>import os
import shutil
import glob2
import os
from tqdm import tqdm
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from keras.preprocessing.image import ImageDataGenerator
from keras.layers.core import *
from keras.layers import *
from keras.models import Model
from keras.models import Sequential
from keras.optimizers import SGD
from keras.utils import plot_model


import pickle

import logging, os

logging.disable(logging.WARNING)
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

cd /Users/assansanogo/Downloads/DATAJAM_RESSOURCES
CATEGORIES = ["dog","cat"]
TRAIN_PATH="./train"
OUTPUT_PATH_TRAIN="./training_set"
OUTPUT_PATH_VAL="./validation_set"
COLUMNS= ["file_path","category"]
BATCH_SIZE=32
# create  train_path
def prepare_train_dataset(train_path,categories):
    
    for cat in categories:
        if not os.path.exists(os.path.join(train_path,cat)):
            try:
                os.makedirs(os.path.join(train_path,cat))
            except IOError:
                print("error with folder : {0}".format(cat))
        else:
            pass
    print("done\n")
def copy_to_category_folder(list_of_files,category,output_path):
    labels=[]
    file_paths=[]
    for file_path in tqdm(list_of_files):
        new_path = os.path.join(output_path,category,file_path.split("/")[-1])
        shutil.copy2(file_path, new_path)
        
        labels.append(category)
        file_paths.append(new_path)
    return(file_paths,labels)

files = glob2.glob(os.path.join(TRAIN_PATH,"*.jpg"))
cat_files = np.array([el for el in files if "cat" in el])
dog_files = np.array([el for el in files if "dog" in el ])
# let's define the ratio between training data and validation data
ratio = 0.8
# we aim to prepare data in 2 different folders (training and validation)
prepare_train_dataset(OUTPUT_PATH_TRAIN,CATEGORIES)
prepare_train_dataset(OUTPUT_PATH_VAL,CATEGORIES)
#training set
cat_files_train = cat_files[:int(ratio*cat_files.shape[0])]
dog_files_train = dog_files[:int(ratio*dog_files.shape[0])]
#validation set
cat_files_val = cat_files[int(ratio*cat_files.shape[0]):]
dog_files_val = dog_files[int(ratio*dog_files.shape[0]):]
# move files to training folder
cat_paths_train, cat_labels_train = copy_to_category_folder(cat_files_train,"cat",output_path=OUTPUT_PATH_TRAIN)
dog_paths_train, dog_labels_train = copy_to_category_folder(dog_files_train,"dog",output_path=OUTPUT_PATH_TRAIN)
# move files to validation folder
cat_paths_val, cat_labels_val = copy_to_category_folder(cat_files_val,"cat",output_path=OUTPUT_PATH_VAL)
dog_paths_val, dog_labels_val = copy_to_category_folder(dog_files_val,"dog",output_path=OUTPUT_PATH_VAL)
# Organize data as a dataframe
## a bit of magic : organise data as dataframe
cat_dataframe_train = pd.DataFrame([cat_paths_train,cat_labels_train]).transpose()
dog_dataframe_train = pd.DataFrame([dog_paths_train,dog_labels_train]).transpose()
cat_dataframe_val = pd.DataFrame([cat_paths_val,cat_labels_val]).transpose()
dog_dataframe_val = pd.DataFrame([dog_paths_val,dog_labels_val]).transpose()
## merge cats (training)  and dogs (training)
train_df = pd.concat([cat_dataframe_train,dog_dataframe_train], axis=0)
train_df = train_df.sample(frac=1).reset_index(drop=True)
train_df.columns = COLUMNS
train_df.head()
print(train_df.shape)
## merge cats (validation) and dogs (validation)
val_df = pd.concat([cat_dataframe_val,dog_dataframe_val], axis=0)
val_df = val_df.sample(frac=1).reset_index(drop=True)
val_df.columns = COLUMNS
val_df.head()
print(val_df.shape)
# merge training data and validation data
master_df = pd.concat([train_df,val_df], axis=0)
master_df.sample(frac=1).reset_index(drop=True)
master_df.head()
# visualize data from the dataset
def represent_picture(a_df, k=0):
    f_path = a_df["file_path"].iloc[k]
    f_category = a_df["category"].iloc[k]
    an_im = plt.imread(f_path)

    plt.imshow(an_im)
    plt.title(f_category)
    plt.show()
represent_picture(master_df, k=2)<jupyter_output><empty_output><jupyter_text>b) Images generator<jupyter_code># initialize the training data augmentation object (TRAINING)
trainAug = ImageDataGenerator(
	rotation_range=30,
	zoom_range=0.15,
	width_shift_range=0.2,
	height_shift_range=0.2,
	shear_range=0.15,
	horizontal_flip=True,
	fill_mode="nearest")
 # initialize the validation/testing data augmentation object (which
# we'll be adding mean subtraction to) (VALIDATION)
valAug = ImageDataGenerator()
 # define the ImageNet mean subtraction (in RGB order) and set the
# the mean subtraction value for each of the data augmentation
# objects
mean = np.array([123.68, 116.779, 103.939], dtype="float32")
trainAug.mean = mean
valAug.mean = mean<jupyter_output><empty_output><jupyter_text>a) Method A : flow from directory by organizing data in a certainf fashion, the training/validation procedure the images can be read and fedto the network<jupyter_code>#set training sets and validation sets to supply the generators with data 
trainPath = OUTPUT_PATH_TRAIN
valPath = OUTPUT_PATH_VAL
# initialize the training generator
trainGen = trainAug.flow_from_directory(
	trainPath,
	target_size=(224, 224),
	color_mode="rgb",
	shuffle=True,
	batch_size=BATCH_SIZE,
	class_mode='binary')
 
# initialize the validation generator
valGen = valAug.flow_from_directory(
	valPath,
	target_size=(224, 224),
	color_mode="rgb",
	shuffle=False,
	batch_size=BATCH_SIZE,
	class_mode='binary')<jupyter_output>Found 20000 images belonging to 2 classes.
Found 5000 images belonging to 2 classes.<jupyter_text>b) Method B : flow from dataframe by getting the link of the files in a dtaaframe, during the training/validation procedure the images can be read and fed to the network<jupyter_code># initialize the training generator

train_generator=trainAug.flow_from_dataframe(dataframe=train_df,  
                                            x_col="file_path", y_col="category", 
                                            class_mode="binary", 
                                            target_size=(32,32), 
                                            batch_size=32)

val_generator=valAug.flow_from_dataframe(dataframe=val_df,  
                                            x_col="file_path", y_col="category", 
                                            class_mode="binary", 
                                            target_size=(32,32), 
                                            batch_size=32)<jupyter_output>Found 20000 images belonging to 2 classes.
Found 5000 images belonging to 2 classes.<jupyter_text>3) Model construction (model created from scratch)<jupyter_code># import the necessary packages
from keras.preprocessing.image import ImageDataGenerator
from keras.applications import VGG16
from keras.layers.core import *
from keras.layers import *
from keras.models import Model
from keras.models import Sequential
from keras.optimizers import SGD

# yet another CNN architecture
def get_my_model():
    model = Sequential()
    
    model.add(Conv2D(32, (3, 3), input_shape=(224, 224,3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(32, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(64, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    
    model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors
    model.add(Dense(64))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))
    model.compile(loss='binary_crossentropy',
              optimizer='sgd',
              metrics=['accuracy'])
    return model

#save the model in a variable
cats_dogs_model_0 = get_my_model()
# get the structure of the model
cats_dogs_model_0.summary()
#save the model
plot_model(cats_dogs_model_0, to_file='./cats_dogs_model0.png')
comp_im = plt.imread('./cats_dogs_model0.png')
plt.figure(figsize=(30,30))
plt.axis('off')
plt.imshow(comp_im)
plt.show()<jupyter_output><empty_output><jupyter_text>4) The training<jupyter_code># start the training: 
# the model will fit the training generator
# the model will be evaluated at each epoch on the validation set
cats_dogs_model_0.fit_generator(
        trainGen,
        steps_per_epoch=2000 //BATCH_SIZE,
        epochs=50,
        validation_data=valGen,
        validation_steps=2000 //BATCH_SIZE)
cats_dogs_model_0.save_weights('first_try.h5')<jupyter_output>Epoch 1/1
16/62 [======>.......................] - ETA: 37s - loss: 8.2959 - acc: 0.4766
{'fraction_non_alphanumeric': 0.10586617781851512, 'fraction_numerical': 0.021081576535288724, 'mean_word_length': 4.33557457212714, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '197346', 'license': 'MIT', 'n_tokens_mistral': 2986, 'n_tokens_neox': 2844, 'n_words': 639, 'path': 'classification/.ipynb_checkpoints/Cats & Dogs EASY-checkpoint.ipynb', 'repo_name': 'nhatminh46vn/datajam.ai', 'size': '8728'}