<jupyter_start><jupyter_text>Download and unzip the dataset<jupyter_code>!wget -O "dog_breed_classification_ai_challenge-dataset.zip" "https://dockship-job-models.s3.ap-south-1.amazonaws.com/5d1d683b041da2669eed8b591fba65ac?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=<KEY>2F20200921%2Fap-south-1%2Fs3%2Faws4_request&X-Amz-Date=20200921T065645Z&X-Amz-Expires=1800&X-Amz-Signature=55f001c317b45e201e10f76a6fa6b764e5f12de75b3f182011d02ea533b34871&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22dog_breed_classification_ai_challenge-dataset.zip%22"
!pip install -U efficientnet
!unzip /content/dog_breed_classification_ai_challenge-dataset.zip<jupyter_output>[1;30;43mStreaming output truncated to the last 5000 lines.[0m
  inflating: dataset/test/472_test.jpg  
  inflating: dataset/test/2551_test.jpg  
  inflating: dataset/test/3798_test.jpg  
  inflating: dataset/test/4062_test.jpg  
  inflating: dataset/test/2953_test.jpg  
  inflating: dataset/test/2952_test.jpg  
  inflating: dataset/test/687_test.jpg  
  inflating: dataset/test/1358_test.jpg  
  inflating: dataset/test/5330_test.jpg  
  inflating: dataset/test/1961_test.jpg  
  inflating: dataset/test/3690_test.jpg  
  inflating: dataset/test/2458_test.jpg  
  inflating: dataset/test/4498_test.jpg  
  inflating: dataset/test/2791_test.jpg  
  inflating: dataset/test/5436_test.jpg  
  inflating: dataset/test/2966_test.jpg  
  inflating: dataset/test/6151_test.jpg  
  inflating: dataset/test/1954_test.jpg  
  inflating: dataset/test/180_test.jpg  
  inflating: dataset/test/181_test.jpg  
  inflating: dataset/test/3451_test.jpg  
  inflating: dataset/test/5758_test.jpg  
  inflating: da[...]<jupyter_text>Import libraries and load labels<jupyter_code>import os
import sys
import numpy as np
from skimage.io import imread
import matplotlib.pyplot as plt
sys.path.append('..')
from keras.applications.imagenet_utils import decode_predictions

from efficientnet.keras import EfficientNetB7
from efficientnet.keras import center_crop_and_resize, preprocess_input
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os, cv2, random, time, shutil, csv
import tensorflow as tf
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from tqdm import tqdm
np.random.seed(42)
%matplotlib inline 

import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Model
from keras.layers import BatchNormalization, Dense, GlobalAveragePooling2D, Lambda, Dropout, InputLayer, Input
from keras.utils import to_categorical
from tensorflow.keras.preprocessing.image import load_img
# train path 
path_t ="/content/dataset/train"
# list all files
import os
lists_fol = os.listdir(path_t)
# Calculate number of files
def get_num_files(path):
    '''
    Counts the number of files in a folder.
    '''
    if not os.path.exists(path):
        return 0
    return sum([len(files) for r, d, files in os.walk(path)])
#Data Paths
train_dir = '/content/dataset/train'
test_dir = '/content/dataset/test'
#Count/Print train and test samples.
data_size = get_num_files(train_dir)
test_size = get_num_files(test_dir)
print('Data samples size: ', data_size)
print('Test samples size: ', test_size)
import pandas as pd
#Read train labels.
labels_dataframe = pd.read_csv('/content/dataset/train.csv')

#Incpect labels_dataframe.
labels_dataframe.head(5)
#Create list of alphabetically sorted labels.
dog_breeds = sorted(list(set(labels_dataframe['Labels'])))
n_classes = len(dog_breeds)
print(n_classes)
dog_breeds[:10]
#Map each label string to an integer label.
class_to_num = dict(zip(dog_breeds, range(n_classes)))
classes_keys = [classes_s for classes_s in class_to_num.values()]
print(classes_keys)       
class_to_num
# function to return key for any value 
def get_key(val): 
    for key, value in my_dict.items(): 
         if val == value: 
                return key 
  
    return "key doesn't exist"
  
# Driver Code 
  
my_dict = class_to_num<jupyter_output><empty_output><jupyter_text>Preaparation of features of train data<jupyter_code>cd /content/drive/My Drive
!mkdir Dog_Classification_Dockship
cd Dog_Classification_Dockship
def images_to_array(data_dir, labels_dataframe, img_size = (330,330,3)):
    '''
    1- Read image samples from certain directory.
    2- Risize it, then stack them into one big numpy array.
    3- Read sample's label form the labels dataframe.
    4- One hot encode labels array.
    5- Shuffle Data and label arrays.
    '''
    images_names = labels_dataframe['Filename']
    images_labels = labels_dataframe['Labels']
    data_size = len(images_names)
    #initailize output arrays.
    X = np.zeros([data_size, img_size[0], img_size[1], img_size[2]], dtype=np.uint8)
    y = np.zeros([data_size,1], dtype=np.uint8)
    #read data and lables.
    for i in tqdm(range(data_size)):
        image_name = images_names[i]
        img_dir = os.path.join(data_dir, image_name)
        img_pixels = load_img(img_dir, target_size=img_size)
        X[i] = img_pixels
        
        image_breed = images_labels[i]
        y[i] = class_to_num[image_breed]
    
    #One hot encoder
    y_n = to_categorical(y)
    #shuffle    
    ind = np.random.permutation(data_size)
    X = X[ind]
    y_n = y_n[ind]
    print('Ouptut Data Size: ', X.shape)
    print('Ouptut Label Size: ', y.shape)
    return X, y_n,y
def get_features(model_name, data_preprocessor, input_size, data):
    '''
    1- Create a feature extractor to extract features from the data.
    2- Returns the extracted features and the feature extractor.
    '''
    #Prepare pipeline.
    input_layer = Input(input_size)
    preprocessor = Lambda(data_preprocessor)(input_layer)
    base_model = model_name(weights='imagenet', include_top=False,
                            input_shape=input_size)(preprocessor)
    avg = GlobalAveragePooling2D()(base_model)
    feature_extractor = Model(inputs = input_layer, outputs = avg)
    #Extract feature.
    feature_maps = feature_extractor.predict(data, batch_size=64, verbose=1)
    print('Feature maps shape: ', feature_maps.shape)
    return feature_maps
#img_size chosen to be 331 to suit the used architectures.
img_size = (331,331,3)
X, y_cat,y_nocat = images_to_array(train_dir, labels_dataframe, img_size)
# Extract features using Xception as extractor.
from keras.applications.xception import preprocess_input
efficeint_preprocessor = preprocess_input
efficient_features = get_features(EfficientNetB7,
                                 efficeint_preprocessor,
                                 img_size, X)
# Extract features using Xception as extractor.
from keras.applications.xception import Xception, preprocess_input
xception_preprocessor = preprocess_input
xception_features = get_features(Xception,
                                 xception_preprocessor,
                                 img_size, X)
from keras.applications import InceptionV3
inception_preprocessor = preprocess_input
inception_features = get_features(InceptionV3,
                                 inception_preprocessor,
                                 img_size, X)
# Extract features using NASNetLarge as extractor.
from keras.applications.nasnet import NASNetLarge, preprocess_input
nasnet_preprocessor = preprocess_input
nasnet_features = get_features(NASNetLarge,
                               nasnet_preprocessor,
                               img_size, X)
# Extract features using InceptionResNetV2 as extractor.
from keras.applications.inception_resnet_v2 import InceptionResNetV2, preprocess_input
inc_resnet_preprocessor = preprocess_input
inc_resnet_features = get_features(InceptionResNetV2,
                                   inc_resnet_preprocessor,
                                   img_size, X)
# Extract features using InceptionResNetV2 as extractor.
from keras.applications import DenseNet121,DenseNet169,DenseNet201
densenet_preprocessor = preprocess_input
densenet_features = get_features(DenseNet201,
                                 densenet_preprocessor,
                                 img_size,X)
final_features = np.concatenate([nasnet_features, inc_resnet_features,
                                 inception_features, xception_features
                                    ], axis=-1)
del X<jupyter_output><empty_output><jupyter_text>Dump train features as a pickle<jupyter_code>X_feature = open('inc_resnet_features_train', 'ab') 

# source, destination 
pickle.dump(inc_resnet_features, X_feature)                      
X_feature.close() 
X_feature = open('nasnet_features_train', 'ab') 
      
# source, destination 
pickle.dump(nasnet_features, X_feature)                      
X_feature.close() 
X_feature = open('inception_features_train', 'ab') 
      
# source, destination 
pickle.dump(inception_features, X_feature)                      
X_feature.close() 
X_feature = open('xception_features_train', 'ab') 
      
# source, destination 
pickle.dump(xception_features, X_feature)                      
X_feature.close() 
X_feature = open('efficient_features_train', 'ab') 
      
# source, destination 
pickle.dump(efficient_features, X_feature)                      
X_feature.close() 
X_feature = open('densenet_features_train', 'ab') 
      
# source, destination 
pickle.dump(densenet_features, X_feature)                      
X_feature.close()<jupyter_output><empty_output><jupyter_text>Get features for test data and dump it<jupyter_code># train path 
path_t ="/content/dataset/test"
import os
lists_fol = os.listdir(path_t)
def images_to_array2(data_dir, img_size = (300,300,3)):
    '''
    Do same as images_to_array but omit some unnecessary steps for test data.
    '''
    images_names = lists_fol
    data_size = len(images_names)
    X = np.zeros([data_size, img_size[0], img_size[1], 3], dtype=np.uint8)
    
    for i in tqdm(range(data_size)):
        image_name = images_names[i]
        img_dir = os.path.join(data_dir, image_name)
        img_pixels = tf.keras.preprocessing.image.load_img(img_dir, target_size=img_size)
        X[i] = img_pixels
        
    print('Ouptut Data Size: ', X.shape)
    return X
img_size =(331,331,3)
test_data = images_to_array2(test_dir, img_size)
from keras.applications.xception import preprocess_input
efficeint_preprocessor = preprocess_input
efficient_features = get_features(EfficientNetB7, efficeint_preprocessor, img_size, test_data)
import pickle
X_feature = open('efficient_features_test', 'ab') 
      
# source, destination 
pickle.dump(efficient_features, X_feature)                      
X_feature.close()
from keras.applications.inception_resnet_v2 import InceptionResNetV2, preprocess_input
inc_resnet_preprocessor = preprocess_input
inc_resnet_features = get_features(InceptionResNetV2, inc_resnet_preprocessor, img_size, test_data)
import pickle
X_feature = open('inc_resnet_features_test', 'ab') 
      
# source, destination 
pickle.dump(inc_resnet_features, X_feature)                      
X_feature.close() 

from keras.applications.nasnet import NASNetLarge, preprocess_input
nasnet_preprocessor = preprocess_input
nasnet_features = get_features(NASNetLarge,nasnet_preprocessor,img_size, test_data)
X_feature = open('nasnet_features_test', 'ab') 
      
# source, destination 
pickle.dump(nasnet_features, X_feature)                      
X_feature.close() 
from keras.applications import DenseNet121,DenseNet169,DenseNet201
densenet_preprocessor = preprocess_input
densenet_features = get_features(DenseNet201,densenet_preprocessor,img_size, test_data)

X_feature = open('densenet_features_test', 'ab') 
      
# source, destination 
pickle.dump(densenet_features, X_feature)                      
X_feature.close() 
from keras.applications.xception import Xception, preprocess_input
xception_preprocessor = preprocess_input
xception_features = get_features(Xception,xception_preprocessor,img_size, test_data)

X_feature = open('xception_features_test', 'ab') 
      
# source, destination 
pickle.dump(xception_features, X_feature)                      
X_feature.close() 
from keras.applications import InceptionV3
inception_preprocessor = preprocess_input
inception_features = get_features(InceptionV3,inception_preprocessor,img_size, test_data)

X_feature = open('inception_features_test', 'ab') 
      
# source, destination 
pickle.dump(inception_features, X_feature)                      
X_feature.close() 
test_features = np.concatenate([efficient_features,inception_res], axis=-1)
print('Final feature maps shape', test_features.shape)<jupyter_output><empty_output><jupyter_text>Load train data from pickle<jupyter_code>import pickle
pickle_in = open("inc_resnet_features_train","rb")
inc_resnet_features = pickle.load(pickle_in)
import pickle
pickle_in = open("nasnet_features_train","rb")
nasnet_features = pickle.load(pickle_in)
import pickle
pickle_in = open("inception_features_train","rb")
inception_features = pickle.load(pickle_in)
import pickle
pickle_in = open("xception_features_train","rb")
xception_features = pickle.load(pickle_in)

import pickle
pickle_in = open("efficient_features_train","rb")
efficient_features = pickle.load(pickle_in)
import pickle
pickle_in = open("densenet_features_train","rb")
densenet_features = pickle.load(pickle_in)
import pickle
pickle_in = open("y_train_hot","rb")
y_train = pickle.load(pickle_in)
y_train.shape<jupyter_output><empty_output><jupyter_text>Loading test features from pickel<jupyter_code>import pickle
pickle_in = open("inc_resnet_features_test","rb")
inc_resnet_features = pickle.load(pickle_in)
import pickle
pickle_in = open("nasnet_features_test","rb")
nasnet_features = pickle.load(pickle_in)
import pickle
pickle_in = open("inception_features_test","rb")
inception_features = pickle.load(pickle_in)
import pickle
pickle_in = open("xception_features_test","rb")
xception_features = pickle.load(pickle_in)
import pickle
pickle_in = open("efficient_features_test","rb")
efficient_features = pickle.load(pickle_in)
import pickle
pickle_in = open("densenet_features_test","rb")
densenet_features = pickle.load(pickle_in)<jupyter_output><empty_output><jupyter_text>DNN training of network<jupyter_code>#from keras.callbacks import EarlyStopping
#Prepare call backs
#EarlyStop_callback = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)



final_features = np.concatenate([0.5*inc_resnet_features,0.3*nasnet_features, 0.2*inception_features, 0.4*efficient_features,0.4*densenet_features], axis=-1)
print('Final feature maps shape', final_features.shape)
print("----------------------------------------------------------------------")
    #Prepare DNN model
dnn = keras.Sequential()
dnn.add(tf.keras.Input(shape=final_features.shape))
dnn.add(Dense(units=1024,activation='relu'))
dnn.add(Dropout(rate=0.72))
dnn.add(Dense(units=720,activation='relu'))
dnn.add(Dropout(rate=0.72))
dnn.add(Dense(120, activation='softmax'))
dnn.compile(optimizer=keras.optimizers.Adam(learning_rate=0.00002304250692421976), loss='categorical_crossentropy',metrics=[get_f1])

    
dnn.fit(final_features, y_train,batch_size=128,epochs=200 )<jupyter_output>Final feature maps shape (14399, 12096)
----------------------------------------------------------------------
Epoch 1/200
WARNING:tensorflow:Model was constructed with shape (None, 14399, 12096) for input Tensor("input_3:0", shape=(None, 14399, 12096), dtype=float32), but it was called on an input with incompatible shape (None, 12096).
WARNING:tensorflow:Model was constructed with shape (None, 14399, 12096) for input Tensor("input_3:0", shape=(None, 14399, 12096), dtype=float32), but it was called on an input with incompatible shape (None, 12096).
113/113 [==============================] - 1s 9ms/step - loss: 4.7185 - get_f1: 0.0000e+00
Epoch 2/200
113/113 [==============================] - 1s 8ms/step - loss: 4.4413 - get_f1: 0.0000e+00
Epoch 3/200
113/113 [==============================] - 1s 8ms/step - loss: 4.0154 - get_f1: 0.0000e+00
Epoch 4/200
113/113 [==============================] - 1s 8ms/step - loss: 3.4873 - get_f1: 0.0025
Epoch 5/200
113/113 [============================[...]<jupyter_text>Predict and save label file<jupyter_code>#Predict test labels given test data features.
final_features = np.concatenate([0.5*inc_resnet_features,0.3*nasnet_features, 0.2*inception_features, 0.3*efficient_features,0.3*densenet_features], axis=-1)
y_pred = dnn.predict(final_features, batch_size=128)
# function to return key for any value 
def get_key(val): 
    for key, value in my_dict.items(): 
         if val == value: 
                return key 
  
    return "key doesn't exist"
  
# Driver Code 
  
my_dict =class_to_num
  
print(get_key(0)) 
preds_lis = []
for i in range(0,y_pred.shape[0]):
    preds_lis.append(np.argmax(y_pred[i]))
real_label = []
for i in range(0,y_pred.shape[0]):
    real_label.append(get_key(preds_lis[i])) 
import pandas as pd


data = {"Filename": lists_fol,
        "Labels": real_label

        }

df = pd.DataFrame (data, columns = ["Filename","Labels"])
df.to_csv("sub20.csv",index=False)<jupyter_output><empty_output><jupyter_text>Hyperparameter tuning pipeline<jupyter_code>search_list =[inc_resnet_features,nasnet_features,inception_features,xception_features,efficient_features,densenet_features]
name_list =["inc_resnet_features","nasnet_features","inception_features","xception_features","efficient_features","densenet_features"]
!pip install keras-tuner
# setting up tuner model
from keras.callbacks import Callback,ModelCheckpoint
from keras.models import Sequential,load_model
from keras.layers import Dense, Dropout
from keras.wrappers.scikit_learn import KerasClassifier
import keras.backend as K
def get_f1(y_true, y_pred): #taken from old keras source code
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    recall = true_positives / (possible_positives + K.epsilon())
    f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())
    return f1_val
from kerastuner import HyperModel
import kerastuner
class ANNModel(HyperModel):
    def __init__(self, input_shape, num_classes):
        self.input_shape = input_shape
        self.num_classes = num_classes

    def build(self, hp):
        model = keras.Sequential()


        model.add(Dense(
            units=hp.Int(
                'layer_1',
                min_value=512,
                max_value=1536,
                step=256,
                default=1024
                 ),
                  activation=hp.Choice(
                      'dense_activation',
                      values=['relu'],
                      default='relu')
                  ))
        

        model.add(
            Dropout(rate=hp.Float(
                'dropout_1',
                min_value=0.1,
                max_value=0.5,
                default=0.2,
                step=0.1,
            ))
        )


        model.add(Dense(
            units=hp.Int(
                'layer_2',
                min_value=256,
                max_value=1024,
                step=256,
                default=512
                 ),
                  activation=hp.Choice(
                      'dense_activation',
                      values=['relu'],
                      default='relu')
                  ))
        
        
        model.add(
            Dropout(rate=hp.Float(
                'dropout_2',
                min_value=0.1,
                max_value=0.5,
                default=0.2,
                step=0.1,
            ))
        )
       
                
        model.add(Dense(self.num_classes, activation='softmax'))


        model.compile(
            optimizer=keras.optimizers.Adam(
                hp.Float(
                    'learning_rate',
                    min_value=1e-4,
                    max_value=1e-2,
                    sampling='LOG',
                    default=1e-3
                )
            ),

            loss='categorical_crossentropy',
            metrics=[get_f1]
        )


        return model

import numpy as np
from kerastuner.tuners import Hyperband
from kerastuner.tuners import RandomSearch

from itertools import combinations 
features = len(search_list)    

numbers = [i for i in range(0, features)]
for i in range(5,features):
  a = combinations(numbers, i)  
  comb=list(a)
  for elem in comb:
    feature = []
    name = []
    for i in elem:
      feature.append(search_list[i])
      name.append(name_list[i])
    final_features = np.concatenate(feature, axis=-1)
    INPUT_SHAPE = final_features.shape
    NUM_CLASSES = 120  # cifar10 number of classes
    hypermodel = ANNModel(input_shape=INPUT_SHAPE, num_classes=NUM_CLASSES)
    HYPERBAND_MAX_EPOCHS = 8
    MAX_TRIALS = 15
    EXECUTION_PER_TRIAL = 2
    SEED = 1
    name =  str(' '.join(name))
    kerastuner.Objective("val_get_f1", direction="max")
    tuner = RandomSearch(
              hypermodel,
              objective=kerastuner.Objective("val_get_f1", direction="max"),
              seed=SEED,
              max_trials=MAX_TRIALS,
              executions_per_trial=EXECUTION_PER_TRIAL,
              directory='random_search',
              project_name=name
          )

    tuner.search_space_summary()
    N_EPOCH_SEARCH = 8
    tuner.search(final_features, y_train, epochs=N_EPOCH_SEARCH, validation_split=0.1)
      # Show a summary of the search
    tuner.results_summary()
    print(tuner.get_best_models(num_models=1)[0])
    print("########################################################################################################################################################################")
!pip install keras==2.3.0<jupyter_output>Collecting keras==2.3.0
[?25l  Downloading https://files.pythonhosted.org/packages/1b/18/2e1ef121e5560ac24c7ac9e363aa5fa7006c40563c989e7211aba95b793a/Keras-2.3.0-py2.py3-none-any.whl (377kB)
[K     |████████████████████████████████| 378kB 9.3MB/s 
[?25hCollecting keras-preprocessing>=1.0.5
[?25l  Downloading https://files.pythonhosted.org/packages/79/4c/7c3275a01e12ef9368a892926ab932b33bb13d55794881e3573482b378a7/Keras_Preprocessing-1.1.2-py2.py3-none-any.whl (42kB)
[K     |████████████████████████████████| 51kB 7.2MB/s 
[?25hCollecting keras-applications>=1.0.6
  Using cached https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl
Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras==2.3.0) (3.13)
Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras==2.3.0) (1.18.5)
Requirement already satisfied: scipy>=0[...]
{'fraction_non_alphanumeric': 0.11860120345338798, 'fraction_numerical': 0.036626842242958056, 'mean_word_length': 3.5751047277079593, 'pattern_counts': {'":': 2, '<': 39, '<?xml version=': 0, '>': 43, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '105933', 'license': 'MIT', 'n_tokens_mistral': 8153, 'n_tokens_neox': 7504, 'n_words': 1420, 'path': 'Dog_Breed_classification.ipynb', 'repo_name': 'Boltuzamaki/Dog-Breed-Classification-AI-Challenge-Dockship', 'size': '22950'}