<jupyter_start><jupyter_text><jupyter_code>!pip install kaggle
from google.colab import files
files.upload()
!mkdir ~/.kaggle
!cp kaggle.json ~/.kaggle
!chmod 600 ~/.kaggle/kaggle.json
! kaggle datasets download -d alxmamaev/flowers-recognition
! ls -lrt
!unzip -q flowers-recognition.zip
! ls -lrt flowers
# imports

import os
import glob
import random
import shutil

import numpy as np 
import pandas as pd
import matplotlib.pyplot as plt
#setting up folder paths
dataset_path = os.path.join(os.getcwd(), 'flowers')
# creating the test data

! mkdir flower_test
test_dataset = os.path.join(os.getcwd(), 'flowers_test')
test_data_size = 0.1
for folder in os.listdir(dataset_path):
  if not (os.path.exists(os.path.join(test_dataset, folder))):
    os.makedirs(os.path.join(test_dataset, folder))

  files = os.listdir(os.path.join(dataset_path, folder))
  
  k = int(np.floor(len(files) * test_data_size))
  test_files = random.sample(files, k)
  [shutil.move(os.path.join(dataset_path, folder, f), os.path.join(test_dataset, folder, f)) for f in test_files ]
  
#count of images in each class
flower_type_count = pd.DataFrame( [[c, len(os.listdir(os.path.join(dataset_path, c)))] for c in os.listdir(dataset_path)], columns=['name', 'count'])
flower_type_count
def plot_images(images, title):
  fig, axes = plt.subplots(1, len(images))
  fig.set_figwidth(10)
  for ax, img in zip(axes, images):
    ax.imshow(img)
    ax.axis('off')
  plt.suptitle(c, fontsize=20, fontweight='bold')
  plt.show()
# visualizing some flowers from each class
for c in os.listdir(dataset_path):
  cpath = os.path.join(dataset_path, c)
  image_files = [random.choice(os.listdir(cpath))for i in range(5)]
  images = [plt.imread(os.path.join(cpath, image_file)) for image_file in image_files]
  plot_images(images, title=c)
  print()<jupyter_output><empty_output><jupyter_text>Preparing Data fro Binary Classification<jupyter_code>!mkdir flowers_binary
!ln -s /content/flowers/rose /content/flowers_binary/rose
!ln -s /content/flowers/daisy /content/flowers_binary/daisy

!mkdir flowers_binary_test
!ln -s /content/flowers_test/rose /content/flowers_binary_test/rose
!ln -s /content/flowers_test/daisy /content/flowers_binary_test/daisy
binary_dataset_path = os.path.join(os.getcwd(), 'flowers_binary')
binary_test_dataset_path = os.path.join(os.getcwd(), 'flowers_binary_test')<jupyter_output><empty_output><jupyter_text>Custom Data Generator<jupyter_code>import tensorflow as tf
from tensorflow import keras
# class MyDataGenerator(keras.utils.Sequence):

#   def __init__(self, dataset_path, mode='train', validation_perc=None, ablation=None, target_cls=None,
#                batch_size=32, dim=(256, 256), preprocessing_fn=None, n_channel=3, shuffle=True):
    
#     self.batch_size = batch_size
#     self.preprocessing_fn = preprocessing_fn
#     self.n_channel = n_channel
#     self.shuffle = shuffle
#     self.dim = dim
#     self.labels = {}
#     self.image_ID = []

#     if target_cls is None:
#       self.target_cls = os.listdir(dataset_path)
#     else:
#       self.target_cls = target_cls

#     self.n_classes = len(self.target_cls)


#     for i, cls in enumerate(self.target_cls):
#       paths = glob.glob(os.path.join(dataset_path, cls, '*'))
      
#       brk_point = len(paths)
#       if validation_perc is not None:
#         brk_point = int(len(paths) * (1- validation_perc))
      
#       if mode == 'train':
#         paths = paths[:brk_point]
#       else:
#         paths = paths[brk_point:]

#       if ablation is not None:
#         abl_point = int(len(paths) * ablation)
#         paths = paths[:abl_point]

#       self.image_ID += paths
#       self.labels.update({img:i for img in paths})

#     self.on_epoch_end()



#   def on_epoch_end(self):
#     self.indices = np.arange(len(self.image_ID))
#     if self.shuffle:
#       np.random.shuffle(self.indices)
      
#   def __len__(self):
#     return int(np.floor(len(self.image_ID)/self.batch_size ))


#   def __iter__(self):
#     for item in ( self.__getitem__(i) for i in range(len(self))):
#       yield item

#   def __getitem__(self, index):
#     print(index)
#     print(index * self.batch_size , (index +1) * self.batch_size)
#     indexes = self.indices[ index * self.batch_size : (index +1) * self.batch_size ]
#     ids = [ self.image_ID[idx] for idx in indexes]
#     X, y = self._generate_data(ids)
#     print(y.shape)
#     return X, y

#   def _generate_data(self, ids):

#     X = np.empty((self.batch_size, *self.dim, self.n_channel))
#     y = np.empty((self.batch_size), dtype=np.int)

#     for idx , id in enumerate(ids):
#       img = plt.imread(id)
#       img = np.resize(img, (*self.dim, self.n_channel))
#       # print(f' image shape :{img.shape}')
#       if self.preprocessing_fn is not None:
#         img = self.preprocessing_fn(img)
#         # print(f'processed image shape : f{img.shape}')
#       X[idx] = img
#       y[idx] = self.labels[id]

#     y = keras.utils.to_categorical(y, self.n_classes)
      
#     return X, y

# train_datagen = MyDataGenerator(dataset_path, target_cls=['rose', 'daisy'], validation_perc=.2, ablation=0.1, mode='train', dim=(224, 224), preprocessing_fn=keras.applications.resnet50.preprocess_input)
# valid_datagen = MyDataGenerator(dataset_path, target_cls=['rose', 'daisy'], validation_perc=.2, mode='validation', dim=(224, 224) preprocessing_fn=keras.applications.resnet50.preprocess_input)
# for d in train_datagen:
#   print(d[0].shape)<jupyter_output><empty_output><jupyter_text>Model Building for binary classification  Data Generator<jupyter_code>from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications.resnet50 import preprocess_input

prepro_fn = preprocess_input

# for ablation experiment
ablation_datagen = ImageDataGenerator(validation_split=0.9, preprocessing_function=prepro_fn )
ablation_train_data = ablation_datagen.flow_from_directory(binary_dataset_path, subset='training')
ablation_validation_data = ablation_datagen.flow_from_directory(binary_dataset_path, subset='validation')

# for final training
datagen  = ImageDataGenerator(validation_split=0.2,  preprocessing_function=prepro_fn )
train_data = datagen.flow_from_directory( binary_dataset_path, subset='training')
validation_data = datagen.flow_from_directory( binary_dataset_path, subset='validation')


test_datagen = ImageDataGenerator(preprocessing_function=prepro_fn )
test_data = test_datagen.flow_from_directory(binary_test_dataset_path)
# for d in ablation_train_data:
#   print(d[0].shape)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>import tensorflow as tf
from tensorflow import keras
base_model = tf.keras.applications.ResNet50(
    include_top=False,
    weights="imagenet",
)
# building the model

image_dim = (224, 224)

inputs = keras.layers.Input(shape=[256,256, 3])
X = keras.layers.Resizing(*image_dim)(inputs)
X = base_model(inputs, training=False)
X = keras.layers.GlobalAveragePooling2D()(X)
X = keras.layers.Dropout(0.2)(X)
X = keras.layers.Dense(128, activation='relu')(X)
X = keras.layers.Dropout(0.5)(X)
outputs = keras.layers.Dense(2, activation='softmax')(X)

model = keras.Model(inputs, outputs)

model.summary()
earlystopping    = tf.keras.callbacks.EarlyStopping(patience=3, monitor='val_accuracy', mode='max', restore_best_weights=True)

filepath = os.path.join(os.getcwd(), 'flower_model_binary')
if not os.path.exists(filepath):
  os.makedirs(filepath)
model_checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_accuracy', verbose=0, save_best_only=True, mode='max')
opt = tf.keras.optimizers.SGD(learning_rate=0.01)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC()])
history = model.fit(ablation_train_data, validation_data=validation_data, epochs=10, callbacks=[earlystopping, model_checkpoint])
plt.plot(history.history['accuracy'], label='Accuracy')
plt.plot(history.history['val_accuracy'], label='Val_Acc')

plt.plot(history.history['loss'], label='Loss')
plt.plot(history.history['val_loss'], label='Val_loss')

plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Running the model on complete data<jupyter_code>opt = tf.keras.optimizers.SGD(learning_rate=0.01)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC()])

history = model.fit(train_data, validation_data=validation_data, epochs=10, callbacks=[earlystopping, model_checkpoint])
final_binary_model = keras.models.load_model(filepath)
final_binary_model.evaluate(test_data)<jupyter_output>5/5 [==============================] - 4s 657ms/step - loss: 0.0332 - accuracy: 0.9870 - auc_1: 0.9997<jupyter_text>Model Building for Multi class classification   Data Generator<jupyter_code>from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications.resnet50 import preprocess_input

prepro_fn = preprocess_input

# for ablation experiment
ablation_datagen = ImageDataGenerator(validation_split=0.9, preprocessing_function=prepro_fn )
ablation_train_data = ablation_datagen.flow_from_directory(dataset_path, subset='training')
ablation_validation_data = ablation_datagen.flow_from_directory(dataset_path, subset='validation')

# for final training
datagen  = ImageDataGenerator(validation_split=0.2,  preprocessing_function=prepro_fn )
train_data = datagen.flow_from_directory(dataset_path, subset='training')
validation_data = datagen.flow_from_directory(dataset_path, subset='validation')


test_datagen  = ImageDataGenerator(preprocessing_function=prepro_fn)
test_data = test_datagen.flow_from_directory(test_dataset)<jupyter_output>Found 390 images belonging to 5 classes.
Found 3497 images belonging to 5 classes.
Found 3111 images belonging to 5 classes.
Found 776 images belonging to 5 classes.
Found 430 images belonging to 5 classes.<jupyter_text>Model<jupyter_code>import tensorflow as tf
from tensorflow import keras
base_model = tf.keras.applications.ResNet50(
    include_top=False,
    weights="imagenet",
)
# building the model

image_dim = (224, 224)

inputs = keras.layers.Input(shape=[256,256, 3])
X = keras.layers.Resizing(*image_dim)(inputs)
X = base_model(inputs, training=False)
X = keras.layers.GlobalAveragePooling2D()(X)
X = keras.layers.Dropout(0.2)(X)
X = keras.layers.Dense(128, activation='relu')(X)
X = keras.layers.Dropout(0.5)(X)
outputs = keras.layers.Dense(5, activation='softmax')(X)

flower_model = keras.Model(inputs, outputs)

flower_model.summary()
earlystopping    = tf.keras.callbacks.EarlyStopping(patience=3, monitor='val_accuracy', mode='max', restore_best_weights=True)

filepath = os.path.join(os.getcwd(), 'flower_model_multiclass')
if not os.path.exists(filepath):
  os.makedirs(filepath)
model_checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_accuracy', verbose=0, save_best_only=True, mode='max')
opt = tf.keras.optimizers.SGD(learning_rate=0.01)
flower_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC(multi_label=True, num_labels=5)])
history = flower_model.fit(ablation_train_data, validation_data=validation_data, epochs=10, callbacks=[earlystopping, model_checkpoint])
plt.plot(history.history['accuracy'], label='Accuracy')
plt.plot(history.history['val_accuracy'], label='Val_Acc')

plt.plot(history.history['loss'], label='Loss')
plt.plot(history.history['val_loss'], label='Val_loss')

plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Running the model on complete data<jupyter_code>opt = tf.keras.optimizers.SGD(learning_rate=0.01)
flower_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC(multi_label=True, num_labels=5)])

history = flower_model.fit(train_data, validation_data=validation_data, epochs=10, callbacks=[earlystopping, model_checkpoint])
final_model = keras.models.load_model(filepath)
final_model.evaluate(test_data)
prediction = final_model.predict(validation_data)
np.argmax(prediction, axis=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13039531237104893, 'fraction_numerical': 0.014525047453990261, 'mean_word_length': 5.815523059617548, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '583179', 'license': 'Apache-2.0', 'n_tokens_mistral': 4381, 'n_tokens_neox': 4190, 'n_words': 748, 'path': 'FlowerClassification.ipynb', 'repo_name': 'ayushabhinav/FlowerClassification', 'size': '12117'}