<jupyter_start><jupyter_text><jupyter_code># import packages that we need
import pandas as pd
import numpy as np
import keras
import os
import zipfile
import random
import shutil
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from shutil import copyfile
import matplotlib.pyplot as plt

from tensorflow.keras.optimizers import RMSprop
from sklearn.utils.class_weight import compute_class_weight

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import Model
from tensorflow.keras.models import load_model
from tensorflow.keras import layers
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator, img_to_array, load_img

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os

import io
import random
import shutil
from PIL import Image


import time
from tqdm import tqdm
from google.colab import drive
drive.mount('/content/gdrive')
#https://drive.google.com/file/d/1EPbihOXPf4NQlRNBs-BxbuKsMWfeQ9YA/view?usp=sharing
#https://drive.google.com/file/d/1Cn36xAzO-TaJtCiL4vyWMb3jQ8itch2P/view?usp=sharing
#https://drive.google.com/file/d/1TbAPcBXHt_G8YWjhPZl8ESpOk1DyMtZX/view?usp=sharing
#https://drive.google.com/drive/folders/15bgZ-rjGLgAzQJJST6LwcUia3uW-b5Xg?usp=sharing
# !gdown --id 15bgZ-rjGLgAzQJJST6LwcUia3uW

# local_zip = '/content/3 Classes.zip'
# zip_ref   = zipfile.ZipFile(local_zip, 'r')
# zip_ref.extractall('/content')
# zip_ref.close()
DATADIR = r"/content/gdrive/MyDrive/Classes"
CATEGORIES = ['ACINETOBACTER','CLOSTRIDIUM','LACTOBACILLUS']
#CATEGORIES = ['Abutilon Hybridum','Alpinia Galanga (Rasna)', 'Amaranthus Viridis (Arive-Dantu)', 'Anthurium Scherzerianum', 'Aporocactus Flagelliformis', 'Basella Alba (Basale)', 'Calathea Crocata', 'daisy', 'dandelion', 'Ficus Religiosa (Peepal Tree)', 'Hibiscus rosa-sinensis', 'Lithops fulleri', 'Lithops optica _Rubra_', 'Lithops pseudotruncatella', 'Moringa Oleifera (Drumstick)', 'Nerium Oleander (Oleander)', 'Ocimum Tenuiflorum (Tulsi)', 'Pachystachys Lutea', 'Paphiopedilum Venustum', 'Peperomia Argyela', 'Phalaenopsis amabilis', 'Plectranthus Amboinicus (Mexican Mint)', 'Punica Granatum (Pomegranate)', 'rose', 'Schlumbergera Bridgesii', 'Strelitzia Reginae', 'sunflower', 'Trigonella Foenum-graecum (Fenugreek)', 'tulip', 'Asplenium Nidus', 'Calathea Roseopicta', 'Calathea Zebrina', 'Chlorophytum Comosum', 'Davallia Fejeensis', 'Ficus Auriculata (Roxburgh fig)', 'Mentha (Mint)', 'Nephrolepis Exaltata', 'Opuntia Microdasys', 'Pilea Cadierei', 'Piper Betle (Betel)', 'Santalum Album (Sandalwood)', 'Sedum Morganianum', 'Monstera Deliciosa', 'snow white aglaonema']
source_path = r"/content/gdrive/MyDrive/Classes"

source_path_dogs = os.path.join(source_path, 'CLOSTRIDIUM')
source_path_cats = os.path.join(source_path, 'LACTOBACILLUS')


# os.listdir returns a list containing all files under the given path
print(f"There are {len(os.listdir(source_path_dogs))} images of Clostridium.perfringens.")
print(f"There are {len(os.listdir(source_path_cats))} images of Lactobacillus.crispatus.")
# Define root directoryeifera (Drumstick)', 'Nerium Oleander (Oleander)', 'Ocimum Tenuiflorum (Tulsi)', 'Pachystachys Lutea', 'Paphiopedilum Venustum', 'Peperomia Argyela', 'Phalaenopsis amabilis', 'Plectranthus Amboinicus (Mexican Mint)', 'Punica Granatum (Pomegranate)', 'rose', 'Schlumbergera Bridgesii', 'Strelitzia Reginae', 'sunflower', 'Trigonella Foenum-graecum (Fenugreek)', 'tulip', 'Asplenium Nidus', 'Calathea Roseopicta', 'Calathea Zebrina', 'Chlorophytum Comosum', 'Davallia Fejeensis', 'Ficus Auriculata (Roxburgh fig)', 'Mentha (Mint)', 'Nephrolepis Exaltata', 'Opuntia Microdasys', 'Pilea Cadierei', 'Piper Betle (Betel)', 'Santalum Album (Sandalwood)', 'Sedum Morganianum', 'Monstera Deliciosa', 'snow white aglaonema']
root_dir = r'Classes'

# Empty directory to prevent FileExistsError is the function is run several times
if os.path.exists(root_dir):
  shutil.rmtree(root_dir)

# GRADED FUNCTION: create_train_test_dirs
def create_train_test_dirs(root_path):

  # Use os.makedirs to create your directories with intermediate subdirectories
  # Don't hardcode the paths. Use os.path.join to append the new directories to the root_path parameter
  train_dir = os.path.join(root_path, 'training')
  os.makedirs(train_dir)
  validation_dir = os.path.join(root_path, 'validation')
  os.makedirs(validation_dir)
  test_dir = os.path.join(root_path, 'testing')
  os.makedirs(test_dir)
  # Directory with training cat/dog pictures
  for bakteri in CATEGORIES:
        os.makedirs(os.path.join(train_dir, bakteri))

  # Directory with validation cat/dog pictures
  for bakteri in CATEGORIES:
        os.makedirs(os.path.join(validation_dir, bakteri))
  for bakteri in CATEGORIES:
        os.makedirs(os.path.join(test_dir, bakteri))


  
try:
  create_train_test_dirs(root_path=root_dir)
except FileExistsError:
  print("You should not be seeing this since the upper directory is removed beforehand")
# Test your create_train_test_dirs function

for rootdir, dirs, files in os.walk(root_dir):
    for subdir in dirs:
        print(os.path.join(rootdir, subdir))
# GRADED FUNCTION: split_data
def split_data(SOURCE, TRAINING, VALIDATION, TESTING, SPLIT_SIZE1, SPLIT_SIZE2):

  ### START CODE HERE
  #os.path.getsize(PATH)
    list_name = os.listdir(SOURCE)
    random.sample(list_name, len(list_name))
  


    part1 = int(len(list_name) * SPLIT_SIZE1)
    training_file = list_name[:part1]
    PREtesting_file =list_name[part1:]
    part2 = int(len(PREtesting_file) * SPLIT_SIZE2)
    validation_file = PREtesting_file[:part2]
    testing_file =PREtesting_file[part2:]

    for file in training_file:
        copyfile(os.path.join(SOURCE, file), os.path.join(TRAINING, file))
    for file in validation_file:
        copyfile(os.path.join(SOURCE, file), os.path.join(VALIDATION, file))
    for file in testing_file:
        copyfile(os.path.join(SOURCE, file), os.path.join(TESTING, file))
#      copyfile(SOURCE+file, TESTING+file)

  ### END CODE HERE

TRAINING_DIR = r"/content/Classes/training"
VALIDATION_DIR = r"/content/Classes/validation"
TESTING_DIR = r"/content/Classes/testing"

for bakteri in CATEGORIES:
    if len(os.listdir(os.path.join(TRAINING_DIR, bakteri))) > 0:
        for file in os.scandir(os.path.join(TRAINING_DIR, bakteri)):
            os.remove(file.path)
    if len(os.listdir(os.path.join(VALIDATION_DIR, bakteri))) > 0:
        for file in os.scandir(os.path.join(VALIDATION_DIR, bakteri)):
            os.remove(file.path)
    if len(os.listdir(os.path.join(TESTING_DIR, bakteri))) > 0:
        for file in os.scandir(os.path.join(TESTING_DIR, bakteri)):
            os.remove(file.path)

# Define proportion of images used for training
split_size1 = 0.8
split_size2 = 1
# Run the function
# NOTE: Messages about zero length images should be printed out
for bakteri in CATEGORIES:
    split_data(os.path.join(source_path, bakteri), os.path.join(TRAINING_DIR, bakteri), os.path.join(VALIDATION_DIR, bakteri), os.path.join(TESTING_DIR, bakteri), split_size1, split_size2)
    

# GRADED FUNCTION: train_val_generators
def train_val_generators(TRAINING_DIR, VALIDATION_DIR):
  ### START CODE HERE

  # Instantiate the ImageDataGenerator class (don't forget to set the arguments to augment the images)
  train_datagen = ImageDataGenerator(rescale=1.0/255.0,
                                     rotation_range=20,
                                     width_shift_range=.0,
                                     height_shift_range=.0,
                                     shear_range=0.2,
                                     zoom_range=0.2,
                                     horizontal_flip=True,
                                     fill_mode="nearest")

  # Pass in the appropriate arguments to the flow_from_directory method
  train_generator = train_datagen.flow_from_directory(directory=TRAINING_DIR,
                                                      batch_size=45,
                                                      class_mode='categorical',
                                                      target_size=(224, 224), color_mode='rgb')

  # Instantiate the ImageDataGenerator class (don't forget to set the rescale argument)
  validation_datagen = ImageDataGenerator(rescale = 1/255.0)

  # Pass in the appropiate arguments to the flow_from_directory method
  validation_generator = validation_datagen.flow_from_directory(directory=VALIDATION_DIR,
                                                                batch_size=45,
                                                                class_mode='categorical',
                                                                target_size=(224, 224),
                                                                color_mode='rgb')
  ### END CODE HERE
  return train_generator, validation_generator
def color_segment_function(img_array):
    img_array= np.rint(img_array)
    img_array= img_array.astype('uint8')
    hsv_img= cv2.cvtColor(img_array, cv2.COLOR_RGB2HSV)
    mask = cv2.inRange(hsv_img, (24, 50, 0), (55, 255, 255))
    result = cv2.bitwise_and(img_array, img_array, mask=mask)
    result= result.astype('float32')
    return result
test_datagen = ImageDataGenerator(rescale=1./255, preprocessing_function=color_segment_function)
test_generator = test_datagen.flow_from_directory(r'/content/Classes/testing',
                                                  target_size=(224, 224),
                                                  batch_size=45,
                                                  class_mode='categorical',
                                                  shuffle=False)
# Test your generators
train_generator, validation_generator = train_val_generators(TRAINING_DIR, VALIDATION_DIR)
class_names = list(train_generator.class_indices.keys())
class_names
base_model = tf.keras.applications.MobileNetV2(include_top=False)
base_model.trainable= False

inputs = layers.Input(shape=(224,224,3),name = "input_shape",dtype=tf.float16)
x = base_model(inputs,training=False)
x = layers.GlobalAvgPool2D(name="global_average_pooling_layer")(x)
outputs = layers.Dense(len(class_names),activation='softmax',dtype=tf.float32,name='outputs')(x)

model= Model(inputs,outputs)

model.compile(optimizer=keras.optimizers.Adam(lr=0.001, amsgrad=True),
                     loss='categorical_crossentropy',
                     metrics = ['accuracy'])

model.summary()
# epochs = 150
# steps_per_epoch = 50
# Train the model
# Note that this may take some time.

history = model.fit(train_generator,
                    epochs=3,
                    verbose=1,
                    validation_data=validation_generator)
#-----------------------------------------------------------
# Retrieve a list of list results on training and test data
# sets for each training epoch
#-----------------------------------------------------------
acc=history.history['accuracy']
val_acc=history.history['val_accuracy']
loss=history.history['loss']
val_loss=history.history['val_loss']

epochs=range(len(acc)) # Get number of epochs
#------------------------------------------------
#------------------------------------------------
plt.plot(epochs, acc, 'r', "Training Accuracy")
plt.plot(epochs, val_acc, 'b', "Validation Accuracy")
plt.title('Training and validation accuracy')
plt.show()
print("")

#------------------------------------------------
# Plot training and validation loss per epoch
#------------------------------------------------
plt.plot(epochs, loss, 'r', "Training Loss")
plt.plot(epochs, val_loss, 'b', "Validation Loss")
plt.show()
model.save('model_best7Juni.h5')
from keras import models, layers, callbacks
model= models.load_model('model_best7Juni.h5')
test_generator.n
model.evaluate(validation_generator)
def img_prep_predict(path):
    img = plt.imread(path)
    img=tf.image.resize(img,(224,224))
    img=tf.expand_dims(img,axis=0)
    img = img/255.
    predmod = np.ravel(model.predict(img))
    a= 0
    nji = 0
    for CATEGORIE in CATEGORIES:
        if predmod[a] > nji:
            mosnt = CATEGORIE
            nji = predmod[a]
        a+=1

    return mosnt
img_prep_predict(R"/content/gdrive/MyDrive/Classes/ACINETOBACTER/Acinetobacter.baumanii_0001.jpg")
def img_prep_predict(path):
    img = plt.imread(path)
    img = tf.image.resize(img,(224,224))
    img = tf.expand_dims(img,axis=0)
    img = img/255.
    return img
# if os.name == 'posix':
#     !make qt5py3
# if os.name =='nt':
#     !cd {LABELIMG_PATH} && pyrcc5 -o libs/resources.py resources.qrc
#TFLite Saved Model
export_dir = 'saved_model/1'

tf.saved_model.save(model, export_dir = export_dir)
# # Select mode of optimization
# mode = "Speed" 

# if mode == 'Storage':
#     optimization = tf.lite.Optimize.OPTIMIZE_FOR_SIZE
# elif mode == 'Speed':
#     optimization = tf.lite.Optimize.OPTIMIZE_FOR_LATENCY
# else:
#     optimization = tf.lite.Optimize.DEFAULT
converter = tf.lite.TFLiteConverter.from_saved_model(export_dir)

converter.optimizations = [tf.lite.Optimize.DEFAULT]

tflite_model = converter.convert()

# converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
# print(converter)
# converter.inference_input_type = [tf.int8]
# converter.inference_output_type = [tf.int8]
# tflite_quant_model = converter.convert()
# tflite_model = converter.convert()
# tflite_model_file = 'converted_model.tflite'

# with open(tflite_model_file, "wb") as f:
#     f.write(tflite_model)
# # Select mode of optimization
# mode = "Speed" 

# if mode == 'Storage':
#     optimization = tf.lite.Optimize.OPTIMIZE_FOR_SIZE
# elif mode == 'Speed':
#     optimization = tf.lite.Optimize.OPTIMIZE_FOR_LATENCY
# else:
#     optimization = tf.lite.Optimize.DEFAULT
# converter = tf.lite.TFLiteConverter.from_saved_model(export_dir)

# # Set the optimzations
# converter.optimizations = [optimization]

# # Invoke the converter to finally generate the TFLite model
# tflite_model = converter.convert()
# import cv2
# thres = 0.45 # Threshold to detect object
 
# cap = cv2.VideoCapture(1)
# cap.set(3,1280)
# cap.set(4,720)
# cap.set(10,70)
 
# classNames= []
# classFile = ‘coco.names’
# with open(classFile,’rt’) as f:
# classNames = f.read().rstrip(‘n’).split(‘n’)
 
# configPath = ‘ssd_mobilenet_v3_large_coco_2020_01_14.pbtxt’
# weightsPath = ‘frozen_inference_graph.pb’
 
# net = cv2.dnn_DetectionModel(weightsPath,configPath)
# net.setInputSize(320,320)
# net.setInputScale(1.0/ 127.5)
# net.setInputMean((127.5, 127.5, 127.5))
# net.setInputSwapRB(True)
 
# while True:
#     success,img = cap.read()
#     classIds, confs, bbox = net.detect(img,confThreshold=thres)
#     print(classIds,bbox)
 
#     if len(classIds) != 0:
#         for classId, confidence,box in zip(classIds.flatten(),confs.flatten(),bbox):
#             cv2.rectangle(img,box,color=(0,255,0),thickness=2)
#             cv2.putText(img,classNames[classId-1].upper(),(box[0]+10,box[1]+30),
#             cv2.FONT_HERSHEY_COMPLEX,1,(0,255,0),2)
#             cv2.putText(img,str(round(confidence*100,2)),(box[0]+200,box[1]+30),
#             cv2.FONT_HERSHEY_COMPLEX,1,(0,255,0),2)
 
#     cv2.imshow(“Output”,img)
#     cv2.waitKey(1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13246414602346807, 'fraction_numerical': 0.01786179921773142, 'mean_word_length': 3.435096848800231, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '209207', 'license': 'Apache-2.0', 'n_tokens_mistral': 5435, 'n_tokens_neox': 5136, 'n_words': 1038, 'path': 'CapstoneModel31Mei.ipynb', 'repo_name': 'AldaffanSheva/it-cert-automation-practice', 'size': '15340'}