<jupyter_start><jupyter_text>Dataset DescriptionWe have used PlantVillage dataset, which contains around 54k leaf images. There are different types of plants and diseases.They have also provided prerpocessed images like grayscale and segmented images. We will perform binary classification, whether given leaf has a disease or not. We will use original image data  Import Libraries<jupyter_code>!pip install basic_image_eda
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_theme(style="whitegrid")
import cv2
import traceback
import sys

from basic_image_eda import BasicImageEDA
from sklearn.model_selection import train_test_split
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.callbacks import EarlyStopping
from keras.callbacks import ReduceLROnPlateau
from keras.applications.vgg16 import VGG16
from keras.applications.resnet50 import ResNet50
from keras.applications.inception_v3 import InceptionV3
from keras.models import  save_model<jupyter_output>Collecting basic_image_eda
  Downloading basic_image_eda-0.0.3-py3-none-any.whl (9.8 kB)
Installing collected packages: basic-image-eda
Successfully installed basic-image-eda-0.0.3<jupyter_text>Exploratory Data Analysis<jupyter_code>data_dir = r"/kaggle/input/plantvillage-dataset/color/"<jupyter_output><empty_output><jupyter_text>View Sample Images<jupyter_code>img = cv2.imread(data_dir+"Apple___healthy/0055dd26-23a7-4415-ac61-e0b44ebfaf80___RS_HL 5672.JPG")
plt.imshow(img)
img = cv2.imread(data_dir+"Corn_(maize)___Cercospora_leaf_spot Gray_leaf_spot/0a403456-5c5e-4aad-aa89-a118175c6ddd___RS_GLSp 4501.JPG")
plt.imshow(img)
BasicImageEDA.explore(data_dir)<jupyter_output>found 1502 images.<jupyter_text>Counting Category Wise Images<jupyter_code>category_count = []

for root, dirs, files in os.walk(data_dir):
    for dir_path in dirs:
        category_count.append((dir_path, len(os.listdir(root+os.sep+dir_path))))
count_df = pd.DataFrame(category_count, columns=['Category', 'Count'])
count_df.head(10)
total_images = count_df['Count'].sum()
total_images
len(count_df)<jupyter_output><empty_output><jupyter_text>Visualize Category Wise Count<jupyter_code>count_df = count_df.sort_values(by='Count', ascending=False)
plt.figure(figsize=(20,20))
sns.barplot(x="Count", y="Category", data=count_df)
plt.plot()
healthy_images_count = count_df[count_df['Category'].str.endswith("healthy")]['Count'].sum()
healthy_images_count
disease_images_count = total_images - healthy_images_count
disease_images_count<jupyter_output><empty_output><jupyter_text>Compare Healthy and Unhealthy image counts<jupyter_code>temp_df = pd.DataFrame(data=[("Unhealthy", disease_images_count), ("Healthy", healthy_images_count)], columns=['Category', 'Count'])
sns.barplot(y="Count",x="Category", data=temp_df)
plt.plot()<jupyter_output><empty_output><jupyter_text>ObservationWe have imabalanced data for healthy abd unhealthy leaves, so we will train on around 10k images each.<jupyter_code>healthy_cat = len(count_df[count_df['Category'].str.endswith('healthy')])
print("healthy_cat",healthy_cat)
print("unhealthy_cat",38 - healthy_cat)<jupyter_output>healthy_cat 12
unhealthy_cat 26<jupyter_text>Build File Path DataFrame<jupyter_code>path_data = []

for root, dirs, files in os.walk(r"/kaggle/input/plantvillage-dataset/color"):
    for d in dirs:
        is_healthy = 1 if d.endswith("healthy") else 0
        leaf_type = d[:d.index("__")]
        
        parent = root+os.sep+d+os.sep
        
        for f in os.listdir(parent):
            path_data.append((leaf_type, is_healthy, d, parent+f))
            
path_df = pd.DataFrame(data=path_data, columns=['Leaf Type', 'Is Healthy', 'Category', 'Image Path'])
path_df.head(10)<jupyter_output><empty_output><jupyter_text>Get Healthy Images Category Wise<jupyter_code>max_samples_per_cat = 10000
replace = False
h_count = max_samples_per_cat // healthy_cat  
uh_count = max_samples_per_cat // (38 - healthy_cat)

new_data = []

for name, group in path_df.groupby(['Is Healthy', 'Category'], as_index=False):
    is_healthy, cat = name
    
    if is_healthy == 1:
        records = h_count 
    else:
        records = uh_count 
    
    for idx, row in group[:records].iterrows():
        new_data.append((is_healthy, cat, row['Image Path']))
        
new_df = pd.DataFrame(data=new_data, columns=['Is Healthy','Category', "Image Path"])
new_df.head()
len(new_df)
len(new_df[new_df['Is Healthy'] == 1])
len(new_df[new_df['Is Healthy'] == 0])
len(new_df['Category'].unique())
len(new_df['Image Path'].unique())<jupyter_output><empty_output><jupyter_text>We can see that we have samples from each category.<jupyter_code>def get_reshaped_image(img, new_shape=(128, 128)):
    resized_image = cv2.resize(img, new_shape, interpolation = cv2.INTER_NEAREST) 
    return resized_image
img = cv2.imread(new_df['Image Path'].iloc[0])
plt.imshow(img)
reshaped_img = get_reshaped_image(img)
plt.imshow(reshaped_img)<jupyter_output><empty_output><jupyter_text>Prepare Data<jupyter_code>def transform_data(df):
    features = []
    labels = []
   
    for i, row in df.iterrows():
        img_path = row['Image Path']
        cat = row['Is Healthy']

        try:
            img = cv2.imread(img_path)

            if img is None:
                continue
             
            resized_image = get_reshaped_image(img)

            features.append(resized_image)
            label = [0.0, 0.0]
            label[cat] = 1.0
            labels.append(label)
            
            if i != 0 and i % 1000 == 0:
                print("Processed Images: ",i)
        except:
            print(f"Error in image: img_path={img_path}, cat={cat}")
            traceback.print_exc()
        
    return features, labels

    
features, labels = transform_data(new_df)
    
print(len(features))
print(len(labels))

features_2 = np.asarray(features)
print(features_2.shape)
labels_2 = np.asarray(labels)
print(labels_2.shape)<jupyter_output>Processed Images:  1000
Processed Images:  2000
Processed Images:  3000
Processed Images:  4000
Processed Images:  5000
Processed Images:  6000
Processed Images:  7000
Processed Images:  8000
Processed Images:  9000
Processed Images:  10000
Processed Images:  11000
Processed Images:  12000
Processed Images:  13000
Processed Images:  14000
Processed Images:  15000
Processed Images:  16000
Processed Images:  17000
17457
17457
(17457, 128, 128, 3)
(17457, 2)<jupyter_text>Train Model<jupyter_code>train_features, test_features, train_labels, test_labels = train_test_split(features_2, labels_2, test_size=0.2, random_state=1)
train_features.shape
test_features.shape
train_labels.shape
test_labels.shape
labels.count([0.0,1.0])
labels.count([1.0,0.0])
input_shape=(128,128,3)
classes = 2
epoch = 10
batch_size = 64

vgg16 = VGG16(weights=None, input_shape=input_shape, classes=classes)

# summarize the model
# vgg16.summary()

vgg16.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])

early_stop= EarlyStopping(monitor='val_loss',patience=10)

mcp_save = ModelCheckpoint('mdl_vgg16_1.hdf5', save_best_only=True, monitor='val_loss', mode='min')

learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', 
                                        patience=10, 
                                        verbose=1, 
                                        factor=0.5, 
                                        min_lr=0.00001)

vgg16.fit(train_features, train_labels, 
          epochs=epoch, 
          batch_size=batch_size, 
          validation_data=(test_features,test_labels), 
          callbacks=[early_stop, learning_rate_reduction])
# callbacks=[early_stop, learning_rate_reduction]
# Save the model
filepath = './vgg16_1'
save_model(vgg16, filepath)

metrics=pd.DataFrame(vgg16.history.history)
metrics
input_shape=(128,128,3)
classes = 2
epoch = 10
batch_size = 64

resnt50 = ResNet50(weights=None, input_shape=input_shape, classes=classes)

# summarize the model
# vgg16.summary()

resnt50.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])

early_stop= EarlyStopping(monitor='val_loss',patience=10)

mcp_save = ModelCheckpoint('mdl_resnt50_1.hdf5', save_best_only=True, monitor='val_loss', mode='min')

learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', 
                                            patience=10, 
                                            verbose=1, 
                                            factor=0.5, 
                                            min_lr=0.00001)

resnt50.fit(train_features, train_labels,
          epochs=epoch,
          batch_size=batch_size,
          validation_data=(test_features,test_labels), 
          callbacks=[early_stop, learning_rate_reduction])
# callbacks=[early_stop, learning_rate_reduction]
# Save the model
filepath = './resnt50'
save_model(resnt50, filepath)

metrics=pd.DataFrame(resnt50.history.history)
metrics
input_shape=(128,128,3)
classes = 2
epoch = 10
batch_size = 64

inception_v3 = InceptionV3(weights=None, input_shape=input_shape, classes=classes)

# summarize the model
# vgg16.summary()

inception_v3.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])

early_stop= EarlyStopping(monitor='val_loss',patience=10)

mcp_save = ModelCheckpoint('mdl_inception_v3_1.hdf5', save_best_only=True, monitor='val_loss', mode='min')

learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', 
                                            patience=10, 
                                            verbose=1, 
                                            factor=0.5, 
                                            min_lr=0.00001)

inception_v3.fit(train_features, train_labels,
          epochs=epoch,
          batch_size=batch_size,
          validation_data=(test_features,test_labels), 
          callbacks=[early_stop, learning_rate_reduction])
# callbacks=[early_stop, learning_rate_reduction]
# Save the model
filepath = './inception_v3'
save_model(inception_v3, filepath)

metrics=pd.DataFrame(inception_v3.history.history)
metrics
results = vgg16.evaluate(test_features,test_labels)
print(f"test acc = {results[1]}\ntest loss = {results[0]}")
results = resnt50.evaluate(test_features,test_labels)
print(f"test acc = {results[1]}\ntest loss = {results[0]}")
results = inception_v3.evaluate(test_features,test_labels)
print(f"test acc = {results[1]}\ntest loss = {results[0]}")<jupyter_output>110/110 [==============================] - 3s 25ms/step - loss: 0.0673 - accuracy: 0.9808
test acc = 0.9808132648468018
test loss = 0.06734218448400497
{'fraction_non_alphanumeric': 0.11170562282924998, 'fraction_numerical': 0.041302919365436966, 'mean_word_length': 4.154329946782777, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '603025', 'license': 'MIT', 'n_tokens_mistral': 3857, 'n_tokens_neox': 3494, 'n_words': 657, 'path': 'eda_and_model_training.ipynb', 'repo_name': 'UtsavVanodiya7/Plant-Disease-Detection', 'size': '10653'}