<jupyter_start><jupyter_text>plant disease detector<jupyter_code>!pip install -U -q PyDrive
!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
!unzip ngrok-stable-linux-amd64.zip

#from pydrive.auth import GoogleAuth
#from pydrive.drive import GoogleDrive
#from google.colab import auth
#from oauth2client.client import GoogleCredentials
#import zipfile
import tensorflow as tf
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg16 import preprocess_input, decode_predictions
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.preprocessing import image
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
import matplotlib.pyplot  as plt  
from tensorflow.keras.layers import Conv2D,Dense,Flatten,Input
from tensorflow.keras.callbacks import *
from tensorflow.keras import metrics

from PIL import Image
#import os
import numpy as np
import json
import sys
import getopt
from time import time

from sklearn.metrics import accuracy_score,classification_report,confusion_matrix
import pandas as pd

#from google.colab import drive
#from google.colab import files
from PIL import Image
import utils

%matplotlib inline
PATH='/content/gdrive'
LOG_DIR = 'log/'
with open('idx_to_class.json', 'r') as fp:
    data = json.load(fp)
    idx_to_class= {int(k):class_name  for k,class_name in data.items() }
get_ipython().system_raw(
    'tensorboard --logdir={} --host 0.0.0.0 --port 6006 &'
    .format(LOG_DIR)
)

get_ipython().system_raw('./ngrok http 6006 &')
! curl -s http://localhost:4040/api/tunnels | python3 -c \
    "import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])"
            
def build_model(num_classes,input_shape=(224,224,3)):
  """
  Args:
            num_classes (int): number of classes
            intut_shape (tuple): shape of image as an input , default value if not specified : (244,224,3)
  """
  
  input_tensor = Input(shape=input_shape)
  base_model = VGG16(include_top=False,weights="imagenet",input_tensor=input_tensor)
  x = base_model.output
  x = Flatten()(x)
  # let's add 2 fully-connected layers with 1024 units
  x = Dense(1024, activation='relu')(x)
  x = Dense(1024, activation='relu')(x)
  # and a logistic layer -- let's say we have 200 classes
  predictions = Dense(num_classes, activation='softmax')(x)
  # this is the model we will train
  model = Model(inputs=base_model.input, outputs=predictions)

  # i.e. freeze all convolutional InceptionV3 layers
  for layer in base_model.layers:
      layer.trainable = False
  return model    

def load_model(best_checkpoint_path="/content/gdrive/My Drive/checkpoints/epochs:009-val_acc:0.953.hdf5"):    
    model = build_model(len(idx_to_class)) 
    model.compile(optimizer='Adam',loss='categorical_crossentropy', metrics=['accuracy'])
    model.load_weights(best_checkpoint_path)
utils.my_auth()<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code

Enter your authorization code:
··········
Mounted at /content/gdrive<jupyter_text>create dataset generators<jupyter_code>#upload the dataset 
utils.upload_data()

init_setting= dict(rescale=1./255,
             #  preprocessing_function=preprocess_input,
               zca_whitening=True,
               horizontal_flip=True,
               vertical_flip=True,
               validation_split=0.25)
setting=dict(target_size=(224,224),
             color_mode='rgb',
             batch_size=32,
             class_mode='categorical',
             shuffle=True)


train_datagen=ImageDataGenerator(**init_setting)
test_datagen=ImageDataGenerator(rescale=1./255)
delete_ugly()
train_generator=train_datagen.flow_from_directory('./dataset_itr2/train', 
                                                   subset='training',**setting)
valid_generator=train_datagen.flow_from_directory('./dataset_itr2/train', 
                                                  subset='validation',**setting)

#since it's a test set , we need to preserve the images order and omit shuffeling 
setting["shuffle"]=False

test_generator=test_datagen.flow_from_directory(directory='./dataset_itr2/test',
                                                  **setting)


(images,labels)=train_generator.next()
plt.imshow(images[0,:])
plt.title(idx_to_class[np.argmax(labels[0,:]) ])
plt.show()<jupyter_output>/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/image_data_generator.py:716: UserWarning: This ImageDataGenerator specifies `featurewise_center`, but it hasn't been fit on any training data. Fit it first by calling `.fit(numpy_data)`.
  warnings.warn('This ImageDataGenerator specifies '
/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/image_data_generator.py:735: UserWarning: This ImageDataGenerator specifies `zca_whitening`, but it hasn't been fit on any training data. Fit it first by calling `.fit(numpy_data)`.
  warnings.warn('This ImageDataGenerator specifies '<jupyter_text>train the model<jupyter_code>model=build_model(len(idx_to_class))  
model.summary()

#---------------------compile model---------------------------------------------------
model.compile(optimizer="Adam",loss='categorical_crossentropy', metrics=["accuracy"])
steps_per_epochs=train_generator.n//train_generator.batch_size
valid_steps=valid_generator.n//valid_generator.batch_size


#---------------------prepare callbacks----------------------------------------

filepath="/content/gdrive/My Drive/checkpoints/epochs:{epoch:03d}-val_acc:{val_acc:.3f}.hdf5"    

tbCallBack = TensorBoard(log_dir='log/'.format(time()), histogram_freq=0,
                         write_graph=True,
                         write_grads=True,
                         batch_size=train_generator.batch_size,
                         write_images=True)
                        
es_callback=EarlyStopping(patience=5,verbose=1,min_delta=1e-3)
rop_callback=ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=2, min_lr=0.01,verbose=1)
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')

#---------------------train model -----------------------------------------------
history=model.fit_generator(generator=train_generator,
                    steps_per_epoch=steps_per_epochs,
                    validation_data=valid_generator,
                    validation_steps=valid_steps,
                    epochs=20,
                    verbose=1,
                    callbacks=[tbCallBack,checkpoint,es_callback,rop_callback])<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5
58892288/58889256 [==============================] - 1s 0us/step
http://613869b9.ngrok.io<jupyter_text>Evaluation  1.model accuracy & loss plots<jupyter_code># summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()<jupyter_output><empty_output><jupyter_text>2.classification report<jupyter_code>#let's first load the best checkpoint
load_model("/content/gdrive/My Drive/checkpoints/epochs:009-val_acc:0.953.hdf5")
steps=test_generator.n//test_generator.batch_size+1
y_test=test_generator.classes
y_pred=model.predict_generator(test_generator,steps=steps,verbose=1)
y_pred=np.argmax(y_pred,axis=1)
target_names=list(idx_to_class.values())

print("test accuracy : {} %".format(round(accuracy_score(y_test,y_pred),4 )*100) )

confusion_mat = pd.DataFrame(confusion_matrix(y_test, y_pred),columns = target_names , index=target_names) 
confusion_mat
print(classification_report(y_test, y_pred, target_names=target_names))<jupyter_output>precision    recall  f1-score   support

                                Apple___Apple_scab       0.91      0.90      0.91       504
                                 Apple___Black_rot       0.99      0.96      0.97       496
                          Apple___Cedar_apple_rust       1.00      0.91      0.95       220
                                   Apple___healthy       0.97      0.95      0.96      1316
                               Blueberry___healthy       0.98      1.00      0.99      1200
          Cherry_(including_sour)___Powdery_mildew       0.97      0.99      0.98       840
                 Cherry_(including_sour)___healthy       1.00      0.97      0.98       680
Corn_(maize)___Cercospora_leaf_spot Gray_leaf_spot       0.91      0.79      0.85       408
                       Corn_(maize)___Common_rust_       1.00      1.00      1.00       952
               Corn_(maize)___Northern_Leaf_Blight       0.88      0.96      0.[...]<jupyter_text>3.plot predictions<jupyter_code>def prediction(model,img_path,topk):
  img = image.load_img(img_path, target_size=(224, 224))
  input_image = image.img_to_array(img)
  input_image = np.expand_dims(input_image, axis=0)
  input_image/=255
  preds = model.predict(input_image)
 
  idx=preds.argsort().squeeze(0).tolist()[-topk:][::-1]
  top_pred=preds.squeeze(0)[idx]
  classes=[idx_to_class[x] for x in idx]
  plot_prob(input_image,top_pred,classes,topk)
  print('Predicted: {} \nclass {} '.format(top_pred,classes))
  
def plot_prob(img,ps,classes,topk):
  fig,(ax1,ax2)=plt.subplots(figsize=(11,20),ncols=2)
  ax1.axis('off')
  img=(img*255).astype(np.uint8)
  ax1.imshow(img.squeeze())
  ax1.set_title('leaf')
  ax2.barh(np.arange(topk),ps,align='center',alpha=.5)
  ax2.set(yticks=range(topk),yticklabels=classes )
  ax2.set_aspect(0.1)
  ax2.set_xlim(0, 1.1)
  plt.tight_layout()
  
prediction(model,'./test4.png',5)<jupyter_output>Predicted: [9.9935287e-01 6.2550453e-04 1.5370711e-05 6.2295358e-06 2.0956929e-08] 
class ['Corn_(maize)___Northern_Leaf_Blight', 'Corn_(maize)___Cercospora_leaf_spot Gray_leaf_spot', 'Corn_(maize)___healthy', 'Corn_(maize)___Common_rust_', 'Strawberry___Leaf_scorch']
{'fraction_non_alphanumeric': 0.1332302655511095, 'fraction_numerical': 0.04074208803201164, 'mean_word_length': 3.634218289085546, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 3, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '435618', 'license': 'MIT', 'n_tokens_mistral': 3912, 'n_tokens_neox': 3619, 'n_words': 591, 'path': 'plants_disease_Detection_.ipynb', 'repo_name': 'othmane42/plant-disease-detection', 'size': '10996'}