<jupyter_start><jupyter_text>You might need to install this on your system:apt-get install python3-opencv git<jupyter_code>import os

if not os.path.isdir('k'):
  !git clone https://github.com/joaopauloschuler/k-neural-api.git k
else:
  !cd k && git pull

!cd k && pip install .
!ls -l
import sys
print("Python version")
print (sys.version)
print("Version info.")
print (sys.version_info)

import skimage
print('skimage version',  skimage.__version__)
import tensorflow as tf
print(tf.__version__)
import sys
import cai
import cai.datasets
import cai.densenet
import cai.util
import cai.models
# Download from https://data.mendeley.com/datasets/tywbtsjrjv/1
url_zip_file="https://data.mendeley.com/datasets/tywbtsjrjv/1/files/d5652a28-c1d8-4b76-97f3-72fb80f94efc/Plant_leaf_diseases_dataset_without_augmentation.zip?dl=1"
local_zip_file="plant_leaf.zip"
expected_folder_name="plant_leaf"
Verbose=True
cai.datasets.download_zip_and_extract(
    url_zip_file=url_zip_file, local_zip_file=local_zip_file, 
    expected_folder_name=expected_folder_name, Verbose=Verbose)
import random
import os
import multiprocessing
import glob
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.utils import class_weight
import keras
from keras.applications.inception_v3 import InceptionV3
from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array
from keras.utils import np_utils
from keras.optimizers import SGD, adam
from keras.layers import *
from keras.models import Model
from keras.callbacks import EarlyStopping
import matplotlib.pyplot as plt
%matplotlib inline
!rm -r plant_leaf/Plant_leave_diseases_dataset_without_augmentation/Background_without_leaves -R
data_dir = "plant_leaf/Plant_leave_diseases_dataset_without_augmentation/"
print(os.listdir(data_dir))
def load_plantvillage(seed=None, root_dir=None):
  if root_dir == None:
    print("define where the plantvillage folder is.")
    return
  random.seed(seed)
  def read_from_paths(paths):
    x=[]
    for path in paths:
      img = load_img(path, target_size=(224,224))
      img = img_to_array(img, dtype='float16')
      x.append(img)
    return x
  
  classes = os.listdir(root_dir)
  classes = sorted(classes)

  train_path = []
  val_path = []
  test_path = []

  train_x,train_y = [],[]
  val_x,val_y = [],[]
  test_x,test_y =[],[]

  #read path and categorize to three groups , 6,2,2
  for i,_class in enumerate(classes):
      paths = glob.glob(os.path.join(root_dir,_class,"*"))
      paths = [n for n in paths if n.endswith(".JPG") or n.endswith(".jpg")]
      random.shuffle(paths)
      cat_total = len(paths)
      train_path.extend(paths[:int(cat_total*0.6)])
      train_y.extend([i]*int(cat_total*0.6))
      val_path.extend(paths[int(cat_total*0.6):int(cat_total*0.8)])
      val_y.extend([i]*len(paths[int(cat_total*0.6):int(cat_total*0.8)]))
      test_path.extend(paths[int(cat_total*0.8):])
      test_y.extend([i]*len(paths[int(cat_total*0.8):]))

  print ("loading training images")
  train_x = read_from_paths(train_path)
  train_x = np.array(train_x, dtype='float16')/255.
  train_y = np.array(train_y)

  print ("loading validation images")
  val_x = read_from_paths(val_path)
  val_x = np.array(val_x, dtype='float16')/255.
  val_y = np.array(val_y)

  print ("loading testing images")
  test_x = read_from_paths(test_path)
  test_x = np.array(test_x, dtype='float16')/255.
  test_y = np.array(test_y)

  #calculate class weight
  classweight = class_weight.compute_class_weight('balanced', np.unique(train_y), train_y)

  #convert to categorical
  train_y = np_utils.to_categorical(train_y, 38)
  val_y = np_utils.to_categorical(val_y, 38)
  test_y = np_utils.to_categorical(test_y, 38)
  print("loaded")

  return train_x,val_x,test_x,train_y,val_y,test_y,classweight,classes
def compiled_two_path_inception_v3(
  classes=1000, 
  input_shape=(224,224,3),
  two_paths_first_block=False,
  two_paths_second_block=False,
  max_mix_idx=10):
  base_model = cai.models.two_path_inception_v3(
    include_top=False,
    weights=None, #'two_paths_plant_leafs'
    input_shape=input_shape,
    pooling=None,
    two_paths_first_block=two_paths_first_block,
    two_paths_second_block=two_paths_second_block,
    max_mix_idx=max_mix_idx)
  x = base_model.output
  x = GlobalAveragePooling2D()(x)
  x = Dense(38, name='preprediction')(x)
  predictions = Activation('softmax',name='prediction')(x)
  model = Model(inputs=base_model.input, outputs=predictions)
  model.compile(loss='categorical_crossentropy',
  optimizer="sgd",
  metrics=['accuracy','top_k_categorical_accuracy'])
  return model
# test model compilation
model = compiled_two_path_inception_v3(classes=38)
model.summary()
train_x, val_x, test_x, train_y, val_y, test_y, classweight, classes = load_plantvillage(seed=7,root_dir=data_dir)
print(train_x.shape,val_x.shape,test_x.shape)
print(train_y.shape,val_y.shape,test_y.shape)
monitor='val_accuracy'
epochs=30
for max_mix_idx in range(-1,10,1):
    basefilename = 'Yosuke-Baseline-V4-'+str(max_mix_idx)
    print('Running: '+basefilename)
    model = compiled_two_path_inception_v3(
      classes=38,
      input_shape=(224,224,3),
      two_paths_first_block=False,
      two_paths_second_block=False,
      max_mix_idx=max_mix_idx)
    best_result_file_name = basefilename+'-best-result.hdf5'
    save_best = keras.callbacks.ModelCheckpoint(
      filepath=best_result_file_name, 
      monitor=monitor, 
      verbose=1, 
      save_best_only=True, 
      save_weights_only=False, 
      mode='max', 
      period=1)
    history = model.fit(train_x, train_y, epochs=epochs, batch_size=32,
      validation_data=(val_x,val_y),
      callbacks=[save_best],class_weight=classweight,
      workers=multiprocessing.cpu_count())
    print('Testing Last Model: '+basefilename)
    evaluated = model.evaluate(test_x,test_y)
    for metric, name in zip(evaluated,["loss","acc","top 5 acc"]):
      print(name,metric)
    print('Best Model Results: '+basefilename)
    model = keras.models.load_model(best_result_file_name, custom_objects={'CopyChannels': cai.layers.CopyChannels})
    evaluated = model.evaluate(test_x,test_y)
    #cai.models.save_model(model, basefilename)
    for metric, name in zip(evaluated,["loss","acc","top 5 acc"]):
      print(name,metric)
    print('Finished: '+basefilename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11801632140615191, 'fraction_numerical': 0.018832391713747645, 'mean_word_length': 5.322420634920635, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '421192', 'license': 'MIT', 'n_tokens_mistral': 2418, 'n_tokens_neox': 2330, 'n_words': 391, 'path': 'raw/baseline/JP24B01-Baseline_various_mixes.ipynb', 'repo_name': 'joaopauloschuler/two-path-noise-lab-plant-disease', 'size': '6372'}