<jupyter_start><jupyter_text>CropDisease> Notebook for exercises making a classifier for predicting crop diseases.- toc: true - badges: true- comments: true- categories: [jupyter]<jupyter_code>!pip uninstall torchtext torch torchvision
!pip install -Uqq fastbook
import fastbook
from fastbook import *
from fastai.vision.widgets import *
fastbook.setup_book()

import pandas as pd
from fastai.callback.fp16 import *
from google.colab import drive
drive.mount('/content/gdrive')
#!unzip '/content/gdrive/MyDrive/train.zip' -d '/content/gdrive/MyDrive/CropDisease'
#!unzip '/content/gdrive/MyDrive/test.zip' -d '/content/gdrive/MyDrive/CropDisease'
train_path = Path('/content/gdrive/MyDrive/CropDisease/train')
fns = get_image_files(train_path)
def get_y(r): return L(parent_label(r))
# method to return dls
def get_dls(bs, size):    
  dblock = DataBlock(blocks=(ImageBlock, MultiCategoryBlock),                   
                     get_items=get_image_files,                   
                     get_y=get_y,                   
                     item_tfms=RandomResizedCrop(128),                   
                     batch_tfms=[*aug_transforms(size=size),                               
                                 Normalize()])    
  return dblock.dataloaders(train_path, bs=bs)

#dls.valid.show_batch(max_n=4, nrows=1)
def accuracy_multi(inp, targ, thresh=0.5, sigmoid=True):    
  "Compute accuracy when `inp` and `targ` are the same size."    
  if sigmoid: inp = inp.sigmoid()    
  return ((inp>thresh)==targ.bool()).float().mean()
#create a dls for faster learning
dls = get_dls(128, 128)
# create the learner
learn = cnn_learner(dls, resnet101, metrics=partial(accuracy_multi, thresh=0.2), cbs=MixUp).to_fp16()
# train pretrained models with fit_one_cycle
learn.fit_one_cycle(4, 1e-3)
# new dls for better learning
learn.dls = get_dls(64, 224)
# train the learner
learn.fine_tune(5, base_lr=1e-4)
learn.loss_func
#lr_min,lr_steep = learn.lr_find()
#print(f"Minimum/10: {lr_min:.2e}, steepest point: {lr_steep:.2e}")
#plot the train and validation loss to see if the model is starting to overfit
learn.recorder.plot_loss()
#export the model if its good
learn.export('resnet101')
preds,targs = learn.get_preds()
preds.shape
accuracy_multi(preds, targs, thresh=0.9, sigmoid=False)
xs = torch.linspace(0.05,0.95,29) 
accs = [accuracy_multi(preds, targs, thresh=i, sigmoid=False) for i in xs] 
plt.plot(xs,accs);
# method for prediction
def predict():
  predictions = []
  for f in Path('/content/gdrive/MyDrive/CropDisease/test').iterdir():
    path = Path(f)
    _,_,x = learn.predict(path)
    predictions.append(x)
  return predictions

preditions = predict()
#file_name_list = []
#for f in Path('/content/gdrive/MyDrive/CropDisease/test').iterdir():
    #print(f)
    #path = Path(f)
#    file_name = os.path.basename(f)
    #file_name_without_extension = file_name.
#    file_name_list.append(os.path.splitext(file_name)[0] + ',')
#file_name_list
#make a dataframe
sample_submission = pd.read_csv('/content/gdrive/MyDrive/CropDisease/sample_submission.csv')

data = {"leaf_rust, stem_rust, healthy_wheat": predictions}
index =  sample_submission['ID']

submission = pd.DataFrame(data, index)
#submission.index.name = 'ID,'
submission
#export to a csv fil
submission.to_csv('/content/gdrive/MyDrive/CropDisease_csv_progresive_resizing_resnet101.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12386263574992662, 'fraction_numerical': 0.015849721162312886, 'mean_word_length': 3.903597122302158, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '644165', 'license': 'Apache-2.0', 'n_tokens_mistral': 1259, 'n_tokens_neox': 1198, 'n_words': 250, 'path': '_notebooks/2021-03-23-crop-disease.ipynb', 'repo_name': 'JP97/SW-AI-Blog', 'size': '3407'}