<jupyter_start><jupyter_text>EYE Disease ClassifierWe will be building an image classifier using Fastai's v1.0 Library. The model we will build will be a pretrained resnet34 model.We will:    Scrape google images for data    Preprocess our data        Remove all images that aren't 'jpg'        Trim our data (images) too 100 for each directory        Rename our images        Create New Directory tree        Create Train, Valid sets    Building & Training Our (Pre-trained) Model<jupyter_code># installing fastai version 1.0 library
!pip install fastai
# importing all libraries
import os
import shutil
import pandas as pd
from fastai.vision import *<jupyter_output><empty_output><jupyter_text>1. Scrape Google Images for DataWe will be usinggoogle_images_downloadfor scraping google images<jupyter_code># Downloading google image scraper & chromedriver (allows us to download more than 100 images)
!pip install google_images_download
!sudo apt-get install chromium-chromedriver
!googleimagesdownload --keywords "Cataract,Normal Eye" --limit 150 --format jpg --type photo --size medium --chromedriver /usr/lib/chromium-browser/chromedriver
# Paths for both benign images, cancer images

# Benign folder
PATH_NORMAL = "./downloads/Normal Eye"

# Cancer folder
PATH_CATARACT = "./downloads/Cataract"
# The following line will list the directory
os.listdir(PATH_CATARACT)[:5]
# function to remove images that aren't jpgs
def clean_dir(PATH):
    for file in os.listdir(PATH):
        if file.endswith('.jpg'):
            pass
        else:
            os.remove(f'{PATH}/{file}')

# function to trim the folder down to 100 images
def shape_dir(PATH):
    count = 0
    for file in os.listdir(PATH):
        count += 1
        if count > 100:
            os.remove(f'{PATH}/{file}')
    print('finished')
# calling functions
clean_dir(PATH_NORMAL)
clean_dir(PATH_CATARACT)

shape_dir(PATH_NORMAL)
shape_dir(PATH_CATARACT)
os.getcwd()

new_name = ['normal', 'cataract']

def parse_images(path, new_name):
    for i, filename in enumerate(os.listdir(path)):
        os.rename(path + "/" + filename, path + "/" + new_name + "." + str(i) + ".jpg")

parse_images(PATH_NORMAL, new_name[0])
parse_images(PATH_CATARACT, new_name[1])
# Let's check if the name's have been changed
print(os.listdir(PATH_NORMAL)[:5])
print(os.listdir(PATH_CATARACT)[:5])
series = pd.Series(os.listdir(PATH_CATARACT))
series.nunique()
# run this following script to make the directory structure
%mkdir cataractdata
%mkdir cataractdata/train | mkdir cataractdata/valid
%mkdir cataractdata/train/normal | mkdir cataractdata/train/cataract
%mkdir cataractdata/valid/normal| mkdir cataractdata/valid/cataract
# Grabbing file paths
source = ['./downloads/Normal Eye', './downloads/Cataract']
train_dest = ['./cataractdata/train/normal', './cataractdata/train/cataract']
valid_dest = ['./cataractdata/valid/normal', './cataractdata/valid/cataract']
# Creating our function to split into Train, Valid sets
def split_images(source, train_dest, valid_dest, f_name):
    files = os.listdir(source)
    iter_t = 80
    iter_v =20
    for i in range(iter_t):
        shutil.move(source + '/' + f_name + '.' + str(i) + '.jpg', train_dest)
    for i in range(iter_v):
        shutil.move(source + '/' + f_name + '.' + str(i + 80) + '.jpg', valid_dest)
# Calling the functions

# normal
split_images(source[0], train_dest[0], valid_dest[0], f_name='normal')

# cataract
split_images(source[1], train_dest[1], valid_dest[1], f_name='cataract')
# Transforming our data in pandas series
series_train_normal = pd.Series(os.listdir(train_dest[0]))
series_train_cataract = pd.Series(os.listdir(train_dest[1]))
series_valid_normal = pd.Series(os.listdir(valid_dest[0]))
series_valid_cataract = pd.Series(os.listdir(valid_dest[1]))
# checking for GPU
torch.cuda.is_available()
# Let's see if acceleration is available
torch.backends.cudnn.enabled
# Listing path of data
PATH = './cataractdata'

os.listdir(PATH) # checks the path

data = ImageDataBunch.from_folder(PATH, ds_tfms=get_transforms(),
                                   size=224).normalize(imagenet_stats)
learn = cnn_learner(data, models.resnet101, metrics=accuracy)
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(4)
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
learn.export('cataract.pkl')
lr=1e-5
learn.fit_one_cycle(4, slice(lr,10*lr))
# Showing top losses
interp = ClassificationInterpretation.from_learner(learn)
interp.plot_top_losses(9, figsize=(10,10))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1159035611590356, 'fraction_numerical': 0.013713780137137801, 'mean_word_length': 4.610421836228288, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '121070', 'license': 'MIT', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1477, 'n_words': 387, 'path': 'cataract prediction.ipynb', 'repo_name': 'Budcoderr/Cataract-Detection', 'size': '4521'}