<jupyter_start><jupyter_text>ISIC 2018 Lesion Diagnosis Challengehttps://challenge2018.isic-archive.com  Step 0: Setup<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline
## Using Fast.ai library from Paperspace
from fastai.imports import *
from fastai.torch_imports import *
from fastai.transforms import *
from fastai.conv_learner import *
from fastai.model import *
from fastai.dataset import *
from fastai.sgdr import *
from fastai.plots import *
torch.cuda.set_device(0)
torch.backends.cudnn.enabled<jupyter_output><empty_output><jupyter_text>Get data from https://challenge2018.isic-archive.com/task3/training/<jupyter_code>PATH = "data/lessionChallenge/"
sz = 224
arch = resnext101_64
bs = 24
label_csv = f'{PATH}labels.csv'
n = len(list(open(label_csv))) - 1 # header is not counted (-1)
val_idxs = get_cv_idxs(n) # random 20% data for validation set
n
len(val_idxs)
# If you haven't downloaded weights.tgz yet, download the file.
#     http://forums.fast.ai/t/error-when-trying-to-use-resnext50/7555
#     http://forums.fast.ai/t/lesson-2-in-class-discussion/7452/222
#!wget -O fastai/weights.tgz http://files.fast.ai/models/weights.tgz

#!tar xvfz fastai/weights.tgz -C fastai<jupyter_output><empty_output><jupyter_text>Step 1: Initial exploration<jupyter_code>!ls {PATH}
label_df = pd.read_csv(label_csv)
label_df.head()
label_df.pivot_table(index="disease", aggfunc=len).sort_values('images', ascending=False)
#We have unbalanced classes 
#to-do
tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)
data = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv',
                                   val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)
fn = PATH + data.trn_ds.fnames[0]; fn
img = PIL.Image.open(fn); img
img.size
size_d = {k: PIL.Image.open(PATH + k).size for k in data.trn_ds.fnames}
row_sz, col_sz = list(zip(*size_d.values()))
row_sz = np.array(row_sz); col_sz = np.array(col_sz)
row_sz[:5]
plt.hist(row_sz);
plt.hist(row_sz[row_sz < 1000])
plt.hist(col_sz);
plt.hist(col_sz[col_sz < 1000])
len(data.classes), data.classes[:5]<jupyter_output><empty_output><jupyter_text>Step 2: Initial model<jupyter_code>def get_data(sz, bs): # sz: image size, bs: batch size
    tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)
    data = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv',
                                       val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)
    
    # http://forums.fast.ai/t/how-to-train-on-the-full-dataset-using-imageclassifierdata-from-csv/7761/13
    # http://forums.fast.ai/t/how-to-train-on-the-full-dataset-using-imageclassifierdata-from-csv/7761/37
    return data if sz > 300 else data.resize(340, 'tmp') # Reading the jpgs and resizing is slow for big images, so resizing them all to 340 first saves time

#Source:   
#    def resize(self, targ, new_path):
#        new_ds = []
#        dls = [self.trn_dl,self.val_dl,self.fix_dl,self.aug_dl]
#        if self.test_dl: dls += [self.test_dl, self.test_aug_dl]
#        else: dls += [None,None]
#        t = tqdm_notebook(dls)
#        for dl in t: new_ds.append(self.resized(dl, targ, new_path))
#        t.close()
#        return self.__class__(new_ds[0].path, new_ds, self.bs, self.num_workers, self.classes)
#File:      ~/fastai/courses/dl1/fastai/dataset.py<jupyter_output><empty_output><jupyter_text>Precompute<jupyter_code>data = get_data(sz, bs)
learn = ConvLearner.pretrained(arch, data, precompute=True)
learn.summary()
learn.fit(2e-2, 3)<jupyter_output><empty_output><jupyter_text>Step 3: Evaluation Metric<jupyter_code>from sklearn.metrics import confusion_matrix


def get_balanced_accuracy(probs,y,classes): # sz: image size, bs: batch size
    preds = np.argmax(probs, axis=1)
    probs = probs[:,1]
    cm = confusion_matrix(y, preds)
    plot_confusion_matrix(cm, classes)
    return ((
    cm[0][0]/(cm[0][0]+cm[0][1]+cm[0][2]+cm[0][3]+cm[0][4]+cm[0][5]+cm[0][6]) + 
    cm[1][1]/(cm[1][0]+cm[1][1]+cm[1][2]+cm[1][3]+cm[1][4]+cm[1][5]+cm[1][6]) +
    cm[2][2]/(cm[2][0]+cm[2][1]+cm[2][2]+cm[2][3]+cm[2][4]+cm[2][5]+cm[2][6]) +
    cm[3][3]/(cm[3][0]+cm[3][1]+cm[3][2]+cm[3][3]+cm[3][4]+cm[3][5]+cm[3][6]) +
    cm[4][4]/(cm[4][0]+cm[4][1]+cm[4][2]+cm[4][3]+cm[4][4]+cm[4][5]+cm[4][6]) +
    cm[5][5]/(cm[5][0]+cm[5][1]+cm[5][2]+cm[5][3]+cm[5][4]+cm[5][5]+cm[5][6]) +
    cm[6][6]/(cm[6][0]+cm[6][1]+cm[6][2]+cm[6][3]+cm[6][4]+cm[6][5]+cm[6][6]) 
    )/7)
    
log_preds,y = learn.TTA()
probs = np.mean(np.exp(log_preds),0)
get_balanced_accuracy(probs,y,data.classes)<jupyter_output>[[  24    6   22    0    3    7    0]        
 [  14   53   17    0    2   20    0]
 [  10    9  121    1    8   57    0]
 [   0    0    3    2    2   15    0]
 [  12    7   33    0   62  125    0]
 [   5   11   38    0   14 1268    2]
 [   0    3    2    0    0    5   20]]<jupyter_text>Step 4: Find Learning Rate<jupyter_code>learn = ConvLearner.pretrained(arch, data, precompute=True)
lrf=learn.lr_find()
learn.sched.plot_lr()
learn.sched.plot()<jupyter_output><empty_output><jupyter_text>The loss is still clearly improving at lr=(1e-2), so that's what we use. Note that the optimal learning rate can change as we train the model, so you may want to re-run this function from time to time.  Step 5: Augment<jupyter_code>from sklearn import metrics
learn.fit(1e-2, 3)
learn.precompute = False
learn.fit(1e-2, 3, cycle_len=1)
learn.save('resnet101_224_pre')
learn.load('resnet101_224_pre')<jupyter_output><empty_output><jupyter_text>Step 6: Fine-tuning and differential learning rate annealing<jupyter_code>learn.unfreeze()
lr=np.array([1e-2/9,1e-2/3,1e-2])
learn.fit(lr, 3, cycle_len=1, cycle_mult=2)
learn.save('resnext101_224_all')
learn.load('resnext101_224_all')
log_preds,y = learn.TTA()
probs = np.mean(np.exp(log_preds),0)
get_balanced_accuracy(probs,y,data.classes)
#learn.load('final_challenge1')<jupyter_output><empty_output><jupyter_text>Step 7: Increase Size<jupyter_code>bs=12
learn.set_data(get_data(299, bs)) 
learn.freeze()
learn.fit(1e-3, 3, cycle_len=1)
log_preds,y = learn.TTA()
probs = np.mean(np.exp(log_preds),0)
get_balanced_accuracy(probs,y,data.classes)
learn.unfreeze()
lr=np.array([1e-2/9,1e-2/3,1e-2])
learn.fit(lr, 3, cycle_len=1, cycle_mult=2)
learn.save('resnet101_299_unfrozen')
learn.load('resnet101_299_unfrozen')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1698527107489815, 'fraction_numerical': 0.05625195863365716, 'mean_word_length': 4.663708961845608, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '486242', 'license': 'Apache-2.0', 'n_tokens_mistral': 2818, 'n_tokens_neox': 2648, 'n_words': 441, 'path': 'courses/dl1/rz-final-skin-lesion.ipynb', 'repo_name': 'RobZuazua/skin-lesion-challenge', 'size': '6382'}