<jupyter_start><jupyter_text>md.path = pathlib.Path('.')<jupyter_code>gc.collect()<jupyter_output><empty_output><jupyter_text>**LOOK AT DOG PICTURES**<jupyter_code>def rand_by_mask(mask): return np.random.choice(np.where(mask)[0], 4, replace=False)
def rand_by_correct(is_correct): return rand_by_mask((preds == data.val_y)==is_correct)
def plot_val_with_title(idxs, title):
    imgs = np.stack([data.val_ds[x][0] for x in idxs])
    title_probs = [probs[x] for x in idxs]
    print(title)
    return plots(data.val_ds.denorm(imgs), rows=1, titles=title_probs)
def plots(ims, figsize=(12,6), rows=1, titles=None):
    f = plt.figure(figsize=figsize)
    for i in range(len(ims)):
        sp = f.add_subplot(rows, len(ims)//rows, i+1)
        sp.axis('Off')
        if titles is not None: sp.set_title(titles[i], fontsize=16)
        plt.imshow(ims[i])
def load_img_id(ds, idx): return np.array(PIL.Image.open(PATH+ds.fnames[idx]))

def plot_val_with_title(idxs, title):
    imgs = [load_img_id(data.val_ds,x) for x in idxs]
    title_probs = [probs[x] for x in idxs]
    print(title)
    return plots(imgs, rows=1, titles=title_probs, figsize=(16,8))
gc.collect()
def most_by_mask(mask, mult):
    idxs = np.where(mask)[0]
    #return idxs[np.argsort(mult * probs[idxs])[:4]]
    return idxs[np.argsort(mult * probs[idxs])[:6]]

def most_by_correct(y, is_correct): 
    mult = -1 if (y==1)==is_correct else 1
    return most_by_mask(((preds == data.val_y)==is_correct) & (data.val_y == y), mult)<jupyter_output><empty_output><jupyter_text>!rm -rf ../working/dogcats/tmp/<jupyter_code>tfms = tfms_from_model(resnet34, sz, aug_tfms=transforms_side_on, max_zoom=1.1)
def get_augs():
    data = ImageClassifierData.from_paths(PATH, bs=2, tfms=tfms, num_workers=1)
    x,_ = next(iter(data.aug_dl))
    return data.trn_ds.denorm(x)[1]
arch=resnet34
data = ImageClassifierData.from_paths(PATH, tfms=tfms, test_name='test')
learn = ConvLearner.pretrained(arch, data, precompute=True)
learn.fit(1e-2, 1)
learn.precompute=False
learn.fit(1e-2, 3, cycle_len=1)
learn.sched.plot_lr()
gc.collect()
learn.unfreeze()
lr=np.array([1e-4,1e-3,1e-2])
learn.fit(lr, 3, cycle_len=1, cycle_mult=2)
learn.sched.plot_lr()
log_preds,y = learn.TTA()
probs = np.mean(np.exp(log_preds),0)
accuracy_np(probs, y)
preds = np.argmax(probs, axis=1)
probs = probs[:,1]
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y, preds)
plot_confusion_matrix(cm, data.classes)<jupyter_output>[[376   3]
 [  1 367]]<jupyter_text>plot_val_with_title(most_by_correct(0, False), "Most incorrect cats") plot_val_with_title(most_by_correct(1, False), "Most incorrect dogs")<jupyter_code>temp = learn.predict(is_test=True)
temp.shape
pred_test = np.argmax(temp, axis=1)
pred_test[:20]
probs = np.exp(temp[:,1])
probs[:10]
os.listdir(f'{PATH}test')[:4]
for image in os.listdir(f'{PATH}test')[:4]:
    img = plt.imread(f'../working/dogcats/test/{image}')
    plt.imshow(img)
    print (image)<jupyter_output>2477.jpg
6392.jpg
8451.jpg
1610.jpg<jupyter_text>img = plt.imread(f'./dogcats/train/dogs/{files[1]}')plt.imshow(img)<jupyter_code>submission = pd.DataFrame({'id':os.listdir(f'{PATH}test'), 'label':probs})
! ls ../working/
submission['id'] = submission['id'].map(lambda x: x.split('.')[0])
submission['id'] = submission['id'].astype(int)
submission = submission.sort_values('id')
submission.to_csv('../working/output.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1753416690898517, 'fraction_numerical': 0.02471648735097412, 'mean_word_length': 6.020408163265306, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '345839', 'license': 'MIT', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1423, 'n_words': 222, 'path': 'Dog vs Cats/Dog vs Cats (less data).ipynb', 'repo_name': 'anshulrai/Kaggle-Challenges', 'size': '3439'}