<jupyter_start><jupyter_text>learn.load('materialist_2')<jupyter_code>lr = .05
learn.precompute = False
learn.fit(lr, 3, cycle_len=1)
learn.load('materials_unfrozen_2')
learn.unfreeze()
lrf=learn.lr_find()
learn.sched.plot()
lr=np.array([1e-4,1e-3,1e-2])
learn.fit(lr, 3, cycle_len=1, cycle_mult=2)
learn.load('materials_unfrozen_2')
def get_data(sz):
    tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)
    return ImageClassifierData.from_csv(PATH, f'{PATH}merged/train', f'{PATH}labels.csv', test_name='test', 
                                    val_idxs=val_idxs, 
                                    tfms=tfms, 
                                    bs=bs)
learn.set_data(get_data(299))
learn.freeze()
learn.fit(1e-2, 3, cycle_len=1)
learn.save('materialist_299')
learn.fit(1e-2, 3, cycle_len=1, cycle_mult=2)
learn.load('materialist_299_2')
log_preds,y = learn.TTA(n_aug=4)
preds = np.mean(log_preds, 0)
probs = np.mean(np.exp(log_preds), axis=0)
indices = probs.argmax(axis=1)
accuracy_np(probs, y)
preds = probs.argmax(axis=1)
indexed_preds = [data.classes[pred] for pred in preds]
def trim_jpeg(filename):
    slash_index = filename.index('/') + 1
    dot_index = filename.index('.')
    return filename[slash_index:dot_index]
def get_filename_indices():
    filenames = data.test_ds.fnames
    filenames = [int(trim_jpeg(filename)) for filename in filenames]
    return filenames
def get_filenames():
    filenames = data.test_ds.fnames
    filenames = [trim_jpeg(filename) for filename in filenames]
    return filenames
def get_missing(indices):
    acc = []
    for x in range(1, 12800):
        if x not in indices:
            acc.append(x)
    return acc
missing_indices = get_missing(get_filename_indices())
filenames = get_filenames()
missing_indices
with open(f'{PATH}large_submission.csv', 'w') as f:
    writer = csv.writer(f)
    writer.writerow(('id', 'predicted'))
    writer.writerows(zip(filenames, indexed_preds))
    for m in missing_indices:
        writer.writerow((m, 1))
def get_missing(indices):
    acc = []
    for x in range(1, 12800):
        if x not in indices:
            acc.append(x)
    return acc
def make_submission(preds, incidces):
    missing = get_missing(indices)
    with open(f'{PATH}submission.csv', 'w') as f:
        writer = csv.writer(f)
        writer.writerow(('id', 'predicted'))
        writer.writerows(zip(indices, preds))
        for m in missing:
            writer.writerow((m, 1))
make_submission(indexed_preds, indices)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13443396226415094, 'fraction_numerical': 0.02240566037735849, 'mean_word_length': 3.644160583941606, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '251122', 'license': 'Apache-2.0', 'n_tokens_mistral': 975, 'n_tokens_neox': 932, 'n_words': 174, 'path': 'courses/dl1/Materialist.ipynb', 'repo_name': 'wpride/fastai', 'size': '2544'}