<jupyter_start><jupyter_text>Settings<jupyter_code>useTiff = False
path = "../data/"
train_dir = "train-tif-v2" if useTiff == True else "train-jpg"
test_dir = "test-tif-v2" if useTiff == True else 'test-merged'
csv = f'{path}train_v2.csv'
ext = ".tif" if useTiff == True else '.jpg'
load_model_from_disk = True
no_validation = False
val_idxs = [0] if no_validation else hp.val_idxs_from_csv(csv, load_from_disk=load_model_from_disk)
architecture = resnet34 #resnext101 # = resnext101_32
max_zoom = 1.05
dropout = None<jupyter_output><empty_output><jupyter_text>High Resolution<jupyter_code>img_sz_high= 256
batch_sz_high = 64
tfms_high = tfms_from_model(architecture, img_sz_high, transforms_top_down, max_zoom)
data_high = ImageClassifierData.from_csv(path, train_dir, csv, batch_sz_high, tfms_high, val_idxs, ext, test_dir)
learn = ConvLearner.pretrained(architecture, data_high, ps=dropout, precompute=False, metrics=[hp.f2_th])
learn.lr_find()
lr_high_last=0.2
learn.sched.plot()
plt.axvline(x=lr_high_last, color="red")
learn.fit(lr_high_last, 3, cycle_len=1, cycle_mult=2)
high_lastlayer = f"high_lastlayer_{img_sz_high}_{batch_sz_high}{'_all' if no_validation else ''}"; high_lastlayer
learn.unfreeze()
lr_high_all = np.array([lr_high_last/9,lr_high_last/3,lr_high_last])
# model has been trained with satellite data already. can decrease learning_rate for low layers
learn.fit(lr_high_all, 3, cycle_len=1, cycle_mult=2)
learn.fit(lr_high_all, 3, cycle_len=1, cycle_mult=2)
learn.fit(lr_high_all, 3, cycle_len=1, cycle_mult=2)
learn.fit(lr_high_all, 3, cycle_len=1, cycle_mult=2)<jupyter_output><empty_output><jupyter_text>Find proper threshold<jupyter_code>val_probs, val_true = learn.TTA()
val_prob = np.average(val_probs, axis=0)
th = hp.get_threshold(val_prob, val_true, stop=1);th 
# Hier stirbt der Kernel!
hp.f2(val_true, val_prob>th)<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>probs_test, true_test = learn.TTA(is_test=True)
probs_test = np.average(probs_test, axis=0)
th= 0.5
submission = hp.create_submission(probs_test, data_high, th)
submission_file = f"submission_{img_sz_high}_{batch_sz_high}{'_all' if no_validation else ''}"; submission_file
submission.to_csv(submission_file, index=False)
!kaggle competitions submit -c planet-understanding-the-amazon-from-space -f submission_256_64 -m "img_sz_low={img_sz_low}, bs_low={batch_sz_low}, img_sz_high={img_sz_high}, bs_high={batch_sz_high}"<jupyter_output>Successfully submitted to Planet: Understanding the Amazon from Space
{'fraction_non_alphanumeric': 0.12023809523809524, 'fraction_numerical': 0.021031746031746033, 'mean_word_length': 8.733590733590734, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '647409', 'license': 'Apache-2.0', 'n_tokens_mistral': 1027, 'n_tokens_neox': 982, 'n_words': 174, 'path': 'notebooks/Implementation_v2_resnet34-Copy1.ipynb', 'repo_name': 'donare/Planets-Competition-Exercise', 'size': '2520'}