<jupyter_start><jupyter_text>Mount Google Drive<jupyter_code># Connect to Google Drive
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.environ['KAGGLE_CONFIG_DIR'] = "/content/gdrive/My Drive/Kaggle"
# Copying the files to the Colab instance gives an error
# !cp -r /content/gdrive/MyDrive/Projekte/deforestation /home/
# I/O error!<jupyter_output><empty_output><jupyter_text>Download Kaggle data<jupyter_code>#!kaggle datasets download -d nikitarom/planets-dataset
#unzipping the zip files and deleting the zip files
#!unzip \*.zip  && rm *.zip<jupyter_output><empty_output><jupyter_text>Clone own code from GitHub<jupyter_code>%cd /content
!git clone https://github.com/danielwiegand/deforestation<jupyter_output>Cloning into 'deforestation'...
remote: Enumerating objects: 238, done.[K
remote: Counting objects: 100% (238/238), done.[K
remote: Compressing objects: 100% (154/154), done.[K
remote: Total 238 (delta 138), reused 175 (delta 75), pack-reused 0[K
Receiving objects: 100% (238/238), 34.75 MiB | 39.31 MiB/s, done.
Resolving deltas: 100% (138/138), done.<jupyter_text>Unzip data from Google Drive to Colab<jupyter_code>!unzip "/content/gdrive/MyDrive/Projekte/deforestation/data.zip" -d "/content/deforestation/"
ls /content/deforestation/data<jupyter_output>[0m[01;34mimages[0m/                [01;34msample_submission_v2.csv[0m/  [01;34my_labels[0m/
rainforest_chips.jpeg  [01;34mtest_v2_file_mapping.csv[0m/<jupyter_text>Training<jupyter_code>%cd /content/deforestation/train
!pip install wandb
import os
import pickle

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
from tensorflow.keras.optimizers import Adam
from wandb.keras import WandbCallback
import wandb
from functions import load_labels, predict_on_testset, create_model, create_callbacks, evaluate_model, generate_generators<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>y_labels, UNIQUE_LABELS = load_labels()
weight_dict = pickle.load(open("pickle/weight_dict.p", "rb"))<jupyter_output><empty_output><jupyter_text>Initialize wandb<jupyter_code>run = wandb.init(project = "deforestation",
           reinit = True,
           name = "transfer learning #1",
           config = {"cnn_layers": None,
                     "filter_layout": None,
                     "batch_norm": None,
                     "max_pooling": None,
                     "dense_layers": None,
                     "dense_units": None,
                     "full_data": True,
                     "data_size": None,
                     "epochs": 100,
                     "patience": 3,
                     "augmentation": False,
                     "class_weight": False,
                     "finetuning": True,
                     "early_stop": True,
                     "transfer_learning": True,
                     "batch_size": 32,
                     "activation": "elu",
                     "optimizer": "adam"})

config = wandb.config<jupyter_output><empty_output><jupyter_text>Create model<jupyter_code>train_set, val_set = train_test_split(y_labels, test_size = 0.2)
train_generator, valid_generator = generate_generators(train_set, val_set, config, UNIQUE_LABELS, 
                                                       transfer_learning = config.transfer_learning, augmentation = config.augmentation)

m, base_model, F2Score = create_model(config, UNIQUE_LABELS, transfer_learning = config.transfer_learning)
early_stopping, checkpoint, reduce_lr = create_callbacks(model_name = wandb.run.name, patience = config.patience)
if config.class_weight == True:
  class_weight = weight_dict
else:
  class_weight = None<jupyter_output><empty_output><jupyter_text>Run model<jupyter_code>STEP_SIZE_TRAIN = train_generator.n // train_generator.batch_size
STEP_SIZE_VALID = valid_generator.n // valid_generator.batch_size
history = m.fit(train_generator,
                steps_per_epoch = STEP_SIZE_TRAIN,
                validation_data = valid_generator,
                validation_steps = STEP_SIZE_VALID,
                epochs = config.epochs,
                class_weight = class_weight,
                callbacks = [WandbCallback(), checkpoint, early_stopping, reduce_lr] # checkpoint, reduce_lr
                )
m.save("/content/gdrive/MyDrive/Projekte/deforestation/models/final_model")<jupyter_output><empty_output><jupyter_text>Finetuning Finetuning: The model is trained with a very low learning rate with all layers unfrozen.<jupyter_code># from tensorflow.keras.models import load_model
# m = load_model("/content/gdrive/MyDrive/Projekte/deforestation/models/final_model", compile = False) # compile = False: Wegen "custom metric". Siehe https://github.com/jakeret/unet/issues/8
from tensorflow.keras.optimizers import Adam

if config.finetuning == True:

  base_model.trainable = True
  m.summary()

  m.compile(optimizer = Adam(1e-5), # low learning rate 
            loss = 'binary_crossentropy', 
            metrics = [F2Score, "AUC"])

  epochs = 10

  history = m.fit(train_generator,
                  steps_per_epoch = STEP_SIZE_TRAIN,
                  validation_data = valid_generator,
                  validation_steps = STEP_SIZE_VALID,
                  epochs = epochs,
                  class_weight = class_weight
                  )<jupyter_output><empty_output><jupyter_text>Finish run<jupyter_code>run.finish()
m.save("/content/gdrive/MyDrive/Projekte/deforestation/models/final_model_finetuned")<jupyter_output>INFO:tensorflow:Assets written to: /content/gdrive/MyDrive/Projekte/deforestation/models/final_model_finetuned_alldata/assets<jupyter_text>Evaluation<jupyter_code>y_train, y_train_pred, y_val, y_val_pred, best_threshold = evaluate_model(m, history, train_generator, valid_generator, UNIQUE_LABELS)<jupyter_output>100%|██████████| 8095/8095 [00:17<00:00, 451.29it/s]<jupyter_text>Prediction<jupyter_code>submission = predict_on_testset(model = m, classes = train_generator.class_indices, threshold = best_threshold, transfer_learning = config.transfer_learning)<jupyter_output>Found 61191 images belonging to 2 classes.
1913/1913 [==============================] - 202s 104ms/step<jupyter_text>Submit to Kaggle<jupyter_code>#from google.colab import files
#uploaded = files.upload() # upload kaggle.JSON
#!mv kaggle.json /content/gdrive/MyDrive/Kaggle
submission.to_csv("submissions/submission.csv")
!kaggle competitions submit -c planet-understanding-the-amazon-from-space -f submissions/submission.csv -m "training on all images"<jupyter_output>Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.4)
100% 2.13M/2.13M [00:00<00:00, 2.74MB/s]
Successfully submitted to Planet: Understanding the Amazon from Space
{'fraction_non_alphanumeric': 0.1077238751282427, 'fraction_numerical': 0.024769163124725196, 'mean_word_length': 3.7988748241912798, 'pattern_counts': {'":': 18, '<': 53, '<?xml version=': 0, '>': 51, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '181293', 'license': 'MIT', 'n_tokens_mistral': 2372, 'n_tokens_neox': 2173, 'n_words': 433, 'path': 'train/colab_deforestation.ipynb', 'repo_name': 'danielwiegand/deforestation', 'size': '6823'}