<jupyter_start><jupyter_text>Training- Use this notebook after you've built your dataset to train the model.- Training will create a model directory with:    - **`model_0000XXX.pth`**: weights that are regularly saved during training    - **`config.yaml`**: the model configuration used during the last training performed.    - **`training.log`**: a log file that contains all the logs displayed in the notebook during training (useful if you close your notebook during a long training session).<jupyter_code>%env CUDA_VISIBLE_DEVICES=0
%load_ext autoreload
%autoreload 2
%matplotlib inline

from pathlib import Path
import logging

import matplotlib.pyplot as plt
import numpy as np

import sys; sys.path.append("../")
import maskflow

root_dir = Path("/home/hadim/.data/Neural_Network/Maskflow/Microtubule")
data_dir = root_dir / "Data"
model_dir = root_dir / "Models"
model_dir.mkdir(exist_ok=True)

# Import the configuration associated with this dataset and network.
config = maskflow.config.load_config(root_dir / "config.yaml")<jupyter_output>env: CUDA_VISIBLE_DEVICES=0
The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Check Dataset<jupyter_code>config['SOLVER']['IMS_PER_BATCH'] = 1
n_batch = 1

# Load some data
data_loader = maskflow.dataset.get_data_loader(config, data_dir, is_train=True)
some_data = [iter(data_loader).next() for _ in range(n_batch)]

# Retrieve category's names
categories = data_loader.dataset.coco.cats

for batch_image, batch_target, batch_idx in some_data:
    maskflow.viz.batch_display_top_masks(batch_image, batch_target, batch_idx, categories,
                                         basesize=14, limit=2, cmap="PuBu_r",
                                         pixel_mean=config['INPUT']['PIXEL_MEAN'],
                                         pixel_std=config['INPUT']['PIXEL_STD'])<jupyter_output>loading annotations into memory...
Done (t=2.66s)
creating index...
index created!<jupyter_text>Training<jupyter_code># Configure the training
config['DATA_DIR'] = str(data_dir)

config['MODEL']['DEVICE'] = "cuda"

config['DATALOADER']['NUM_WORKERS'] = 8
config['SOLVER']['IMS_PER_BATCH'] = 8

# Disable those settings if you want to use the training
# parameters from your config file.
custom_training = True
if custom_training:
    max_iter = 50000
    config['SOLVER']['MAX_ITER'] = max_iter
    config['SOLVER']['BASE_LR'] = 0.05
    config['SOLVER']['GAMMA'] = 0.4
    config['SOLVER']['STEPS'] = []
    config['SOLVER']['STEPS'].append(int(max_iter * 0.50))
    config['SOLVER']['STEPS'].append(int(max_iter * 0.60))
    config['SOLVER']['STEPS'].append(int(max_iter * 0.66))
    config['SOLVER']['STEPS'].append(int(max_iter * 0.80))
    config['SOLVER']['STEPS'].append(int(max_iter * 0.90))
    config['SOLVER']['CHECKPOINT_PERIOD'] = 1000
    config['SOLVER']['WARMUP_ITERS'] = 1000
    config['SOLVER']['WARMUP_FACTOR'] = 0.005

#config['SOLVER']
# Setup training

# Use the last model in the model dir
# If False, create a new model from scratch.
# If `model_to_use` is set, we use this one instead.

builder_args = {}
builder_args['config'] = config
builder_args['model_dir'] = model_dir
builder_args['use_last_model'] = False
builder_args['model_to_use'] = None
builder_args['use_pretrained_weights'] = True
builder_args['distributed'] = False

training_args = maskflow.training.build_model(**builder_args)<jupyter_output>2018-11-20 12:15:32,913:INFO:root: Training model directory set: /home/hadim/.data/Neural_Network/Maskflow/Microtubule/Models/2018.11.20-12:15:32
2018-11-20 12:15:32,914:INFO:root: Building the model...
2018-11-20 12:15:37,890:INFO:root: Use pretrained weights from https://download.pytorch.org/models/maskrcnn/e2e_mask_rcnn_R_50_FPN_1x.pth
2018-11-20 12:15:38,239:INFO:maskrcnn_benchmark.utils.checkpoint: No checkpoint found. Initializing model from scratch
2018-11-20 12:15:38,241:INFO:maskrcnn_benchmark.utils.checkpoint: Saving checkpoint to /home/hadim/.data/Neural_Network/Maskflow/Microtubule/Models/2018.11.20-12:15:32/model_0000000.pth
2018-11-20 12:15:38,451:INFO:root: Create the data loader.
2018-11-20 12:15:38,452:WARNING:maskrcnn_benchmark.data.build: When using more than one image per GPU you may encounter an out-of-memory (OOM) error if your GPU does not have sufficient memory. If this happens, you can reduce SOLVER.IMS_PER_BATCH (for training) or TEST.IMS_PER_BATCH (for infere[...]<jupyter_text>Check learning rates profilescheduler = training_args['scheduler']scheduler.last_epoch = 0epochs = np.arange(0, config['SOLVER']['MAX_ITER'])lrs = []for epoch in epochs:    scheduler.step()    lrs.append(scheduler.get_lr()[0])scheduler.last_epoch = -1plt.plot(epochs, lrs)<jupyter_code># Run training

training_args['log_period'] = 100
training_args['log_losses_detailed'] = False
training_args['save_metrics'] = True
training_args['tensorboard'] = True
training_args['tensorboard_log_period'] = 10

maskflow.training.do_train(**training_args)<jupyter_output>2018-11-20 12:15:41,663:INFO:maskfow.training: Start training at iteration 0
2018-11-20 12:15:41,835:INFO:maskfow.training: tensorboard --logdir /home/hadim/.data/Neural_Network/Maskflow/Microtubule/Models/2018.11.20-12:15:32/logs
2018-11-20 12:15:41,837:INFO:maskfow.training: Training will stop at 50000
2018-11-20 12:15:45,615:INFO:maskfow.training: Step: 0 | Loss: 5.133054 | ETA: 1 day, 15:36:17 | LR: 0.000250 | Memory: 5228 MB
{'fraction_non_alphanumeric': 0.13014705882352942, 'fraction_numerical': 0.06158088235294118, 'mean_word_length': 5.348891481913652, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '267561', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2094, 'n_tokens_neox': 1828, 'n_words': 405, 'path': 'notebooks/Detection/2_Training.ipynb', 'repo_name': 'hadim/maskflow', 'size': '5440'}