<jupyter_start><jupyter_text>Execute le GRU-D<jupyter_code>from __future__ import absolute_import, division, print_function

import argparse
from datetime import datetime
import numpy as np
import os

from keras import backend as K
from keras.callbacks import EarlyStopping, TensorBoard
import numpy as np
from sklearn.metrics import roc_auc_score

from data_handler import DataHandler
from models import create_grud_model, load_grud_model
from nn_utils.callbacks import ModelCheckpointwithBestWeights

# set GPU usage for tensorflow backend
if K.backend() == 'tensorflow':
    import tensorflow as tf
    config = tf.ConfigProto()
    config.gpu_options.per_process_gpu_memory_fraction = .1
    config.gpu_options.allow_growth = True
    K.set_session(tf.Session(config=config))
# parse arguments
## general
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--working_path', default='.')

## data
arg_parser.add_argument('dataset_name', default='detection',
                        help='The data files should be saved in [working_path]/data/[dataset_name] directory.')
arg_parser.add_argument('label_name', default='risk_situation')
arg_parser.add_argument('--max_timesteps', type=int, default=200, 
                        help='Time series of at most # time steps are used. Default: 200.')
arg_parser.add_argument('--max_timestamp', type=int, default=48*60*60,
                        help='Time series of at most # seconds are used. Default: 48 (hours).')

## model
arg_parser.add_argument('--recurrent_dim', type=lambda x: x and [int(xx) for xx in x.split(',')] or [], default='64')
arg_parser.add_argument('--hidden_dim', type=lambda x: x and [int(xx) for xx in x.split(',')] or [], default='64')
arg_parser.add_argument('--model', default='GRUD', choices=['GRUD', 'GRUforward', 'GRU0', 'GRUsimple'])
arg_parser.add_argument('--use_bidirectional_rnn', default=False)
                           
## training
arg_parser.add_argument('--pretrained_model_file', default=None,
                        help='If pre-trained model is provided, training will be skipped.') # e.g., [model_name]_[i_fold].h5
arg_parser.add_argument('--epochs', type=int, default=100)
arg_parser.add_argument('--early_stopping_patience', type=int, default=10)
arg_parser.add_argument('--batch_size', type=int, default=32)


## set the actual arguments if running in notebook
if not (__name__ == '__main__' and '__file__' in globals()):
    ARGS = arg_parser.parse_args([
        'detection',
        'risk_situation',
        '--model', 'GRUD',
        '--hidden_dim', '',
        '--epochs', '100'
    ])
else:
    ARGS = arg_parser.parse_args()

print('Arguments:', ARGS)
# get dataset
dataset = DataHandler(
    data_path=os.path.join(ARGS.working_path, 'data', ARGS.dataset_name), 
    label_name=ARGS.label_name, 
    max_steps=ARGS.max_timesteps,
    max_timestamp=ARGS.max_timestamp
)
# k-fold cross-validation
pred_y_list_all = []
auc_score_list_all = []

timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
print('Timestamp: {}'.format(timestamp))

for i_fold in range(dataset.folds):
    print('{}-th fold...'.format(i_fold))

    # Load or train the model.
    if ARGS.pretrained_model_file is not None:
        model = load_grud_model(os.path.join(ARGS.working_path, 
                                             ARGS.pretrained_model_file.format(i_fold=i_fold)))
    else:
        model = create_grud_model(input_dim=dataset.input_dim,
                                  output_dim=dataset.output_dim,
                                  output_activation=dataset.output_activation,
                                  recurrent_dim=ARGS.recurrent_dim,
                                  hidden_dim=ARGS.hidden_dim,
                                  predefined_model=ARGS.model,
                                  use_bidirectional_rnn=ARGS.use_bidirectional_rnn
                                 )
        if i_fold == 0:
            model.summary()
        model.compile(optimizer='adam', loss=dataset.loss_function)
        model.fit_generator(
            generator=dataset.training_generator(i_fold, batch_size=ARGS.batch_size),
            steps_per_epoch=dataset.training_steps(i_fold, batch_size=ARGS.batch_size),
            epochs=ARGS.epochs,
            verbose=1,
            validation_data=dataset.validation_generator(i_fold, batch_size=ARGS.batch_size),
            validation_steps=dataset.validation_steps(i_fold, batch_size=ARGS.batch_size),
            callbacks=[
                EarlyStopping(patience=ARGS.early_stopping_patience),
                ModelCheckpointwithBestWeights(
                    file_dir=os.path.join(ARGS.working_path, 'model', timestamp + '_' + str(i_fold))
                ),
                TensorBoard(
                    log_dir=os.path.join(ARGS.working_path, 'tb_logs', timestamp + '_' + str(i_fold))
                )
            ]
            )
        model.save(os.path.join(ARGS.working_path, 'model', 
                                timestamp + '_' + str(i_fold), 'model.h5'))

    # Evaluate the model
    true_y_list = [
        dataset.training_y(i_fold), dataset.validation_y(i_fold), dataset.testing_y(i_fold)
    ]
    pred_y_list = [
        model.predict_generator(dataset.training_generator_x(i_fold, batch_size=ARGS.batch_size),
                                steps=dataset.training_steps(i_fold, batch_size=ARGS.batch_size)),
        model.predict_generator(dataset.validation_generator_x(i_fold, batch_size=ARGS.batch_size),
                                steps=dataset.validation_steps(i_fold, batch_size=ARGS.batch_size)),
        model.predict_generator(dataset.testing_generator_x(i_fold, batch_size=ARGS.batch_size),
                                steps=dataset.testing_steps(i_fold, batch_size=ARGS.batch_size)),
    ]
    auc_score_list = [roc_auc_score(ty, py) for ty, py in zip(true_y_list, pred_y_list)] # [3, n_task]
    print('AUC score of this fold: {}'.format(auc_score_list))
    pred_y_list_all.append(pred_y_list)
    auc_score_list_all.append(auc_score_list)

print('Finished!', '='*20)
auc_score_list_all = np.stack(auc_score_list_all, axis=0)
print('Mean AUC score: {}; Std AUC score: {}'.format(
    np.mean(auc_score_list_all, axis=0),
    np.std(auc_score_list_all, axis=0)))

result_path = os.path.join(ARGS.working_path, 'results', timestamp)
if not os.path.exists(result_path):
    os.makedirs(result_path)
np.savez_compressed(os.path.join(result_path, 'predictions.npz'),
                    pred_y_list_all=pred_y_list_all)
np.savez_compressed(os.path.join(result_path, 'auroc_score.npz'),
                    auc_score_list_all=auc_score_list_all)<jupyter_output>Timestamp: 20200306_174649_795018
0-th fold...
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_10 (InputLayer)           (None, None, 10)     0                                            
__________________________________________________________________________________________________
input_11 (InputLayer)           (None, None, 10)     0                                            
__________________________________________________________________________________________________
input_12 (InputLayer)           (None, None, 1)      0                                            
__________________________________________________________________________________________________
external_masking_7 (ExternalMas (None, None, 10)     0        [...]
{'fraction_non_alphanumeric': 0.11780036486838676, 'fraction_numerical': 0.010294500912170967, 'mean_word_length': 2.9623128549303046, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '85403', 'license': 'MIT', 'n_tokens_mistral': 2303, 'n_tokens_neox': 2194, 'n_words': 390, 'path': 'Run.ipynb', 'repo_name': 'edwinlebreton/Birds', 'size': '7674'}