<jupyter_start><jupyter_text>Dependencies<jupyter_code>import json, warnings, shutil
from tweet_utility_scripts import *
from tweet_utility_preprocess_roberta_scripts_aux import *
from transformers import TFRobertaModel, RobertaConfig
from tokenizers import ByteLevelBPETokenizer
from tensorflow.keras.models import Model
from tensorflow.keras import optimizers, metrics, losses, layers
from scripts_step_lr_schedulers import *
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint

SEED = 0
seed_everything(SEED)
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code># Unzip files
!tar -xf /kaggle/input/tweet-dataset-5fold-roberta-64-clean/fold_1.tar.gz
!tar -xf /kaggle/input/tweet-dataset-5fold-roberta-64-clean/fold_2.tar.gz
!tar -xf /kaggle/input/tweet-dataset-5fold-roberta-64-clean/fold_3.tar.gz
!tar -xf /kaggle/input/tweet-dataset-5fold-roberta-64-clean/fold_4.tar.gz
!tar -xf /kaggle/input/tweet-dataset-5fold-roberta-64-clean/fold_5.tar.gz
database_base_path = '/kaggle/input/tweet-dataset-5fold-roberta-64-clean/'
k_fold = pd.read_csv(database_base_path + '5-fold.csv')
print(f'Training set samples: {len(k_fold)}')
display(k_fold.head())<jupyter_output>Training set samples: 26882<jupyter_text>Model parameters<jupyter_code>vocab_path = database_base_path + 'vocab.json'
merges_path = database_base_path + 'merges.txt'
base_path = '/kaggle/input/qa-transformers/roberta/'

config = {
  'MAX_LEN': 64,
  'BATCH_SIZE': 32,
  'EPOCHS': 7,
  'LEARNING_RATE': 3e-5,
  'ES_PATIENCE': 2,
  'N_FOLDS': 5,
  'question_size': 4,
  'base_model_path': base_path + 'roberta-base-tf_model.h5',
  'config_path': base_path + 'roberta-base-config.json'
}

with open('config.json', 'w') as json_file:
    json.dump(json.loads(json.dumps(config)), json_file)<jupyter_output><empty_output><jupyter_text>Tokenizer<jupyter_code>tokenizer = ByteLevelBPETokenizer(vocab_file=vocab_path, merges_file=merges_path, 
                                  lowercase=True, add_prefix_space=True)
tokenizer.save('./')<jupyter_output><empty_output><jupyter_text>Learning rate schedule<jupyter_code>lr_min = 1e-6
lr_start = 1e-7
lr_max = config['LEARNING_RATE']
train_size = len(k_fold[k_fold['fold_1'] == 'train'])
step_size = train_size // config['BATCH_SIZE']
total_steps = config['EPOCHS'] * step_size
warmup_steps = total_steps * 0.1
decay = .9985

rng = [i for i in range(0, total_steps, config['BATCH_SIZE'])]
y = [exponential_schedule_with_warmup(tf.cast(x, tf.float32), warmup_steps=warmup_steps, lr_start=lr_start, 
                                      lr_max=lr_max, lr_min=lr_min, decay=decay) for x in rng]

sns.set(style="whitegrid")
fig, ax = plt.subplots(figsize=(20, 6))
plt.plot(rng, y)
print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<jupyter_output>Learning rate schedule: 1e-07 to 2.96e-05 to 1e-06<jupyter_text>Model<jupyter_code>module_config = RobertaConfig.from_pretrained(config['config_path'], output_hidden_states=True)

def model_fn(MAX_LEN):
    input_ids = layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_ids')
    attention_mask = layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name='attention_mask')
    
    base_model = TFRobertaModel.from_pretrained(config['base_model_path'], config=module_config, name="base_model")
    _, _, hidden_states = base_model({'input_ids': input_ids, 'attention_mask': attention_mask})
    
    h11 = hidden_states[-2]
    
    start_logits = layers.Dense(1, name="start_logit", use_bias=False)(h11)
    start_logits = layers.Flatten()(start_logits)

    end_logits = layers.Dense(1, name="end_logit", use_bias=False)(h11)
    end_logits = layers.Flatten()(end_logits)

    start_probs = layers.Activation('softmax', name='y_start')(start_logits)
    end_probs = layers.Activation('softmax', name='y_end')(end_logits)

    model = Model(inputs=[input_ids, attention_mask], outputs=[start_probs, end_probs])
    
    return model<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>AUTO = tf.data.experimental.AUTOTUNE
strategy = tf.distribute.get_strategy()
history_list = []

for n_fold in range(config['N_FOLDS']):
    n_fold +=1
    print('\nFOLD: %d' % (n_fold))
    # Load data
    base_data_path = 'fold_%d/' % (n_fold)
    x_train = np.load(base_data_path + 'x_train.npy')
    y_train = np.load(base_data_path + 'y_train.npy')
    x_valid = np.load(base_data_path + 'x_valid.npy')
    y_valid = np.load(base_data_path + 'y_valid.npy')
    
    step_size = x_train.shape[1] // config['BATCH_SIZE']

    # Train model
    model_path = 'model_fold_%d.h5' % (n_fold)
    model = model_fn(config['MAX_LEN'])
    
    es = EarlyStopping(monitor='val_loss', mode='min', patience=config['ES_PATIENCE'], 
                       restore_best_weights=True, verbose=1)
    checkpoint = ModelCheckpoint(model_path, monitor='val_loss', mode='min', 
                                 save_best_only=True, save_weights_only=True)
    
    optimizer = optimizers.Adam(learning_rate=lambda: exponential_schedule_with_warmup(tf.cast(optimizer.iterations, tf.float32), 
                                                                                       warmup_steps=warmup_steps, lr_start=lr_start, 
                                                                                       lr_max=lr_max, lr_min=lr_min, decay=decay))
    
    model.compile(optimizer, loss={'y_start': losses.CategoricalCrossentropy(label_smoothing=0.2),
                                   'y_end': losses.CategoricalCrossentropy(label_smoothing=0.2)})
    
    history = model.fit(get_training_dataset(x_train, y_train, config['BATCH_SIZE'], AUTO, seed=SEED),
                        validation_data=(get_validation_dataset(x_valid, y_valid, config['BATCH_SIZE'], AUTO, repeated=False, seed=SEED)),
                        epochs=config['EPOCHS'], 
                        steps_per_epoch=step_size,
                        callbacks=[checkpoint, es],
                        verbose=2).history
    
    history_list.append(history)

    # Make predictions
    predict_eval_df(k_fold, model, x_train, x_valid, get_test_dataset, decode, n_fold, tokenizer, config, config['question_size'])
    
    ### Delete data dir
    shutil.rmtree(base_data_path)<jupyter_output>FOLD: 1
Train for 672 steps, validate for 168 steps
Epoch 1/7
672/672 - 175s - loss: 4.6513 - y_start_loss: 2.2740 - y_end_loss: 2.3773 - val_loss: 3.8394 - val_y_start_loss: 1.9411 - val_y_end_loss: 1.8983
Epoch 2/7
672/672 - 163s - loss: 3.7998 - y_start_loss: 1.9021 - y_end_loss: 1.8977 - val_loss: 3.7876 - val_y_start_loss: 1.9164 - val_y_end_loss: 1.8712
Epoch 3/7
672/672 - 163s - loss: 3.6798 - y_start_loss: 1.8524 - y_end_loss: 1.8274 - val_loss: 3.7810 - val_y_start_loss: 1.9135 - val_y_end_loss: 1.8675
Epoch 4/7
672/672 - 163s - loss: 3.6437 - y_start_loss: 1.8372 - y_end_loss: 1.8065 - val_loss: 3.7759 - val_y_start_loss: 1.9129 - val_y_end_loss: 1.8630
Epoch 5/7
672/672 - 163s - loss: 3.6253 - y_start_loss: 1.8280 - y_end_loss: 1.7974 - val_loss: 3.7737 - val_y_start_loss: 1.9112 - val_y_end_loss: 1.8624
Epoch 6/7
672/672 - 161s - loss: 3.6213 - y_start_loss: 1.8282 - y_end_loss: 1.7932 - val_loss: 3.7749 - val_y_start_loss: 1.9123 - val_y_end_loss: 1.8627
Epoch 7/7
Restori[...]<jupyter_text>Model loss graph<jupyter_code>for n_fold in range(config['N_FOLDS']):
    print('Fold: %d' % (n_fold+1))
    plot_metrics(history_list[n_fold])<jupyter_output>Fold: 1<jupyter_text>Model evaluation<jupyter_code>display(evaluate_model_kfold(k_fold, config['N_FOLDS']).style.applymap(color_map))<jupyter_output><empty_output><jupyter_text>Visualize predictions<jupyter_code>display(k_fold[[c for c in k_fold.columns if not (c.startswith('textID') or 
                                                  c.startswith('text_len') or 
                                                  c.startswith('selected_text_len') or 
                                                  c.startswith('text_wordCnt') or 
                                                  c.startswith('selected_text_wordCnt') or 
                                                  c.startswith('fold_') or 
                                                  c.startswith('start_fold_') or 
                                                  c.startswith('end_fold_'))]].head(15))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1253279274982113, 'fraction_numerical': 0.04257095158597663, 'mean_word_length': 3.372784150156413, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '630541', 'license': 'MIT', 'n_tokens_mistral': 3312, 'n_tokens_neox': 3044, 'n_words': 469, 'path': 'Model backlog/Train/218-tweet-train-5fold-roberta-hidden-11-reference.ipynb', 'repo_name': 'dimitreOliveira/Tweet-Sentiment-Extraction', 'size': '8386'}