<jupyter_start><jupyter_text>TRANSFER LEARNING HERE<jupyter_code>n_labels =2
filetrain = '/home/gwiedemann/notebooks/OffLang/sample_train.txt'
filedev = '/home/gwiedemann/notebooks/OffLang/sample_dev.txt'
train_sentences, train_labels, dev_sentences, dev_labels = train_dev_sentences(filetrain=filetrain,
                   filedev=filedev, check=3)
print(dev_sentences[0])
print(dev_labels[:20])
generator = sequential_generator
train_sentences = train_sentences
devLabels = dev_labels
number_of_tests = 5
number_of_epochs = 50
combined_tl_log = '/home/jindal/notebooks/jindal/NER/language_model/results_tl_combined.txt' 
combined_tl_save_weigths='/home/jindal/notebooks/jindal/NER/language_model/classification_model_tl_combined.h5'
batch_size=32
combined_tl_train_file='/home/gwiedemann/notebooks/OffLang/sample_train.txt'
f1_measure='binary'
pos_label=1
load_model_weights=True
model_weights_file = '/home/jindal/notebooks/jindal/NER/language_model/model_pretrained_combined.h5'
nb_sequence_length = nb_sequence_length
nb_embedding_dims= nb_embedding_dims
check_for_generator=3
test_model(generator=generator, 
           train_sentences=train_sentences, 
           devLabels=devLabels, 
           number_of_tests= number_of_tests,
           number_of_epochs=number_of_epochs, 
           filename_to_log=combined_tl_log, 
           filename_to_save_weigths=combined_tl_save_weigths,
           batch_size=batch_size,
           train_file=combined_tl_train_file, 
           f1_measure=f1_measure, 
           pos_label=pos_label, 
           load_model_weights=load_model_weights,
           model_weights_file = model_weights_file, 
           nb_sequence_length=nb_sequence_length, 
           nb_embedding_dims=nb_embedding_dims, 
           check_for_generator= check_for_generator)<jupyter_output>Test 1/5
Epoch: 1
Epoch 1/1
132/132 [==============================] - 34s 256ms/step - loss: 0.6604 - acc: 0.7060
model saved. F1 is 0.544601
Test-Data: Prec: 0.730, Rec: 0.434, F1: 0.545, Acc: 0.760
Epoch: 2
Epoch 1/1
132/132 [==============================] - 32s 241ms/step - loss: 0.5137 - acc: 0.7588
model saved. F1 is 0.649706
Test-Data: Prec: 0.680, Rec: 0.622, F1: 0.650, Acc: 0.778
Epoch: 3
Epoch 1/1
132/132 [==============================] - 32s 240ms/step - loss: 0.4732 - acc: 0.7744
model saved. F1 is 0.676864
Test-Data: Prec: 0.691, Rec: 0.663, F1: 0.677, Acc: 0.791
Epoch: 4
Epoch 1/1
132/132 [==============================] - 31s 239ms/step - loss: 0.4399 - acc: 0.7952
model saved. F1 is 0.686792
Test-Data: Prec: 0.692, Rec: 0.682, F1: 0.687, Acc: 0.795
Epoch: 5
Epoch 1/1
132/132 [==============================] - 31s 238ms/step - loss: 0.4117 - acc: 0.8151
Test-Data: Prec: 0.684, Rec: 0.682, F1: 0.683, Acc: 0.791
Epoch: 6
Epoch 1/1
132/132 [==============================][...]
{'fraction_non_alphanumeric': 0.1650692225772098, 'fraction_numerical': 0.09407170749023784, 'mean_word_length': 4.247672253258846, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '256526', 'license': 'MIT', 'n_tokens_mistral': 1252, 'n_tokens_neox': 1047, 'n_words': 187, 'path': 'language_model/training_on_combined.ipynb', 'repo_name': 'uhh-lt/microNER-de', 'size': '2817'}