<commit_before>; This is an example configuration for training POS Tagger. For a more detailed
; description of an INI example, please refer to the translation.ini file


[main]
name="pos tagging"
output="out-example-tagging"
tf_manager=<tf_manager>

train_dataset=<train_data>
val_dataset=<val_data>
test_datasets=[<val_data>]

runners=[<runner>]
trainer=<trainer>
evaluation=[("tags", evaluators.accuracy.Accuracy)]

batch_size=128
runners_batch_size=128
epochs=100

validation_period=500
logging_period=20


[tf_manager]
class=tf_manager.TensorFlowManager
num_sessions=1
num_threads=4


[train_data]
class=dataset.load_dataset_from_files
s_source="examples/data/tagging/train.forms-cs"
s_tags="examples/data/tagging/train.tags-cs.subpos"

[val_data]
class=dataset.load_dataset_from_files
s_source="examples/data/tagging/val.forms-cs"
s_tags="examples/data/tagging/val.tags-cs.subpos"
s_tags_out="out-example-tagging/val.tags-cs.subpos.out"


[source_vocabulary]
class=vocabulary.from_dataset
datasets=[<train_data>]
series_ids=["source"]
max_size=50000

[tags_vocabulary]
class=vocabulary.from_dataset
datasets=[<train_data>]
series_ids=["tags"]
max_size=50


[encoder]
class=encoders.sentence_encoder.SentenceEncoder
name="sentence_encoder"
rnn_size=300
max_input_len=50
embedding_size=300
dropout_keep_prob=0.5
data_id="source"
vocabulary=<source_vocabulary>

[decoder]
class=decoders.sequence_labeler.SequenceLabeler
name="tagger"
encoder=<encoder>
data_id="tags"
dropout_keep_prob=0.5
vocabulary=<tags_vocabulary>

<commit_msg>cosmetic changes in the tagging ini<commit_after>; This is an example configuration for training POS Tagger. For a more detailed
; description of an INI example, please refer to the translation.ini file

[main]
name="pos tagging"
output="out-example-tagging"
tf_manager=<tf_manager>

train_dataset=<train_data>
val_dataset=<val_data>
test_datasets=[<val_data>]

runners=[<runner>]
trainer=<trainer>
evaluation=[("tags", evaluators.accuracy.Accuracy)]

batch_size=50
runners_batch_size=50
epochs=100

validation_period=1500
logging_period=40

[tf_manager]
class=tf_manager.TensorFlowManager
num_sessions=1
num_threads=4

[train_data]
class=dataset.load_dataset_from_files
s_source="examples/data/tagging/train.forms-cs"
s_tags="examples/data/tagging/train.tags-cs.subpos"

[val_data]
class=dataset.load_dataset_from_files
s_source="examples/data/tagging/val.forms-cs"
s_tags="examples/data/tagging/val.tags-cs.subpos"
s_tags_out="out-example-tagging/val.tags-cs.subpos.out"


[source_vocabulary]
class=vocabulary.from_dataset
datasets=[<train_data>]
series_ids=["source"]
max_size=50000

[tags_vocabulary]
class=vocabulary.from_dataset
datasets=[<train_data>]
series_ids=["tags"]
max_size=50


[encoder]
class=encoders.sentence_encoder.SentenceEncoder
name="sentence_encoder"
rnn_size=100
max_input_len=50
embedding_size=300
dropout_keep_prob=0.5
data_id="source"
vocabulary=<source_vocabulary>

[decoder]
class=decoders.sequence_labeler.SequenceLabeler
name="tagger"
encoder=<encoder>
data_id="tags"
dropout_keep_prob=0.5
vocabulary=<tags_vocabulary>


{'fraction_non_alphanumeric': 0.11730395333765392, 'fraction_numerical': 0.02235904082955282, 'mean_word_length': 9.464406779661017, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5653227', 'n_tokens_mistral': 1213, 'n_tokens_neox': 1142, 'n_words': 167}