<jupyter_start><jupyter_text>Tutorial - Training Sequence Classifier Head> Training a Sequence Classifier<jupyter_code>from adaptnlp import EasyDocumentEmbeddings, SequenceClassifierTrainer
from flair.datasets import TREC_6<jupyter_output><empty_output><jupyter_text>Initialize corpus, output directory for the model, and document embeddings<jupyter_code>corpus = TREC_6() # Or path to directory of train.csv, test.csv, dev.csv files at "Path/to/data/directory" 
OUTPUT_DIR = "Path/to/model/output/directory" 
doc_embeddings = EasyDocumentEmbeddings("bert-base-cased", methods = ["rnn"])<jupyter_output>2020-04-17 06:32:27,268 Reading data from /home/andrew/.flair/datasets/trec_6
2020-04-17 06:32:27,269 Train: /home/andrew/.flair/datasets/trec_6/train.txt
2020-04-17 06:32:27,269 Dev: None
2020-04-17 06:32:27,269 Test: /home/andrew/.flair/datasets/trec_6/test.txt
May need a couple moments to instantiate...
RNN embeddings loaded<jupyter_text>Initialize Sequence Classification Trainer<jupyter_code>sc_configs = {
              "corpus": corpus,
              "encoder": doc_embeddings,
              "column_name_map": {0: "text", 1: "label"},
              "corpus_in_memory": True,
              "predictive_head": "flair",
             }
sc_trainer = SequenceClassifierTrainer(**sc_configs)<jupyter_output>2020-04-17 06:32:30,279 Computing label dictionary. Progress:<jupyter_text>Find Learning Rate with automated LR finder<jupyter_code>sc_lr_configs = {
        "output_dir": OUTPUT_DIR,
        "file_name": "learning_rate.tsv",
        "start_learning_rate": 1e-8,
        "end_learning_rate": 10,
        "iterations": 100,
        "mini_batch_size": 32,
        "lr_diff": 16,
        "stop_early": True,
        "smoothing_factor": 0.8,
        "plot_learning_rate": True,
}
learning_rate = sc_trainer.find_learning_rate(**sc_lr_configs)<jupyter_output>[1.5135612484362082e-08]
[1.8620871366628676e-08]
[2.2908676527677733e-08]
[2.818382931264454e-08]
[3.4673685045253164e-08]
[4.265795188015927e-08]
[5.2480746024977265e-08]
[6.456542290346554e-08]
[7.943282347242815e-08]
[9.772372209558107e-08]
[1.2022644346174127e-07]
[1.4791083881682077e-07]
[1.819700858609984e-07]
[2.2387211385683393e-07]
[2.754228703338167e-07]
[3.3884415613920264e-07]
[4.168693834703353e-07]
[5.128613839913649e-07]
[6.309573444801935e-07]
[7.762471166286916e-07]
[9.54992586021436e-07]
[1.1748975549395298e-06]
[1.4454397707459273e-06]
[1.778279410038923e-06]
[2.187761623949553e-06]
[2.6915348039269168e-06]
[3.311311214825913e-06]
[4.0738027780411255e-06]
[5.011872336272722e-06]
[6.165950018614822e-06]
[7.585775750291839e-06]
[9.332543007969913e-06]
[1.1481536214968832e-05]
[1.4125375446227536e-05]
[1.737800828749375e-05]
[2.1379620895022316e-05]
[2.6302679918953824e-05]
[3.2359365692962836e-05]
[3.981071705534974e-05]
[4.8977881936844595e-05]
[6.025595860743576e-05[...]<jupyter_text>Train Sequence Classifier<jupyter_code>sc_train_configs = {
        "output_dir": OUTPUT_DIR,
        "learning_rate": learning_rate,
        "mini_batch_size": 32,
        "anneal_factor": 0.5,
        "patience": 5,
        "max_epochs": 150,
        "plot_weights": False,
        "batch_growth_annealing": False,
}
sc_trainer.train(**sc_train_configs)<jupyter_output>2020-04-17 06:32:36,404 ----------------------------------------------------------------------------------------------------
2020-04-17 06:32:36,405 Model: "TextClassifier(
  (document_embeddings): DocumentRNNEmbeddings(
    (embeddings): StackedEmbeddings(
      (list_embedding_0): BertEmbeddings(
        (model): BertModel(
          (embeddings): BertEmbeddings(
            (word_embeddings): Embedding(28996, 768, padding_idx=0)
            (position_embeddings): Embedding(512, 768)
            (token_type_embeddings): Embedding(2, 768)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (encoder): BertEncoder(
            (layer): ModuleList(
              (0): BertLayer(
                (attention): BertAttention(
                  (self): BertSelfAttention(
                    (query): Linear(in_features=768, out_features=768, bias=True)
                    (key): Linear(in_featu[...]<jupyter_text>Load and Predict<jupyter_code>from adaptnlp import EasySequenceClassifier
# Set example text and instantiate tagger instance
example_text = '''Where was the Queen's wedding held? '''

classifier = EasySequenceClassifier()

sentences = classifier.tag_text(example_text, model_name_or_path=OUTPUT_DIR + "/final-model.pt")
print("Label output:\n")
for sentence in sentences:
    print(sentence.labels)<jupyter_output>2020-04-17 06:37:46,875 loading file Path/to/model/output/directory/final-model.pt
Label output:

[LOC (0.9990556836128235)]<jupyter_text>Training Sequence Classifier on Finetuned Model<jupyter_code>corpus = TREC_6() # Or path to directory of train.csv, test.csv, dev.csv files at "Path/to/data/directory" 
OUTPUT_DIR = "Path/to/model/output/directory"
FINETUNED_MODEL_DIR = "Path/to/finetuned/model/directory"
doc_embeddings = EasyDocumentEmbeddings(FINETUNED_MODEL_DIR, methods = ["rnn"])
sc_configs = {
              "corpus": corpus,
              "encoder": doc_embeddings,
              "column_name_map": {0: "text", 1: "label"},
              "corpus_in_memory": True,
              "predictive_head": "flair",
             }
sc_trainer = SequenceClassifierTrainer(**sc_configs)
sc_lr_configs = {
        "output_dir": OUTPUT_DIR,
        "start_learning_rate": 1e-8,
        "end_learning_rate": 10,
        "iterations": 100,
        "mini_batch_size": 32,
        "stop_early": True,
        "smoothing_factor": 0.8,
        "plot_learning_rate": True,
}
learning_rate = sc_trainer.find_learning_rate(**sc_lr_configs)
sc_train_configs = {
        "output_dir": OUTPUT_DIR,
        "learning_rate": learning_rate,
        "mini_batch_size": 32,
        "anneal_factor": 0.5,
        "patience": 5,
        "max_epochs": 150,
        "plot_weights": False,
        "batch_growth_annealing": False,
}
sc_trainer.train(**sc_train_configs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1416058394160584, 'fraction_numerical': 0.16123276561232766, 'mean_word_length': 3.629129129129129, 'pattern_counts': {'":': 44, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '112822', 'license': 'Apache-2.0', 'n_tokens_mistral': 2903, 'n_tokens_neox': 2285, 'n_words': 347, 'path': 'nbs/20c_tutorial.flair_seq_class_trainer.ipynb', 'repo_name': 'ElliottP-13/adaptnlp', 'size': '6165'}