<commit_before>include ../../_includes/_mixins

p
    |  This tutorial describes how to train new statistical models for spaCy's
    |  part-of-speech tagger, named entity recognizer and dependency parser.

p
    |  I'll start with some quick code examples, that describe how to train
    |  each model. I'll then provide a bit of background about the algorithms,
    |  and explain how the data and feature templates work.

+h(2, "train-pos-tagger") Training the part-of-speech tagger

+code.
    from spacy.vocab import Vocab
    from spacy.pipeline import Tagger
    from spacy.tokens import Doc

    vocab = Vocab(tag_map={'N': {'pos': 'NOUN'}, 'V': {'pos': 'VERB'}})
    tagger = Tagger(vocab)

    doc = Doc(vocab, words=['I', 'like', 'stuff'])
    tagger.update(doc, ['N', 'V', 'N'])

    tagger.model.end_training()

p
    +button(gh("spaCy", "examples/training/train_tagger.py"), false, "secondary") Full example

+h(2, "train-entity") Training the named entity recognizer

+code.
    from spacy.vocab import Vocab
    from spacy.pipeline import EntityRecognizer
    from spacy.tokens import Doc

    vocab = Vocab()
    entity = EntityRecognizer(vocab, entity_types=['PERSON', 'LOC'])

    doc = Doc(vocab, words=['Who', 'is', 'Shaka', 'Khan', '?'])
    entity.update(doc, ['O', 'O', 'B-PERSON', 'L-PERSON', 'O'])

    entity.model.end_training()

p
    +button(gh("spaCy", "examples/training/train_ner.py"), false, "secondary") Full example<commit_msg>update to training doc<commit_after>include ../../_includes/_mixins

p
    |  This tutorial describes how to train new statistical models for spaCy's
    |  part-of-speech tagger, named entity recognizer and dependency parser.

p
    |  I'll start with some quick code examples, that describe how to train
    |  each model. I'll then provide a bit of background about the algorithms,
    |  and explain how the data and feature templates work.

+h(2, "train-pos-tagger") Training the part-of-speech tagger

+code.
    from spacy.vocab import Vocab
    from spacy.tagger import Tagger
    from spacy.tokens import Doc
    from spacy.gold import GoldParse


    vocab = Vocab(tag_map={'N': {'pos': 'NOUN'}, 'V': {'pos': 'VERB'}})
    tagger = Tagger(vocab)

    doc = Doc(vocab, words=['I', 'like', 'stuff'])
    gold = GoldParse(doc, tags=['N', 'V', 'N'])
    tagger.update(doc, gold)

    tagger.model.end_training()

p
    +button(gh("spaCy", "examples/training/train_tagger.py"), false, "secondary") Full example

+h(2, "train-entity") Training the named entity recognizer

+code.
    from spacy.vocab import Vocab
    from spacy.pipeline import EntityRecognizer
    from spacy.tokens import Doc

    vocab = Vocab()
    entity = EntityRecognizer(vocab, entity_types=['PERSON', 'LOC'])

    doc = Doc(vocab, words=['Who', 'is', 'Shaka', 'Khan', '?'])
    entity.update(doc, ['O', 'O', 'B-PERSON', 'L-PERSON', 'O'])

    entity.model.end_training()

p
    +button(gh("spaCy", "examples/training/train_ner.py"), false, "secondary") Full example
{'fraction_non_alphanumeric': 0.14361702127659576, 'fraction_numerical': 0.0013297872340425532, 'mean_word_length': 3.7163009404388716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5648633', 'n_tokens_mistral': 1043, 'n_tokens_neox': 1008, 'n_words': 303}