<commit_before>//- 💫 DOCS > USAGE > LIGHTNING TOUR

include ../../_includes/_mixins

p
    |  The following examples and code snippets give you an overview of spaCy's
    |  functionality and its usage.

+h(2, "models") Install and load models

+code(false, "bash").
    python -m spacy download en

+code.
    import spacy
    nlp = spacy.load('en')

+h(2, "examples-resources") Load resources and process text

+code.
    import spacy
    en_nlp = spacy.load('en')
    de_nlp = spacy.load('de')
    en_doc = en_nlp(u'Hello, world. Here are two sentences.')
    de_doc = de_nlp(u'ich bin ein Berliner.')

+h(2, "multi-threaded") Multi-threaded generator

+code.
    texts = [u'One document.', u'...', u'Lots of documents']
    # .pipe streams input, and produces streaming output
    iter_texts = (texts[i % 3] for i in xrange(100000000))
    for i, doc in enumerate(nlp.pipe(iter_texts, batch_size=50, n_threads=4)):
        assert doc.is_parsed
        if i == 100:
            break

+h(2, "examples-tokens-sentences") Get tokens and sentences

+code.
    token = doc[0]
    sentence = next(doc.sents)
    assert token is sentence[0]
    assert sentence.text == 'Hello, world.'

+h(2, "examples-integer-ids") Use integer IDs for any string

+code.
    hello_id = nlp.vocab.strings['Hello']
    hello_str = nlp.vocab.strings[hello_id]

    assert token.orth  == hello_id  == 3125
    assert token.orth_ == hello_str == 'Hello'

+h(2, "examples-string-views-flags") Get and set string views and flags

+code.
    assert token.shape_ == 'Xxxxx'
    for lexeme in nlp.vocab:
        if lexeme.is_alpha:
            lexeme.shape_ = 'W'
        elif lexeme.is_digit:
            lexeme.shape_ = 'D'
        elif lexeme.is_punct:
            lexeme.shape_ = 'P'
        else:
            lexeme.shape_ = 'M'
    assert token.shape_ == 'W'

+h(2, "examples-numpy-arrays") Export to numpy arrays

+code.
    from spacy.attrs import ORTH, LIKE_URL, IS_OOV

    attr_ids = [ORTH, LIKE_URL, IS_OOV]
    doc_array = doc.to_array(attr_ids)
    assert doc_array.shape == (len(doc), len(attr_ids))
    assert doc[0].orth == doc_array[0, 0]
    assert doc[1].orth == doc_array[1, 0]
    assert doc[0].like_url == doc_array[0, 1]
    assert list(doc_array[:, 1]) == [t.like_url for t in doc]

+h(2, "examples-word-vectors") Word vectors

+code.
    doc = nlp("Apples and oranges are similar. Boots and hippos aren't.")

    apples = doc[0]
    oranges = doc[2]
    boots = doc[6]
    hippos = doc[8]

    assert apples.similarity(oranges) > boots.similarity(hippos)

+h(2, "examples-pos-tags") Part-of-speech tags

+code.
    from spacy.parts_of_speech import ADV

    def is_adverb(token):
        return token.pos == spacy.parts_of_speech.ADV

    # These are data-specific, so no constants are provided. You have to look
    # up the IDs from the StringStore.
    NNS = nlp.vocab.strings['NNS']
    NNPS = nlp.vocab.strings['NNPS']
    def is_plural_noun(token):
        return token.tag == NNS or token.tag == NNPS

    def print_coarse_pos(token):
        print(token.pos_)
<commit_msg>make this work in python2.7<commit_after>//- 💫 DOCS > USAGE > LIGHTNING TOUR

include ../../_includes/_mixins

p
    |  The following examples and code snippets give you an overview of spaCy's
    |  functionality and its usage.

+h(2, "models") Install and load models

+code(false, "bash").
    python -m spacy download en

+code.
    import spacy
    nlp = spacy.load('en')

+h(2, "examples-resources") Load resources and process text

+code.
    import spacy
    en_nlp = spacy.load('en')
    de_nlp = spacy.load('de')
    en_doc = en_nlp(u'Hello, world. Here are two sentences.')
    de_doc = de_nlp(u'ich bin ein Berliner.')

+h(2, "multi-threaded") Multi-threaded generator

+code.
    texts = [u'One document.', u'...', u'Lots of documents']
    # .pipe streams input, and produces streaming output
    iter_texts = (texts[i % 3] for i in xrange(100000000))
    for i, doc in enumerate(nlp.pipe(iter_texts, batch_size=50, n_threads=4)):
        assert doc.is_parsed
        if i == 100:
            break

+h(2, "examples-tokens-sentences") Get tokens and sentences

+code.
    token = doc[0]
    sentence = next(doc.sents)
    assert token is sentence[0]
    assert sentence.text == 'Hello, world.'

+h(2, "examples-integer-ids") Use integer IDs for any string

+code.
    hello_id = nlp.vocab.strings['Hello']
    hello_str = nlp.vocab.strings[hello_id]

    assert token.orth  == hello_id  == 3125
    assert token.orth_ == hello_str == 'Hello'

+h(2, "examples-string-views-flags") Get and set string views and flags

+code.
    assert token.shape_ == 'Xxxxx'
    for lexeme in nlp.vocab:
        if lexeme.is_alpha:
            lexeme.shape_ = 'W'
        elif lexeme.is_digit:
            lexeme.shape_ = 'D'
        elif lexeme.is_punct:
            lexeme.shape_ = 'P'
        else:
            lexeme.shape_ = 'M'
    assert token.shape_ == 'W'

+h(2, "examples-numpy-arrays") Export to numpy arrays

+code.
    from spacy.attrs import ORTH, LIKE_URL, IS_OOV

    attr_ids = [ORTH, LIKE_URL, IS_OOV]
    doc_array = doc.to_array(attr_ids)
    assert doc_array.shape == (len(doc), len(attr_ids))
    assert doc[0].orth == doc_array[0, 0]
    assert doc[1].orth == doc_array[1, 0]
    assert doc[0].like_url == doc_array[0, 1]
    assert list(doc_array[:, 1]) == [t.like_url for t in doc]

+h(2, "examples-word-vectors") Word vectors

+code.
    doc = nlp(u"Apples and oranges are similar. Boots and hippos aren't.")

    apples = doc[0]
    oranges = doc[2]
    boots = doc[6]
    hippos = doc[8]

    assert apples.similarity(oranges) > boots.similarity(hippos)

+h(2, "examples-pos-tags") Part-of-speech tags

+code.
    from spacy.parts_of_speech import ADV

    def is_adverb(token):
        return token.pos == spacy.parts_of_speech.ADV

    # These are data-specific, so no constants are provided. You have to look
    # up the IDs from the StringStore.
    NNS = nlp.vocab.strings['NNS']
    NNPS = nlp.vocab.strings['NNPS']
    def is_plural_noun(token):
        return token.tag == NNS or token.tag == NNPS

    def print_coarse_pos(token):
        print(token.pos_)

{'fraction_non_alphanumeric': 0.12789755227751662, 'fraction_numerical': 0.014913276057707894, 'mean_word_length': 2.85625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4924410', 'n_tokens_mistral': 2350, 'n_tokens_neox': 2221, 'n_words': 602}