<jupyter_start><jupyter_text>Extensions____________________*This notebook is based on / inspired by excellent course materials from https://campus.datacamp.com/courses/advanced-nlp-with-spacy course at DataCamp*____________________To set an extension we use `.set_extension()` method. This method can be used on:* `Doc`* `Span`* `Token`Let's see a set of examples:* `Doc.set_extension('title', default = None)`* `Span.set_extension('is_german_word', default = False)`* `Token.set_extension('has_color', default = False)`To access extensions we need to use `._.` to distinguish them from built-in properties:`doc._.title = 'Document 1'` Types of extensions* Attribute extensions* Property extensions* Method extensions<jupyter_code># Get the model
nlp = spacy.load('en_core_web_sm')
# Define some data
my_str = 'I used to live in Vienna 5 years ago.'
doc1 = nlp(my_str)<jupyter_output><empty_output><jupyter_text>Token level extensions<jupyter_code># Token level attributes
Token.set_extension('is_city', default = False)
# Set extension
doc1[5]._.is_city = True

# Let's see how it works
print([(token.text, token._.is_city) for token in doc1])<jupyter_output>[('I', False), ('used', False), ('to', False), ('live', False), ('in', False), ('Vienna', True), ('5', False), ('years', False), ('ago', False), ('.', False)]<jupyter_text>Doc level extensions<jupyter_code># Define a getter function

def get_has_number(doc):
    # If any token is like_num - return True
    return any(token.like_num for token in doc)

# Register the Doc property extension 'has_number' with the getter get_has_number
Doc.set_extension('has_number', getter = get_has_number)
# Check how it works
print('has_number:', doc1._.has_number)<jupyter_output>has_number: True<jupyter_text>Span level extensions<jupyter_code># Define a method

def to_html(span, tag):
    # Wrap the span text in a HTML tag and return it
    return '<{tag}>{text}</{tag}>'.format(tag = tag, text = span.text)
# Register the Span property extension 'to_html' with the method to_html
Span.set_extension('to_html', method = to_html)
# Process the text and call to_html method on the span with `h1` tag
doc = nlp("Hello world, this is my sentence.")
span = doc[0:2]
print(span._.to_html('h1'))<jupyter_output><h1>Hello world</h1><jupyter_text>Extensions and entities  Example 1<jupyter_code>def get_wikipedia_url(span):
    # Get a Wikipedia URL if the span has one of the labels
    if span.label_ in ('PERSON', 'ORG', 'GPE', 'LOCATION'):
        entity_text = span.text.replace(' ', '_')
        return "https://en.wikipedia.org/w/index.php?search=" + entity_text
    
# Set the Span extension wikipedia_url using get getter get_wikipedia_url
Span.set_extension('wikipedia_url', getter = get_wikipedia_url, force = True)
# Phrase
doc = nlp("In over fifty years from his very first recordings right through to his last album, <NAME> was at the vanguard of contemporary culture. <NAME>")

# Get names and links
for ent in doc.ents:
    # Print the text and Wikipedia URL of the entity
    print(ent.text, ent._.wikipedia_url)<jupyter_output>fifty years None
<NAME> https://en.wikipedia.org/w/index.php?search=David_Bowie
<NAME> https://en.wikipedia.org/w/index.php?search=Annie_Lennox<jupyter_text>Example 2<jupyter_code>nlp2 = spacy.load('en_core_web_sm')
# Let's define a list of districts in Tel Aviv
distrs = ['Old Yafo', 'Shapira', 'Ezra', 'Florentin']

# Add patterns
patterns = list(nlp2.pipe(distrs))
# Initialize Matcher
matcher = PhraseMatcher(nlp2.vocab)

matcher.add('DISTRICT', None, *patterns)
def tlv_component(doc_):
    
    # Apply the matcher to the doc
    matches = matcher(doc_)
    
    # Create a Span for each match and assign the label 'TLV_DISTRICT'
    spans = [Span(doc_, start, end, label = 'TLV_DISTRICT')
             for match_id, start, end in matches]
    
    # Overwrite the doc.ents with the matched spans
    doc_.ents = tuple(spans)
    
    return doc_
district_loc = {
    'Old Yafo': 'Southwest',
    'Shapira': 'South',
    'Ezra': 'Southeast',
    'Florentin': 'South'
}
# Add the component to the pipeline after the 'ner' component 
nlp2.add_pipe(tlv_component, after = 'ner')
nlp2.pipeline
# Create a document
doc2 = nlp2('I stayed in Old Yafo for a couple of days and then moved to Shapira to visit my friends. They told me\
that in their opinion Ezra is nicer than Florentin. I disagreed.')
# Register district_loc and getter that looks up the span text in TLV districts
Span.set_extension('district_location', getter = lambda span: district_loc[span.text], force = True)
for ent in doc2.ents:
    print(f'DISTRICT: {ent.text:10} | LABEL: {ent.label_:10} | LOCATION: {ent._.district_location}')<jupyter_output>DISTRICT: Old Yafo   | LABEL: TLV_DISTRICT | LOCATION: Southwest
DISTRICT: Shapira    | LABEL: TLV_DISTRICT | LOCATION: South
DISTRICT: Ezra       | LABEL: TLV_DISTRICT | LOCATION: Southeast
DISTRICT: Florentin  | LABEL: TLV_DISTRICT | LOCATION: South
{'fraction_non_alphanumeric': 0.11801492838410328, 'fraction_numerical': 0.005648577768811782, 'mean_word_length': 4.959134615384615, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '154705', 'license': 'MIT', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1624, 'n_words': 512, 'path': 'SpaCy/02_Spacy_custom_extensions.ipynb', 'repo_name': 'AlxndrMlk/NLP', 'size': '4979'}