<jupyter_start><jupyter_text>Author : <NAME>Date   : 10/July/2020Purpose: Basics of Spacy<jupyter_code>from spacy.lang.en import English
nlp = English()

doc = nlp("Hello World!")
for token in doc:
  print(token.text + ":", token.is_alpha)
#Slicing of text
#Process the text
doc = nlp("I like tree kangaroos and narwhals.")
first_token = doc[0]
print(first_token.text)
# Process the text
doc = nlp("I like tree kangaroos and narwhals.")

# A slice of the Doc for "tree kangaroos"
tree_kangaroos = doc[2:4]
print(tree_kangaroos.text)

# A slice of the Doc for "tree kangaroos and narwhals" (without the ".")
tree_kangaroos_and_narwhals = doc[2:6]
print(tree_kangaroos_and_narwhals.text)
# Process the text
doc = nlp(
    "In 1990, more than 60% of people in East Asia were in extreme poverty. "
    "Now less than 4% are."
)

# Iterate over the tokens in the doc
for token in doc:
    # Check if the token resembles a number
    if token.like_num:
        # Get the next token in the document
        next_token = doc[token.i + 1]
        # Check if the next token's text equals "%"
        if next_token.text == "%":
            print("Percentage found:", token.text)
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("She ate the pizza at Apple")
print("1.Printing all tokens")
for token in doc:
  print(token.text,token.pos_)

print("2.Print all entity")
for token in doc.ents:
  print(token.text, token.label_)


import spacy

nlp = spacy.load("en_core_web_sm")

text = "It’s official: Apple is the first U.S. public company to reach a $1 trillion market value"

# Process the text
doc = nlp(text)

for token in doc:
    # Get the token text, part-of-speech tag and dependency label
    token_text = token.text
    token_pos = token.pos_
    token_dep = token.dep_
    # This is for formatting only
    print(f"{token_text:<12}{token_pos:<10}{token_dep:<10}")
import spacy

nlp = spacy.load("en_core_web_sm")

text = "It’s official: Apple is the first U.S. public company to reach a $1 trillion market value"

# Process the text
doc = nlp(text)

# Iterate over the predicted entities
for ent in doc.ents:
    # Print the entity text and its label
    print(ent.text, ent.label_)
# A case of not recognizing a new entity which is not part of the training data-set
import spacy

nlp = spacy.load("en_core_web_sm")

text = "Upcoming iPhone X release date leaked as Apple reveals pre-orders"

# Process the text
doc = nlp(text)

# Iterate over the entities
for token in doc.ents:
    # Print the entity text and label
    print(token.text, token.label_)

# Get the span for "iPhone X"
iphone_x = doc[1:3]

# Print the span text
print("Missing entity:", iphone_x.text)
#Pattern matching for "iPhone X"
import spacy
from spacy.matcher import Matcher
nlp = spacy.load("en_core_web_sm")
matcher = Matcher(nlp.vocab)
pattern = [{"TEXT":"iPhone"},{"TEXT":"X"}]
matcher.add("IPH_MATCHER", None, pattern)
doc = nlp("Upcoming iPhone X release date leaked")
matches = matcher(doc)
print(matches)
for match_id,start,end in matches:
  matched_span = doc[start:end]
  print(matched_span.text)
#Pattern Matching for "iOS 7 or iOS 10 or iOS 11"
import spacy
from spacy.matcher import Matcher

nlp = spacy.load("en_core_web_sm")
matcher = Matcher(nlp.vocab)

doc = nlp(
    "After making the iOS update you won't notice a radical system-wide "
    "redesign: nothing like the aesthetic upheaval we got with iOS 7. Most of "
    "iOS 11's furniture remains the same as in iOS 10. But you will discover "
    "some tweaks once you delve a little deeper."
)

# Write a pattern for full iOS versions ("iOS 7", "iOS 11", "iOS 10")
pattern = [{"TEXT": "iOS"}, {"IS_DIGIT": True}]

# Add the pattern to the matcher and apply the matcher to the doc
matcher.add("IOS_VERSION_PATTERN", None, pattern)
matches = matcher(doc)
print("Total matches found:", len(matches))

# Iterate over the matches and print the span text
for match_id, start, end in matches:
    print("Match found:", doc[start:end].text)
#Pattern match for Pronoun version of the lemma download
import spacy
from spacy.matcher import Matcher

nlp = spacy.load("en_core_web_sm")
matcher = Matcher(nlp.vocab)

doc = nlp(
    "i downloaded Fortnite on my laptop and can't open the game at all. Help? "
    "so when I was downloading Minecraft, I got the Windows version where it "
    "is the '.zip' folder and I used the default program to unpack it... do "
    "I also need to download Winzip?"
)

# Write a pattern that matches a form of "download" plus proper noun
pattern = [{"LEMMA": "download"}, {"POS": "PROPN"}]

# Add the pattern to the matcher and apply the matcher to the doc
matcher.add("DOWNLOAD_THINGS_PATTERN", None, pattern)
matches = matcher(doc)
print("Total matches found:", len(matches))

# Iterate over the matches and print the span text
for match_id, start, end in matches:
    print("Match found:", doc[start:end].text)
#Pattern match for ADJ + Noun + Optional Noun
import spacy
from spacy.matcher import Matcher

nlp = spacy.load("en_core_web_sm")
matcher = Matcher(nlp.vocab)

doc = nlp(
    "Features of the app include a beautiful design, smart search, automatic "
    "labels and optional voice responses."
)

# Write a pattern for adjective plus one or two nouns
pattern = [{"POS": "ADJ"}, {"POS": "NOUN"}, {"POS": "NOUN", "OP": "+"}]

# Add the pattern to the matcher and apply the matcher to the doc
matcher.add("ADJ_NOUN_PATTERN", None, pattern)
matches = matcher(doc)
print("Total matches found:", len(matches))

# Iterate over the matches and print the span text
for match_id, start, end in matches:
    print("Match found:", doc[start:end].text)
#Convert between hash and strings
import spacy
from spacy.lang.en import English
nlp = English()
doc = nlp("I have a cat")


cat_hash = nlp.vocab.strings[doc[3].text]
print(cat_hash)

cat_string = nlp.vocab.strings[cat_hash]
print(cat_string)

# Doc object creation 
from spacy.lang.en import English

nlp = English()

# Import the Doc class
from spacy.tokens import Doc

# Desired text: "Oh, really?!"
words = ["Oh","," ,"really", "?", "!"]
spaces = [False, True, False, False, False]

# Create a Doc from the words and spaces
doc = Doc(nlp.vocab, words=words, spaces=spaces)
print(doc.text)
from spacy.lang.en import English

nlp = English()

# Import the Doc and Span classes
from spacy.tokens import Doc, Span

words = ["I", "like", "David", "Bowie"]
spaces = [True, True, True, False]

# Create a doc from the words and spaces
doc = Doc(nlp.vocab, words=words, spaces=spaces)
print(doc.text)

# Create a span for "<NAME>" from the doc and assign it the label "PERSON"
span = Span(doc, 2, 4, label="PERSON")
print(span.text, span.label_)

# Add the span to the doc's entities
doc.ents = [span]

# Print entities' text and labels
print([(ent.text, ent.label_) for ent in doc.ents])
# Use token attributes only for better performance
# spacy converts every token into hash, better to use the hash to do most operation
import spacy
from spacy.lang.en import English

#Without the model the below code will NOT identify parts-of-speec
nlp = spacy.load("en_core_web_sm")

doc = nlp("Berlin run")
print(doc)
for  token in doc:
  print(token.text,token.pos_)
  if(token.pos_ == "PROPN"):
    next_token = doc[token.i + 1]
    print("Found a proper noun",token.text)
    if(next_token.pos_ == "VERB"):
      print("Found a proper noun:", token.text, "followed by a verb:",next_token.text)
      
#Inspect word vectors, the internals of spacy
!python -m spacy download en_core_web_lg
import spacy
from spacy.lang.en import English

nlp = spacy.load("en_core_web_lg")
doc = nlp("Bananna has high pottasium content.")
print(doc[0].vector)

#Comparing two documents similary
import spacy
from spacy.lang.en import English
nlp = spacy.load("en_core_web_lg")

doc1 = nlp("It is a warm summer day.")
doc2 = nlp("It's sunny outside.")

similarity = doc1.similarity(doc2)
print(similarity)

import spacy
from spacy.matcher import Matcher

nlp = spacy.load("en_core_web_sm")
doc = nlp(
    "Twitch Prime, the perks program for Amazon Prime members offering free "
    "loot, games and other benefits, is ditching one of its best features: "
    "ad-free viewing. According to an email sent out to Amazon Prime members "
    "today, ad-free viewing will no longer be included as a part of Twitch "
    "Prime for new members, beginning on September 14. However, members with "
    "existing annual subscriptions will be able to continue to enjoy ad-free "
    "viewing until their subscription comes up for renewal. Those with "
    "monthly subscriptions will have access to ad-free viewing until October 15."
)

# Create the match patterns
#pattern1 = [{"LOWER": "Amazon"}, {"IS_TITLE": True, "POS": "PROPN"}]
pattern1 = [{"TEXT":"Amazon"}]
pattern2 = [{"TEXT": "ad-free"}]

# Initialize the Matcher and add the patterns
matcher = Matcher(nlp.vocab)
matcher.add("PATTERN1", None, pattern1)
matcher.add("PATTERN2", None, pattern2)

# Iterate over the matches
for match_id, start, end in matcher(doc):
    # Print pattern string name and text of matched span
    print(doc.vocab.strings[match_id], doc[start:end].text)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10982532751091703, 'fraction_numerical': 0.007205240174672489, 'mean_word_length': 4.3045744064852345, 'pattern_counts': {'":': 17, '<': 10, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '239516', 'license': 'Apache-2.0', 'n_tokens_mistral': 3113, 'n_tokens_neox': 3014, 'n_words': 1133, 'path': 'NLP_101.ipynb', 'repo_name': 'sateesh12/deep_learning_basics', 'size': '9174'}