<jupyter_start><jupyter_text>Practical 3 Read text and perform basic preprocessing techniques on it<jupyter_code># Word tokenization
import spacy
from spacy.lang.en import English
# Load English tokenizer, tagger, parser, NER and word vectors
nlp = English()
text = '''When learning data science, you shouldn't get discouraged.
Challenges and setbacks aren't failures, they're just a part of the journey. You've got this!'''

# nlp object is used to create documents with linguistic annotations
doc = nlp(text)
print(doc)
# Create a list of word tokens
import nltk
word_tokens = nltk.word_tokenize(doc.text) # does not give '\n' like the other two methods
print(word_tokens)
# or:
token_list = []
for token in doc:
    token_list.append(token.text)
print(token_list)
# or:
word_list = [word.text for word in doc]
print(word_list)
# Sentence tokenization:
# Load English tokenizer, tagger, parser, NER and word vectors
nlp = English()

# Create the pipeline sentencizer compeonent
#sbd = nlp.create_pipe('sentencizer')
#print(sbd)

# Add the component to the pipeline
nlp.add_pipe('sentencizer')

text = '''When learning data science, you shouldn't get discouraged.
Challenges and setbacks aren't failures, they're just a part of the journey. You've got this!'''

# nlp object is used to create documents with linguistic annotations
doc = nlp(text)

# Create a list of sentence tokens
sentence_list = []
for sentence in doc.sents:
    sentence_list.append(sentence.text)
print(sentence_list)
# Stopwords are those words that do not contribute to the processing of the language
spacy_stopwords = spacy.lang.en.stop_words.STOP_WORDS
print(spacy_stopwords)
print('when' in spacy_stopwords)
# Filter the word tokens such that there are no stop words
list_without_stopwords = []
for word in token_list:
    wordx = word.lower() # if we don't do this, When will be there, but it is a stopword, so turn into lower case
    if wordx not in spacy_stopwords:
        list_without_stopwords.append(word)
print(list_without_stopwords)
# or:
from spacy.lang.en.stop_words import STOP_WORDS

filtered_sent = []
doc = nlp(text)
for word in doc:
    if word.is_stop == False:
        filtered_sent.append(word)
print(filtered_sent)
# Implementing Lemmatization
lem = nlp('run runner running runs')

# Finding lemma for each word:
for word in lem:
    print(word.text, word.lemma_) # word.lemma_ not printing ?!<jupyter_output>run 
runner 
running 
runs
{'fraction_non_alphanumeric': 0.08254620123203285, 'fraction_numerical': 0.0004106776180698152, 'mean_word_length': 4.9559902200489, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '314357', 'license': 'MIT', 'n_tokens_mistral': 781, 'n_tokens_neox': 754, 'n_words': 286, 'path': 'NLP_with_spacy.ipynb', 'repo_name': 'reeha-parkar/natural-language-processing', 'size': '2435'}