<jupyter_start><jupyter_text>Stemming<jupyter_code>from nltk.stem import PorterStemmer
tokenized_sentences = sent_tokenize(text)
porter_stemmer = PorterStemmer()

for i in range(len(tokenized_sentences)):
    tokenized_words = word_tokenize(tokenized_sentences[i])
    stem_words = [porter_stemmer.stem(word) for word in tokenized_words]
    tokenized_sentences[i] = ' '.join(stem_words)
print(tokenized_sentences)<jupyter_output>['If I could be a superhero , just for the day , I would want to be supergirl , in everi way .', 'she ’ s the young cousin of superman with long golden lock , but don ’ t let that fool you becaus she ’ s tougher than rock .', 'her power consist of fli with speed , To the moon , around the world or wherev the need .', 'she can hear a pin drop or the beat of a human ’ s heart , not to mention the faintest whisper , oh how veri smart !', 'In addit to mind control , supergirl ’ s vision is x-ray she also ha eye that gener heat without delay .', 'just like her cousin , she ha her weak too , kryptonit , oh kryptonit .', 'there ’ s onli one and not two .', 'So whi would I want to be thi superhero for the day ?', 'well , that ’ s easi , I will tell you .', 'So listen to what I say .', 'bulli ha becom a major problem everywher we turn .', 'our teacher discuss the issu , but there is more that we need to learn .', 'throughout the school hall and at the lunchroom tabl , student are teas or push ,[...]<jupyter_text>Lemmatization<jupyter_code>from nltk.stem import WordNetLemmatizer
tokenized_sentences = sent_tokenize(text)
wordnet_lemmatizer = WordNetLemmatizer()

for i in range(len(tokenized_sentences)):
    tokenized_words = word_tokenize(tokenized_sentences[i])
    lemmatized_words = [wordnet_lemmatizer.lemmatize(word) for word in tokenized_words]
    tokenized_sentences[i] = ' '.join(lemmatized_words)
print(tokenized_sentences)<jupyter_output>['If I could be a superhero , just for the day , I would want to be Supergirl , in every way .', 'She ’ s the young cousin of Superman with long golden lock , But don ’ t let that fool you because she ’ s tougher than rock .', 'Her power consist of flying with speed , To the moon , around the world or wherever the need .', 'She can hear a pin drop or the beat of a human ’ s heart , Not to mention the faintest whisper , oh how very smart !', 'In addition to mind control , Supergirl ’ s vision is x-ray She also ha eye that generate heat without delay .', 'Just like her cousin , she ha her weakness too , Kryptonite , oh Kryptonite .', 'There ’ s only one and not two .', 'So why would I want to be this superhero for the day ?', 'Well , that ’ s easy , I will tell you .', 'So listen to what I say .', 'Bullying ha become a major problem everywhere we turn .', 'Our teacher discus the issue , but there is more that we need to learn .', 'Throughout the school hall and at the lunchroom table , S[...]<jupyter_text>Stop Words<jupyter_code>from nltk.corpus import stopwords
tokenized_sentences = sent_tokenize(text)

for i in range(len(tokenized_sentences)):
    tokenized_words = word_tokenize(tokenized_sentences[i])
    stop_words = [word for word in tokenized_words if word not in stopwords.words('english')]
    tokenized_sentences[i] = ' '.join(stop_words)
print(tokenized_sentences)<jupyter_output>['If I could superhero , day , I would want Supergirl , every way .', 'She ’ young cousin Superman long golden locks , But ’ let fool ’ tougher rocks .', 'Her powers consist flying speed , To moon , around world wherever need .', 'She hear pin drop beat human ’ heart , Not mention faintest whisper , oh smart !', 'In addition mind control , Supergirl ’ vision x-ray She also eyes generate heat without delay .', 'Just like cousin , weakness , Kryptonite , oh Kryptonite .', 'There ’ one two .', 'So would I want superhero day ?', 'Well , ’ easy , I tell .', 'So listen I say .', 'Bullying become major problem everywhere turn .', 'Our teachers discuss issue , need learn .', 'Throughout school halls lunchroom tables , Students teased pushed , fighting back , well ’ able .', 'As Supergirl I would stop nonsense , By using powers aid every victim ’ defense .', 'Throughout day , I would listen negative chatter And change bully ’ insults words matter .', 'Before first punch thrown foot trips anothe[...]<jupyter_text>Part of Speech Tagging<jupyter_code>words = word_tokenize(text)
tagged_words = nltk.pos_tag(words)
tagged_words
word_tags = []

for word in tagged_words:
    word_tags.append(word[0] + "_" + word[1])
tagged_text = ' '.join(word_tags)
print(tagged_text)<jupyter_output>If_IN I_PRP could_MD be_VB a_DT superhero_NN ,_, just_RB for_IN the_DT day_NN ,_, I_PRP would_MD want_VB to_TO be_VB Supergirl_NNP ,_, in_IN every_DT way_NN ._. She_PRP ’_VBD s_PRP the_DT young_JJ cousin_NN of_IN Superman_NNP with_IN long_JJ golden_JJ locks_NNS ,_, But_CC don_VBZ ’_JJ t_NNS let_VBP that_IN fool_NN you_PRP because_IN she_PRP ’_VBZ s_RB tougher_JJR than_IN rocks_NNS ._. Her_PRP$ powers_NNS consist_VBP of_IN flying_VBG with_IN speed_NN ,_, To_TO the_DT moon_NN ,_, around_IN the_DT world_NN or_CC wherever_VB the_DT need_NN ._. She_PRP can_MD hear_VB a_DT pin_JJ drop_NN or_CC the_DT beat_NN of_IN a_DT human_JJ ’_NN s_JJ heart_NN ,_, Not_RB to_TO mention_VB the_DT faintest_JJS whisper_NN ,_, oh_UH how_WRB very_RB smart_JJ !_. In_IN addition_NN to_TO mind_VB control_NN ,_, Supergirl_NNP ’_NNP s_JJ vision_NN is_VBZ x-ray_JJ She_PRP also_RB has_VBZ eyes_NNS that_IN generate_VBP heat_NN without_IN delay_NN ._. Just_NNP like_IN her_PRP$ cousin_NN ,_, she_PRP has_VBZ her_PRP$ weak[...]<jupyter_text>Named Entity Recognition<jupyter_code>tokenized_words = word_tokenize(text)
from nltk import pos_tag
tagged_words = pos_tag(tokenized_words)
tagged_words
from nltk import ne_chunk
named_entity = ne_chunk(tagged_words)
#named_entity.draw()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08109028960817717, 'fraction_numerical': 0.00034071550255536625, 'mean_word_length': 4.871, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '277091', 'license': 'MIT', 'n_tokens_mistral': 1930, 'n_tokens_neox': 1861, 'n_words': 726, 'path': 'NLP-Core/NLP-Basics.ipynb', 'repo_name': 'anujdutt9/Natural-Language-Processing', 'size': '5870'}