<jupyter_start><jupyter_text>Tokenizing : Splitinig For Example - Word tokenizing from Sentence or Sentence tokenizing from Paragraph.<jupyter_code>from nltk.tokenize import sent_tokenize, word_tokenize
text = "Hello friends, how are you doing? python is awesome."
print(sent_tokenize(text)) # sentence tokenizer split into sentences
print(word_tokenize(text)) #word tokeizer split into words
#Removing stopwords - Useless data
from nltk.corpus import stopwords
print(set(stopwords.words('english'))) #printing stopwords
example='This is an example for showing off stopwords filteration'
stop_words = set(stopwords.words('english'))
#first tokenize the example
words_tokenize=word_tokenize(example)
filtered_sentence=[] #list to store filtered data
for w in words_tokenize:
    if w not in stop_words:
        filtered_sentence.append(w)
        
#another method using inline for loop
sentence=[w for w in words_tokenize if w not in stop_words]
print(words_tokenize)
print(sentence)
print(filtered_sentence)


#Stemming words in NLTK  
from nltk.stem import PorterStemmer
ps = PorterStemmer()

example_words= ['ride','riding','rider','rides']

for w in example_words:
    print(ps.stem(w))

#stemming in entire sentence 
text = 'When riders are riding their horses, they often think how cowboys rode horses'

words = word_tokenize(text)
for w in words:
    print(ps.stem(w))<jupyter_output>when
rider
are
ride
their
hors
,
they
often
think
how
cowboy
rode
hors<jupyter_text>Part of speech tagging - labling words as noun,pronoun,adjective,verbs etc<jupyter_code>nltk.download()<jupyter_output>showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml<jupyter_text>CORPUS : a corpus (plural corpora) or text corpus is a large and structured set of texts (nowadays usually electronically stored and processed).<jupyter_code>from nltk.corpus import udhr
print(udhr.raw('English-Latin1'))
from nltk.corpus import state_union
from nltk.tokenize import PunktSentenceTokenizer

train_text = state_union.raw('2005-GWBush.txt')
sample_text= state_union.raw('2006-GWBush.txt')
import nltk 
#nltk.download()
print(train_text)
#Now we have some text , We can train PunktSentenceTokenizer
custome_sent_tokenizer=PunktSentenceTokenizer(train_text)   
#Now tokenize the sample text
tokenized = custome_sent_tokenizer.tokenize(sample_text)
print(tokenized)
print(custome_sent_tokenizer)
#define a function that will tag each tokenized word with part of speech
def process_content():
    try:
        for i in tokenized[:5]:    #first five lines in tokenized
            word=nltk.word_tokenize(i)
            tagged=nltk.pos_tag(word)  #part of speech tag in each word
            print(tagged)
    except Exception as e:
        print(str(e))

process_content()
#check representation of part of speech 
nltk.help.upenn_tagset()<jupyter_output>$: dollar
    $ -$ --$ A$ C$ HK$ M$ NZ$ S$ U.S.$ US$
'': closing quotation mark
    ' ''
(: opening parenthesis
    ( [ {
): closing parenthesis
    ) ] }
,: comma
    ,
--: dash
    --
.: sentence terminator
    . ! ?
:: colon or ellipsis
    : ; ...
CC: conjunction, coordinating
    & 'n and both but either et for less minus neither nor or plus so
    therefore times v. versus vs. whether yet
CD: numeral, cardinal
    mid-1890 nine-thirty forty-two one-tenth ten million 0.5 one forty-
    seven 1987 twenty '79 zero two 78-degrees eighty-four IX '60s .025
    fifteen 271,124 dozen quintillion DM2,000 ...
DT: determiner
    all an another any both del each either every half la many much nary
    neither no some such that the them these this those
EX: existential there
    there
FW: foreign word
    gemeinschaft hund ich jeux habeas <NAME>'ang-si vous
    lutihaw alai je jour objets salutaris fille quibusdam pas trop Monte
    terram fiche oui corporis ...
IN: preposition or[...]<jupyter_text>What is chunkingShallow parsing (also chunking, "light parsing") is an analysis of a sentence which first identifies constituent parts of sentences (nouns, verbs, adjectives, etc.) and then links them to higher order units .It gives you the information about any phrase in an unstructured text..  show it will be useful that which words we can use in combination or which not.<jupyter_code>#chunking
train_text = state_union.raw('2005-GWBush.txt')
sample_text= state_union.raw('2006-GWBush.txt')

#Now we have some text , We can train PunktSentenceTokenizer
custome_sent_tokenizer=PunktSentenceTokenizer(train_text)

#Now tokenize the sample text
tokenized = custome_sent_tokenizer.tokenize(sample_text)

#define a function that will tag each tokenized word with part of speech
def process_content():
    try:
        for i in tokenized[:5]:    #first five lines in tokenized
            word=nltk.word_tokenize(i)
            tagged=nltk.pos_tag(word)  #part of speech tag in each word
            
            #combine the part of speech tag with a regular expression 
            chunkGram = r"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}"""
            chunkParser = nltk.RegexpParser(chunkGram)
            chunked = chunkParser.parse(tagged)

            #draw the chunks with nltk
            chunked.draw()
            
    except Exception as e:
        print(str(e))

process_content()
"""
<RB.?>* ="0 or more of any tense of adverb, " followed by :
<VB.?>* = "0 or more of any tense of verb," followed by
<NNP>+ ="one or more proper nouns," followed by
<NN>? ="zero or one singular noun"
"""
train_text = state_union.raw('2005-GWBush.txt')
sample_text= state_union.raw('2006-GWBush.txt')

#Now we have some text , We can train PunktSentenceTokenizer
custome_sent_tokenizer=PunktSentenceTokenizer(train_text)

#Now tokenize the sample text
tokenized = custome_sent_tokenizer.tokenize(sample_text)

#define a function that will tag each tokenized word with part of speech
def process_content():
    try:
        for i in tokenized[:5]:    #first five lines in tokenized
            word=nltk.word_tokenize(i)
            tagged=nltk.pos_tag(word)  #part of speech tag in each word
            
            #combine the part of speech tag with a regular expression 
            chunkGram = r"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}"""
            chunkParser = nltk.RegexpParser(chunkGram)
            chunked = chunkParser.parse(tagged)

            for subtree in chunked.subtrees(filter = lambda t: t.label()=="Chunk"):
                print(subtree)
            
            
    except Exception as e:
        print(str(e))

process_content()
# Chinking with nltk -- sometime we dont want words in chunking so using this method we can remove these words
#chunking
train_text = state_union.raw('2005-GWBush.txt')
sample_text= state_union.raw('2006-GWBush.txt')

#Now we have some text , We can train PunktSentenceTokenizer
custome_sent_tokenizer=PunktSentenceTokenizer(train_text)

#Now tokenize the sample text
tokenized = custome_sent_tokenizer.tokenize(sample_text)

#define a function that will tag each tokenized word with part of speech
def process_content():
    try:
        for i in tokenized[:5]:    #first five lines in tokenized
            word=nltk.word_tokenize(i)
            tagged=nltk.pos_tag(word)  #part of speech tag in each word
            
            #combine the part of speech tag with a regular expression 
            chunkGram = r"""Chunk: {<.*>+}
                                       }<VB.?|IN|DT|TO>+{"""
            chunkParser = nltk.RegexpParser(chunkGram)
            chunked = chunkParser.parse(tagged)

            #draw the chunks with nltk
            chunked.draw()
            
    except Exception as e:
        print(str(e))

process_content()<jupyter_output><empty_output><jupyter_text>Named entity recognition (NER)It is probably the first step towards information extraction that seeks to locate and classify named entities in text into pre-defined categories such as the names of persons, organizations, locations, expressions of times, quantities, monetary values, percentages, etc. NER is used in many fields in Natural Language Processing (NLP), and it can help answering many real-world questions, such as:    Which companies were mentioned in the news article?    Were specified products mentioned in complaints or reviews?    Does the tweet contain the name of a person? Does the tweet contain this person’s location?<jupyter_code>#Name Entity Recognition
def process_contant():
    try:
        for i in tokenized[:20]:
            word=nltk.word_tokenize(i)
            tagged=nltk.pos_tag(word)
            nameEnt=nltk.ne_chunk(tagged,binary=False)
            
            nameEnt.draw()
            
    except Exception as e:
        print(str(e))

   
process_contant()<jupyter_output><empty_output><jupyter_text>Text Classification<jupyter_code>import random
import nltk
from nltk.corpus import movie_reviews
nltk.download()
#build list of documents
documents=[(list(movie_reviews.words(fileid)),category)
          for category in movie_reviews.categories()                         #movie reviews are ['neg', 'pos']
          for fileid in movie_reviews.fileids(category)]
#shuffle the documents
random.shuffle(documents)

print('The number of documents: {}'.format(len(documents)))
print('first review :{}'.format(documents[0]))

all_words=[]
for w in movie_reviews.words():
    all_words.append(w.lower())

all_words = nltk.FreqDist(all_words)

print('Most common words: {}'.format(all_words.most_common(15)))
print('The word happy : {}'.format(all_words["happy"]))
print(len(all_words)) #length of all words
#we'll use the 4000 most common words as features
words_feature = list(all_words.keys())[:4000]

#Build a find_feature function that will find these top 4,000 words in our positive and negative documents, marking their presence as either positive or negative:
def find_features(document):
    words =set(document)
    features={}
    
    for w in words_feature:         #In list of 4000 most common words
        features[w] = (w in words)  
    return features

#lets use an example from a negative review 
features =find_features(movie_reviews.words('neg/cv000_29416.txt'))
for key,value in features.items():
    if value == True:
        print(key)
print(features)
#now do it for all the documents
featuresets= [(find_features(rev),category) for (rev,category) in documents]
#we can split the features into trainning and testing dataset using sklearn
from sklearn import model_selection

#define a seed for reprodusibility
seed = 1

#split the dataset into trainning and testing datasets
trainning,testing = model_selection.train_test_split(featuresets,test_size=0.25,random_state = seed)
print(len(trainning))
print(len(testing))
#now we use sklearn algorithm in nltk 
from nltk.classify.scikitlearn import SklearnClassifier
from sklearn.svm import SVC
model = SklearnClassifier(SVC(kernel='linear'))
#train the model on trainning data 
model.train(trainning)
#test the testing dataset
accuracy = nltk.classify.accuracy(model,testing)
print('SVM accuracy. {}'.format(accuracy))<jupyter_output>SVM accuracy. 0.786
{'fraction_non_alphanumeric': 0.08811290030758097, 'fraction_numerical': 0.009498823955129364, 'mean_word_length': 3.8275109170305677, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '612993', 'license': 'Apache-2.0', 'n_tokens_mistral': 3497, 'n_tokens_neox': 3340, 'n_words': 1109, 'path': 'NaturalLanguageProcessing.ipynb', 'repo_name': 'sachinsaini4278/javascript-quiz', 'size': '11066'}