<jupyter_start><jupyter_text>Build our own statistical POS tagger form scratch         3.1 Import dependencies       3.2 Explore dataset                      3.2.1 Explore Brown Corpus                      3.2.2 Explore Penn-Treebank Corpus       3.3 Generate features              3.4 Transform Dataset       3.5 Build training and testing dataset                     3.6 Train model       3.7 Measure Accuracy       3.8 Generate POS tags for given sentence __Assumption__: * We are considering the __tagged dataset / corpus as input__ and apply __supervised Machine Learning algorithm.__ __Fact__: * This is __multi-class classification__ task.  3.1 Import dependencies<jupyter_code>import nltk
from nltk import word_tokenize
from nltk.corpus import brown as cb
from nltk.corpus import treebank as tb
import pprint
from sklearn.tree import DecisionTreeClassifier
from sklearn.feature_extraction import DictVectorizer
from sklearn.pipeline import Pipeline<jupyter_output><empty_output><jupyter_text>3.2 Explore dataset  3.2.1 Explore Brown Corpus<jupyter_code>raw_text = nltk.Text(cb.words('ca01'))
print (raw_text)
print (cb.words()[0:20])
print (cb.tagged_words()[0:10])
tagged_sentences_brown_corpus = nltk.corpus.brown.tagged_sents()
pprint.pprint(tagged_sentences_brown_corpus[0])<jupyter_output>[('The', 'AT'),
 ('Fulton', 'NP-TL'),
 ('County', 'NN-TL'),
 ('Grand', 'JJ-TL'),
 ('Jury', 'NN-TL'),
 ('said', 'VBD'),
 ('Friday', 'NR'),
 ('an', 'AT'),
 ('investigation', 'NN'),
 ('of', 'IN'),
 ("Atlanta's", 'NP$'),
 ('recent', 'JJ'),
 ('primary', 'NN'),
 ('election', 'NN'),
 ('produced', 'VBD'),
 ('``', '``'),
 ('no', 'AT'),
 ('evidence', 'NN'),
 ("''", "''"),
 ('that', 'CS'),
 ('any', 'DTI'),
 ('irregularities', 'NNS'),
 ('took', 'VBD'),
 ('place', 'NN'),
 ('.', '.')]<jupyter_text>3.2.2 Explore Penn-Treebank Corpus<jupyter_code>raw_text = nltk.Text(tb.words()[0:10])
print (raw_text)
print (tb.words()[0:10])
tagged_sentences_treebank_corpus = nltk.corpus.treebank.tagged_sents()
pprint.pprint (tagged_sentences_treebank_corpus[0])<jupyter_output>[('Pierre', 'NNP'),
 ('Vinken', 'NNP'),
 (',', ','),
 ('61', 'CD'),
 ('years', 'NNS'),
 ('old', 'JJ'),
 (',', ','),
 ('will', 'MD'),
 ('join', 'VB'),
 ('the', 'DT'),
 ('board', 'NN'),
 ('as', 'IN'),
 ('a', 'DT'),
 ('nonexecutive', 'JJ'),
 ('director', 'NN'),
 ('Nov.', 'NNP'),
 ('29', 'CD'),
 ('.', '.')]<jupyter_text>We will be using Treebank corpus to build our own POS tagger<jupyter_code>print ("Tagged sentences: ", len(tagged_sentences_treebank_corpus))
print ("Tagged words:", len(nltk.corpus.treebank.tagged_words()))<jupyter_output>Tagged words: 100676<jupyter_text>3.3 Generate features  Function for generating features form tagged corpus<jupyter_code>def features(sentence, index):
    # "sentence: [w1, w2, ...], index: the index of the word"
    return {
    'word': sentence[index],
    'is_first': index == 0,
    'is_last': index == len(sentence) - 1,
    'is_capitalized': sentence[index][0].upper() == sentence[index][0],
    'is_all_caps': sentence[index].upper() == sentence[index],
    'is_all_lower': sentence[index].lower() == sentence[index],
    'prefix-1': sentence[index][0],
    'prefix-2': sentence[index][:2],
    'prefix-3': sentence[index][:3],
    'suffix-1': sentence[index][-1],
    'suffix-2': sentence[index][-2:],
    'suffix-3': sentence[index][-3:],
    'prev_word': '' if index == 0 else sentence[index - 1],
    'next_word': '' if index == len(sentence) - 1 else sentence[index + 1],
    'has_hyphen': '-' in sentence[index],
    'is_numeric': sentence[index].isdigit(),
    'capitals_inside': sentence[index][1:].lower() != sentence[index][1:]
    }
pprint.pprint(features(['This', 'is', 'a', 'sentence'], 0))<jupyter_output>{'capitals_inside': False,
 'has_hyphen': False,
 'is_all_caps': False,
 'is_all_lower': False,
 'is_capitalized': True,
 'is_first': True,
 'is_last': False,
 'is_numeric': False,
 'next_word': 'is',
 'prefix-1': 'T',
 'prefix-2': 'Th',
 'prefix-3': 'Thi',
 'prev_word': '',
 'suffix-1': 's',
 'suffix-2': 'is',
 'suffix-3': 'his',
 'word': 'This'}<jupyter_text>3.4 Transform Dataset  Extract words form tagged sentences using 'untag' function<jupyter_code>def untag(tagged_sentence):
    return [w for w, t in tagged_sentence]<jupyter_output><empty_output><jupyter_text>Transform dataset into X, y pairs where X = Features Y = POS lables¶<jupyter_code>def transform_to_dataset(tagged_sentences):
    X, y = [], []
    for tagged in tagged_sentences:
        for index in range(len(tagged)):
            X.append(features(untag(tagged), index))
            y.append(tagged[index][1])
            #pprint.pprint(" original word: "+ str(tagged) + " Word: "+ str(untag(tagged))+ "Y: " + y[index])
    return X, y<jupyter_output><empty_output><jupyter_text>3.5 Build training and testing dataset<jupyter_code>cutoff = int(.75 * len(tagged_sentences_treebank_corpus))
training_sentences = tagged_sentences_treebank_corpus[:cutoff]
test_sentences = tagged_sentences_treebank_corpus[cutoff:]
print (len(training_sentences))
print (len(test_sentences))
X, y = transform_to_dataset(training_sentences)
print(len(X)) 
print(len(y))<jupyter_output>75784
75784<jupyter_text>3.6 Train model  Initialize the classifier<jupyter_code>clf = Pipeline([
    ('vectorizer', DictVectorizer(sparse=False)),
    ('classifier', DecisionTreeClassifier(criterion='entropy'))
])
# Use only the first 10K samples if you're running it multiple times. It takes a fair bit :)
clf.fit(X[:20000],y[:20000])<jupyter_output><empty_output><jupyter_text>3.7 Measure Accuracy<jupyter_code>X_test, y_test = transform_to_dataset(test_sentences)
print ("Accuracy:{:.3%}".format(clf.score(X_test, y_test)))<jupyter_output>Accuracy:91.387%<jupyter_text>3.8 Generate POS tags for given sentence<jupyter_code>def pos_tag(sentence):
    tagged_sentence = []
    tags = clf.predict([features(sentence, index) for index in range(len(sentence))])
    return zip(sentence, tags)

POS_list = list(pos_tag(word_tokenize('This is my friend, John.')))
for t in POS_list:
    print(u"{:<16}{:>2}".format(str(t[0]),str(t[1])))   
POS_list = list(pos_tag(word_tokenize("We will meet at eight o'clock on Thursday morning.")))
for t in POS_list:
    print(u"{:<16}{:>2}".format(str(t[0]),str(t[1])))   
POS_list = list(pos_tag(word_tokenize('Alexander, the great...!')))
for t in POS_list:
    print(u"{:<16}{:>2}".format(str(t[0]),str(t[1])))   
POS_list = list(pos_tag(word_tokenize('Alexander the Great, was a king of the ancient Greek kingdom of Macedon.')))
for t in POS_list:
    print(u"{:<16}{:>2}".format(str(t[0]),str(t[1])))<jupyter_output>Alexander       IN
the             DT
Great           NNP
,                ,
was             VBD
a               DT
king            NN
of              IN
the             DT
ancient         DT
Greek           NNP
kingdom         NN
of              IN
Macedon         NNP
.                .
{'fraction_non_alphanumeric': 0.17848870874348582, 'fraction_numerical': 0.022003474232773594, 'mean_word_length': 4.505179282868526, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '30453', 'license': 'MIT', 'n_tokens_mistral': 2596, 'n_tokens_neox': 2442, 'n_words': 543, 'path': 'POS-tag-workshop-master/POS_from_scratch.ipynb', 'repo_name': 'RITUait/NLP-Projects', 'size': '6908'}