<jupyter_start><jupyter_text>Named Entity Recognition  Do necessary imports<jupyter_code>%matplotlib inline
### INSTALL GHOSTSCRIPT (Required to display NLTK trees) ###
!apt install ghostscript python3-tk

### CREATE VIRTUAL DISPLAY ###
!apt-get install -y xvfb # Install X Virtual Frame Buffer
import os
os.system('Xvfb :1 -screen 0 1600x1200x16  &')    # create virtual display with size 1600x1200 and 16 bit color. Color can be changed to 24 or 8
os.environ['DISPLAY']=':1.0'    # tell X clients to use our virtual DISPLAY :1.0.
import nltk
from nltk import word_tokenize, pos_tag
from nltk.corpus import wordnet
from nltk.corpus import stopwords   
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')
nltk.download('stopwords')

from IPython.display import display
lemmatizer = nltk.WordNetLemmatizer()
from tqdm import tqdm
import re 

import pandas as pd
import numpy as np

from nltk.chunk import conlltags2tree, tree2conlltags
from pprint import pprint
import spacy
from spacy import displacy
from collections import Counter
import en_core_web_sm
nlp = en_core_web_sm.load()
# COLAB CONFIG
# change colab flag to false if train using jupyter notebook
COLAB_FLAG = True
COLAB_FILEPATH = './drive/My Drive/4034-amazon-review-classification/' if COLAB_FLAG == True else './'
pd.options.mode.chained_assignment = None  # default='warn'
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Loading of data<jupyter_code># read the crawled data
data_ = pd.read_csv(COLAB_FILEPATH+'data/trip-advisor-comments.csv')
print(f'Shape of the dataset:{data_.shape}')
data_.head()
# filtering the test data by removing duplicates
data=data_.drop_duplicates(subset={"Reviewer\'s Name","Comment"}, 
                                             keep='first', inplace=False)
print(f'Shape of the dataset:{data.shape}')
data_comment_ = data['Comment']
data_comment = data_comment_.to_frame()
print(data_comment.shape)
data_comment.head()
# convert data into a list
data_comment_list = list(data_comment_)
data_comment_list[1]<jupyter_output><empty_output><jupyter_text>Text Preprocessing to check POS and IOB Chunking pattern consists of one rule, that a noun phrase, NP, should be formed whenever the chunker finds an optional determiner, DT, followed by any number of adjectives, JJ, and then a noun, NN.<jupyter_code>def preprocess(sent):
    sent = nltk.word_tokenize(sent)
    sent = nltk.pos_tag(sent)
    return sent

selected_list = [5,7,88,777,1000]

for i in selected_list:
    sent = preprocess(data_comment_list[i])

    pattern = 'NP: {<DT>?<JJ>*<NN>}'

    cp = nltk.RegexpParser(pattern)
    cs = cp.parse(sent)
    #print(cs)

    # NER Portion
    # look at the IOB tagged
    iob_tagged = tree2conlltags(cs)
    pprint(iob_tagged)
    print()<jupyter_output>[('Place', 'NN', 'B-NP'),
 ('has', 'VBZ', 'O'),
 ('great', 'JJ', 'B-NP'),
 ('food', 'NN', 'I-NP'),
 (',', ',', 'O'),
 ('great', 'JJ', 'B-NP'),
 ('ambience', 'NN', 'I-NP'),
 ('and', 'CC', 'O'),
 ('the', 'DT', 'B-NP'),
 ('staff', 'NN', 'I-NP'),
 ('are', 'VBP', 'O'),
 ('very', 'RB', 'O'),
 ('friendly', 'RB', 'O'),
 ('.', '.', 'O'),
 ('Shout', 'NN', 'B-NP'),
 ('out', 'IN', 'O'),
 ('to', 'TO', 'O'),
 ('Naufal', 'NNP', 'O'),
 ('and', 'CC', 'O'),
 ('Ain', 'NNP', 'O'),
 ('for', 'IN', 'O'),
 ('their', 'PRP$', 'O'),
 ('pleasant', 'JJ', 'B-NP'),
 ('service', 'NN', 'I-NP'),
 ('.', '.', 'O')]

[('Love', 'VB', 'O'),
 ('their', 'PRP$', 'O'),
 ('risotto', 'NN', 'B-NP'),
 ('.', '.', 'O'),
 ('This', 'DT', 'O'),
 ('is', 'VBZ', 'O'),
 ('the', 'DT', 'B-NP'),
 ('third', 'JJ', 'I-NP'),
 ('time', 'NN', 'I-NP'),
 ('coming', 'VBG', 'O'),
 ('here', 'RB', 'O'),
 ('.', '.', 'O'),
 ('It', 'PRP', 'O'),
 ('wo', 'MD', 'O'),
 ("n't", 'RB', 'O'),
 ('be', 'VB', 'O'),
 ('the', 'DT', 'O'),
 ('last', 'JJ', 'O'),
 ('.', '.',[...]<jupyter_text>Some examples of named entity recognition<jupyter_code>doc = nlp(data_comment_list[5])
displacy.render(doc, jupyter=True, style='ent')
doc = nlp(data_comment_list[7])
displacy.render(doc, jupyter=True, style='ent')
doc = nlp(data_comment_list[88])
displacy.render(doc, jupyter=True, style='ent')
doc = nlp(data_comment_list[777])
displacy.render(doc, jupyter=True, style='ent')
doc = nlp(data_comment_list[1000])
displacy.render(doc, jupyter=True, style='ent')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20041657023837076, 'fraction_numerical': 0.01388567461235825, 'mean_word_length': 5.061711079943899, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '145327', 'license': 'MIT', 'n_tokens_mistral': 1691, 'n_tokens_neox': 1620, 'n_words': 417, 'path': '03-classification/named_entity_recognition.ipynb', 'repo_name': 'nicholasneo78/information-retrieval-4034', 'size': '4321'}