<jupyter_start><jupyter_text>This file needs to be run throughout for every airline  Importing project dependencies<jupyter_code>!python -m spacy download en_core_web_sm
import json
import spacy
import pandas as pd
import nltk
import glob
from nltk.sentiment.vader import SentimentIntensityAnalyzer
nltk.download('vader_lexicon')
sid = SentimentIntensityAnalyzer()
from stanfordcorenlp import StanfordCoreNLP
from pycorenlp import StanfordCoreNLP
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
stopWords = list(stopwords.words('english'))
newStop = ['was', 'and', 'the', 'to', 'in', 'of', 'a', 'an', 'is', 'were', 'for', 'with', 'are', 'one', 'our', 'gave', 'have',
           'me', 'an', 'i', 'or', 'had', 'did', 'get', 'made', 'take', 'given', 'told', 'let', 'us']
stopWords = stopWords+newStop<jupyter_output>[nltk_data] Downloading package stopwords to C:\Users\Kanishk
[nltk_data]     Verma\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>Using Stanford's CoreNLP Dependency Parsing and other Parsers<jupyter_code>dep_parser = nltk.parse.CoreNLPDependencyParser(url='http://localhost:9000')
nnlp = StanfordCoreNLP('http://localhost:9000')
lbls = {'Entertainment':'e', 'Bookings':'o', 'Cabin':'c',
        'Food':'f', 'In-flight service':'i', 
          'Off-flight service':'o', 'Possession':'p', 
        'Seat':'s', 'Staff':'st'}<jupyter_output><empty_output><jupyter_text>Specify Path and Load Files<jupyter_code>#Change for both airlines
path_json1_ta = r'C:\Users\<NAME>\Desktop\Practicum\preprocess\ToAnnotate\TripAdvisorAnnotated\json1'
path_meta_ta = r'C:\Users\<NAME>\Desktop\Practicum\preprocess\ToAnnotate\TripAdvisorAnnotated\metadata'
file_json1 = glob.glob(path_json1_ta+r'\*.json1')
file_meta = glob.glob(path_meta_ta+r'\*_a.json')
len(file_json1), len(file_meta)
def loadFiles(file):
    '''
    This function load's the metadata and annotated file generated after labelling
    '''
    infoAnn = []
    infoMeta = []
    for i in range(len(file)):
        if (file[i].endswith('.json1')):
            jsonDic = {}
            airline = file[i].split('\\')[-1].split(".")[0].split('Annotation')[0].lower() 
            info = []
            for line in open(file[i], encoding='utf-8'):
                info.append(json.loads(line))
            jsonDic['Airline'] = airline
            jsonDic['Info'] = info
            print("Annotation File Loaded for Airline: "+airline)
            infoAnn.append(jsonDic)
        else:
            jsonDic = {}
            airline = file[i].split('\\')[-1].split(".")[0].split('_a')[0].lower()
            info = []
            for line in open(file[i], encoding='utf-8'):
                info.append(json.loads(line))
            jsonDic['Airline'] = airline
            jsonDic['Info'] = info
            print("Metadata File Loaded for Airline: "+airline)
            infoMeta.append(jsonDic)
    if (infoAnn != [] and infoMeta == []):
        return infoAnn
    elif (infoMeta != [] and infoAnn == []):
        return infoMeta<jupyter_output><empty_output><jupyter_text>Loading Annotation Files<jupyter_code>infoAnn = loadFiles(file_json1)<jupyter_output>Annotation File Loaded for Airline: arabia
Annotation File Loaded for Airline: asia
Annotation File Loaded for Airline: british
Annotation File Loaded for Airline: canada
Annotation File Loaded for Airline: emirates
Annotation File Loaded for Airline: france
Annotation File Loaded for Airline: india
Annotation File Loaded for Airline: united
Annotation File Loaded for Airline: virginatlantic
Annotation File Loaded for Airline: virginaus
Annotation File Loaded for Airline: vistara<jupyter_text>Loading Metadata files<jupyter_code>infoMeta = loadFiles(file_meta)<jupyter_output>Metadata File Loaded for Airline: arabia
Metadata File Loaded for Airline: asia
Metadata File Loaded for Airline: british
Metadata File Loaded for Airline: canada
Metadata File Loaded for Airline: emirates
Metadata File Loaded for Airline: france
Metadata File Loaded for Airline: india
Metadata File Loaded for Airline: united
Metadata File Loaded for Airline: virginatlantic
Metadata File Loaded for Airline: virginaus
Metadata File Loaded for Airline: vistara<jupyter_text>Getting Airline Names<jupyter_code>airlineNames = []
for i in range(len(infoAnn)):
    airlineNames.append(infoAnn[i]['Airline'])
len(airlineNames)
airlineNames<jupyter_output><empty_output><jupyter_text>Getting number of reviews in each airline<jupyter_code>sums = 0
for i in range(len(infoAnn)):
    for airline in airlineNames:
        if (infoAnn[i]['Airline'] == airline):
            ann = infoAnn[i]['Info']
            c = 0
            for j in range(len(ann)):
                if (ann[j]['labels'] != []):
                    c += 1
            print("Airline: "+airline+" has "+str(c)+" reviews")
            sums = sums+c
print("Total Review:"+str(sums))<jupyter_output>Airline: arabia has 96 reviews
Airline: asia has 99 reviews
Airline: british has 101 reviews
Airline: canada has 99 reviews
Airline: emirates has 100 reviews
Airline: france has 99 reviews
Airline: india has 19 reviews
Airline: united has 59 reviews
Airline: virginatlantic has 82 reviews
Airline: virginaus has 78 reviews
Airline: vistara has 68 reviews
Total Review:900<jupyter_text>Gathering detailed information airline-wise<jupyter_code>for i in range(len(infoAnn)):
    for j in range(len(infoMeta)):
        if (infoAnn[i]['Airline']=='vistara'): # Change Airline Name here
            ann = infoAnn[i]['Info']
        if (infoMeta[j]['Airline']=='vistara'): # Change Airline Name here
            meta = infoMeta[j]['Info'][0]
len(ann), len(meta)
annNew = []
for i in range(len(ann)):
    if (ann[i-1]['text'] != ann[i]['text']):
        if(ann[i]['labels'] != []):
            annNew.append(ann[i])
len(annNew)
metaNew = []
for i in range(len(meta)):
    for j in range(len(annNew)):
        if(meta[i]['id'] == annNew[j]['id']):
            metaNew.append(meta[i])
len(metaNew)<jupyter_output><empty_output><jupyter_text>The detailed information generated by Doccano<jupyter_code>annNew[1]
metaNew[1]
labels = ['Entertainment', 'Bookings', 'Cabin', 'Food',
'In-flight service', 'Off-flight service', 
'Positive Opinion', 'Negative Opinion', 'Neutral Opinion',
'Possession', 'Seat','Staff']
nlp = spacy.load("en_core_web_sm")
def getDetails(ann, meta):
    '''
    This function captures details like id, text, sentences and labels for each review
    '''
    finLs = []
    for i in range(len(ann)):
        for j in range(len(meta)):
            nDic = {}
            if ann[i]['id'] == meta[j]['id']:
                ids = meta[j]['id']
                txt = ann[i]['text']
                nDic['id'] = ids
                nDic['text'] = txt
                doc = nlp(txt)
                sentence = []
                for sent in doc.sents:
                    sentence.append(sent)
                nDic['sentence'] = sentence
                ls = []
                for k, v in meta[j].items():
                    lb = []
                    dic = {}
                    if k in labels and v != []:
                        for l in range(len(v)):
                            lb.append(v[l])
                        dic[k] = lb
                        ls.append(dic)
                nDic['Labels'] = ls
                finLs.append(nDic)
    return finLs
finLs = getDetails(annNew, metaNew)
len(finLs)
def getVal(sentTkn, lab, ab):
    '''
    This function captures dependency parsing in the form of governor (head)  and dependent word relation
    '''
    
    ls = []
    for i in range(len(sentTkn)):
        for j in range(len(lab)):
            for k, v in lab[j].items():
                for l in range(len(v)):
                    lbtkn = v[l].split()
                    for m in range(len(lbtkn)):
                        if (lbtkn[m] == sentTkn[i]):
                            for o in range(len(ab)):
                                for lbl, val in lbls.items():
                                    if (k == lbl):
                                        if (ab[o]['governorGloss'] == lbtkn[m] and ab[o]['dependentGloss'] not in stopWords):
                                            if (i == 0):
                                                a = (val, lbtkn[m], ab[o]['dependentGloss'], ab[o]['dep'])
                                            elif (i == len(sentTkn)-1):
                                                a = (val, lbtkn[m], ab[o]['dependentGloss'], ab[o]['dep'])
                                            else:
                                                a = (val, lbtkn[m], ab[o]['dependentGloss'], ab[o]['dep'])
                                        elif (ab[o]['dependentGloss'] == lbtkn[m] and ab[o]['governorGloss'] not in stopWords and ab[o]['governorGloss'] != 'ROOT'):
                                            if (i == 0):
                                                a = (val, ab[o]['governorGloss'], lbtkn[m], ab[o]['dep'])
                                            elif (i == len(sentTkn)-1):
                                                a = (val, ab[o]['governorGloss'], lbtkn[m], ab[o]['dep'])
                                            else:
                                                a = (val, ab[o]['governorGloss'], lbtkn[m], ab[o]['dep'])
                                            ls.append(a)
    return ls
def getDep(finLs):
    '''
    Ths function captures Part-of-speech tag, dependency realtion and dependency tag of all labelled words in a review
    '''
    nnLs = []
    for i in range(len(finLs)):
        d = finLs[i]
        sents = d['sentence']
        lab = d['Labels']
        dic = {}
        dic['id'] = d['id']
        dic['sentence'] = sents
        dic['labels'] = lab
        nls = []
        print("Running for Review: "+str(i))
        for j in range(len(sents)):
            try:
                sents[j] = str(sents[j])
                dictval = {}
                sentTkn = str(sents[j]).split()
                output = nnlp.annotate(str(sents[j]),
                              properties={
                      'annotators': 'tokenize,ssplit,pos,depparse,parse',
                      'outputFormat': 'json'
                      })
        
                dictval[j] = getVal(sentTkn, lab, ab)
                nls.append(dictval)
            except:
                sents[j] = str(sents[j])
                dictval = {}
                s = str(sents[j]).split()
                ns = []
                for k in range(len(s)):
                    if (k%5 == 0 and (k<len(s)-1)):
                        if (len(s)-k == 4):
                            ns.append(s[k]+' '+s[k+1]+' '+s[k+2]+' '+s[k+3])#+' '+s[k+4])
                        elif (len(s)-k == 3):
                            ns.append(s[k]+' '+s[k+1]+' '+s[k+2])#+' '+s[k+3])
                        elif (len(s)-k == 2):
                            ns.append(s[k]+' '+s[k+1])
                        elif (len(s)-k == 1):
                            ns.append(s[k])
                        
                for l in range(len(ns)):
                    sentTkn = ns[l].split()
                    output = nnlp.annotate(str(sents[j]),
                              properties={
                      'annotators': 'tokenize,ssplit,pos,depparse,parse',
                      'outputFormat': 'json'
                      })
                    if (isinstance(output, str) == False):
                        ab = output['sentences'][0]['enhancedDependencies']
                        dictval[j] = getVal(sentTkn, lab, ab)
                        nls.append(dictval)
                    else:
                        continue;
        dic['Values'] = nls
        nnLs.append(dic)
    return nnLs<jupyter_output><empty_output><jupyter_text>Processing dependency and pos tags for all review<jupyter_code>nnLs = getDep(finLs)
len(nnLs)
nnLs[0]
lsPosDep = []
for i in range(len(nnLs)):
    dicPos = {}
    ids = nnLs[i]['id']
    lab = nnLs[i]['labels']
    vals = nnLs[i]['Values']
    sent = nnLs[i]['sentence']
    dicPos['id'] = ids
    dicPos['sentence'] = sent
    dicPos['labels'] = lab
    tmpls = []
    
    for j in range(len(vals)):
        for k,v in vals[j].items():
            for l in range(len(sent)):
                if (v!=[] and k==l):
                    par, = dep_parser.raw_parse(sent[l])
                    abx = par.nodes
                    for o in range(len(lab)):
                        for p,q in lab[o].items():
                            for s in range(len(q)):
                                lbtkn = q[s].split()
                            
                                for r in range(len(lbtkn)):
                                    for m in range(len(v)):
                                        
                                        tempdic = {}
                                        for key, value in abx.items():
                                            abxls = []
                                            if(v[m][1] == lbtkn[r] and lbtkn[r] == abx[key]['word']):
                                            
                                                rg = ((v[m][1], abx[key]['tag']), v[m][2], v[m][-1], v[m][0])
                                                #abxls.append(rg)
                                            elif(v[m][2] == lbtkn[r] and lbtkn[r] == abx[key]['word']):
                                            
                                                rg = (v[m][1], (v[m][2], abx[key]['tag']), v[m][-1], v[m][0])
                                            
                                            abxls.append(rg)
                                        tempdic[l] = list(set(abxls))
                                        tmpls.append(tempdic)
    cmn = []
    td = {}
    for i in range(len(tmpls)):
        for k,v in tmpls[i].items():
            for ke, va in tmpls[i-1].items():
                if (k==ke):
                    if(v==va):
                        cmn.append(va[0])
    cmn = list(set(cmn))
    dicPos['Values'] = cmn
    
    lsPosDep.append(dicPos)
len(lsPosDep)
lsPosDep[0]<jupyter_output><empty_output><jupyter_text>Capturing Sentiment score for the words<jupyter_code>lsPosSentDep = []
for h in range(len(lsPosDep)):
    newDic = {}
    values = lsPosDep[h]['Values']
    sent = lsPosDep[h]['sentence']
    ids = lsPosDep[h]['id']
    nval = []
    print("Processing for Review:" +str(h+1))
    for i in range(len(values)):
        for j in range(len(sent)):
            par, = dep_parser.raw_parse(sent[j])
            a = par.nodes
            wTkn = nltk.word_tokenize(sent[j])
            for word in wTkn:
                for k,v in a.items():
                    if (isinstance(values[i][0], tuple) == True):
                        if (values[i][1] == a[k]['word']):
                            if (values[i][0][0] == word):
                                    pol = sid.polarity_scores(word)['compound']
                                    pol = round(pol,2)
                                ax = (0, (values[i][0][0],values[i][0][1],pol), (a[k]['word'], a[k]['tag']), values[i][-2], values[i][-1])
                    
                    elif (isinstance(values[i][1], tuple) == True):
                        if (values[i][0] == a[k]['word']):
                            if (values[i][1][0] == word):
                                pol = sid.polarity_scores(word)['compound']
                                pol = round(pol,2)
                                ax = (1, (a[k]['word'], a[k]['tag']), (values[i][1][0],values[i][1][1], pol), values[i][-2], values[i][-1])
                    
                    nval.append(ax)
                nval = list(set(nval))
    newDic['id'] = ids
    newDic['sentence'] = sent
    newDic['Details'] = nval
    newDic['Labels'] = lsPosDep[h]['labels']
    lsPosSentDep.append(newDic)
print("****Finito****")
len(lsPosSentDep)
lsPosSentDep[0]<jupyter_output><empty_output><jupyter_text>Saving airline wise dependency tag and other features in json format<jupyter_code>savePath = r'C:\Users\<NAME>\Desktop\Practicum\DepTag\TripAdvisor'
with open(savePath+r'\Vistara_TA'+'DepTag.json', 'w') as outfile:
    json.dump(lsPosSentDep, outfile)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14210558512265248, 'fraction_numerical': 0.007218449868477397, 'mean_word_length': 1.7466397849462365, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 70, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '31092', 'license': 'Apache-2.0', 'n_tokens_mistral': 5127, 'n_tokens_neox': 4880, 'n_words': 1120, 'path': 'src/Feature Engineering/src/Entity_Level_Feature_Generation.ipynb', 'repo_name': 'vaibhavoberoi/implicit_aspect_airline_reviews', 'size': '16368'}