<jupyter_start><jupyter_text>**Import Libraries**<jupyter_code>import numpy as np
import pandas as pd
import nltk
import chardet
from importlib import reload
import re
import sys
reload(sys)
sys.getdefaultencoding()<jupyter_output><empty_output><jupyter_text>**Read Movies Metadata, Lines and Characters**<jupyter_code>movies_metadata=pd.read_csv("../cornell movie-dialogs corpus/movies_metadata.csv")
f1 = open("../cornell movie-dialogs corpus/movie_lines.csv", "rb")
movie_lines= f1.read().decode(errors='replace').splitlines()
print(type(movie_lines))
f2 =open("C://Users/PrishitaRay/Desktop/DSER_Movie_Subtitles/cornell movie-dialogs corpus/movie_characters_metadata.csv","rb")
movie_characters_metadata = f2.read().decode(errors='replace').splitlines()
movies_metadata=np.array(movies_metadata)
movies_metadata
movie_lines=np.array(movie_lines)
movie_lines
L=[]
for i in range(0,len(movie_lines)):
    x=movie_lines[i].split(',')
    L.append(x)
lines=[]
for i in range(0,len(L)):
    l=[]
    for j in range(0,5):
        l.append(L[i][j])
    lines.append(l)   
lines[0]
movie_characters=np.array(movie_characters_metadata)
movie_characters  
L=[]
for i in range(0,len(movie_characters)):
    x=movie_characters[i].split(',')
    L.append(x)
characters=[]
for i in range(0,len(L)):
    l=[]
    for j in range(0,6):
        l.append(L[i][j])
    characters.append(l)
characters[0]
L=[]
for i in range(0,len(lines)):
    L.append(lines[i][2])
ids=set(L)
movie_ids=list(ids)
movie_ids[0:10]<jupyter_output><empty_output><jupyter_text>**Map Movie Lines to respective Movie IDs**<jupyter_code>lines_dict=dict.fromkeys(movie_ids)
for x in movie_ids:
    l=[]
    for i in range(0,len(lines)):
        if lines[i][2]==x:
            l.append(lines[i][4])
    lines_dict[x]=l
lines_dict[' m1 '][0:10]<jupyter_output><empty_output><jupyter_text>**Map Movie Characters to respective Movie IDs**<jupyter_code>characters_dict=dict.fromkeys(movie_ids)
for x in movie_ids:
    l=[]
    for i in range(0,len(characters)):
        if characters[i][2]==x:
            l.append(characters[i][1])
    characters_dict[x]=l
characters_dict[' m2 ']
movies_dict=dict.fromkeys(movie_ids)
for x in movie_ids:
    l=[]
    for i in range(0,len(characters)):
        if characters[i][2]==x:
            l.append(characters[i][3])
            break
    movies_dict[x]=l
movies_dict[' m3 ']<jupyter_output><empty_output><jupyter_text>**Map lines spoken in the movie to their actual speakers**<jupyter_code>actual_speakers=dict.fromkeys(movie_ids)
for x in movie_ids:
    l=[]
    for i in range(0,len(lines)):
        if lines[i][2]==x:
            l.append(lines[i][3])
    actual_speakers[x]=l
actual_speakers[' m4 '][0:50]
import tensorflow as tf
from tensorflow.contrib.layers import fully_connected<jupyter_output><empty_output><jupyter_text>**Perform word tokenization and POS Tagging on the movie lines**<jupyter_code>from nltk import pos_tag,word_tokenize

l=lines_dict[' m510 ']
p=[]
p.append(pos_tag(word_tokenize(l[3])))
p
POS_Tags=[]
for key in lines_dict.keys():
    p=[]
    for x in lines_dict[key]:
        p.append(pos_tag(word_tokenize(x)))
    POS_Tags.append(p)
POS_Tags[100][0:10]<jupyter_output><empty_output><jupyter_text>**Lemmatize the POS Tagged Lines using NLTK's inbuilt WordNetLemmatizer**<jupyter_code>from nltk.stem import WordNetLemmatizer
lemmatizer=WordNetLemmatizer()
for i in range(0,len(POS_Tags)):
    for j in range(0,len(POS_Tags[i])):
        for k in range(0,len(POS_Tags[i][j])):
            l=list(POS_Tags[i][j][k])
            l[0]=lemmatizer.lemmatize(l[0])
            t=tuple(l)
            POS_Tags[i][j][k]=t   
lemmatized_with_POS=POS_Tags
lemmatized_with_POS[600][0:10]<jupyter_output><empty_output><jupyter_text>**Remove Stop Words from Lemmatized Lines**<jupyter_code>remove_stop_words=[]
from nltk.corpus import stopwords
import nltk
stop_words = stopwords.words('english')
stop_words[:10]
for i in range(0,len(POS_Tags)):
    for j in range(0,len(POS_Tags[i])):
        n=len(POS_Tags[i][j])
        for k in range(0,n):
            l=list(POS_Tags[i][j][k])
            l[0]=l[0].lower()
            t=tuple(l)
            POS_Tags[i][j][k]=t
len(POS_Tags)
for i in range(0,len(POS_Tags)):
    l2=[]
    for j in range(0,len(POS_Tags[i])):
        l1=[]
        for k in range(0,len(POS_Tags[i][j])):
            l=list(POS_Tags[i][j][k])
            if l[0] not in stop_words:
                l1.append(POS_Tags[i][j][k])
        l2.append(l1)
    remove_stop_words.append(l2)<jupyter_output><empty_output><jupyter_text>**Create pairs of consecutive dialogs from the processed movie lines**<jupyter_code>pp_lines_dict=dict.fromkeys(movie_ids)

cnt=0
for key in pp_lines_dict.keys():
    l=[]
    for j in range(0,len(POS_Tags[cnt])):
        s=""
        for pair in POS_Tags[cnt][j]:
            if s=="":
                s=s+pair[0]
            else:
                s=s+" "+pair[0]
        l.append(s)
    pp_lines_dict[key]=l
    cnt= cnt+1<jupyter_output><empty_output><jupyter_text>**Read the Switchboard Dialog Act Corpus for training**<jupyter_code>import os
files=os.listdir("C://Users/PrishitaRay/Desktop/DSER_Movie_Subtitles/Switchboard Dialog Act Corpus/swda")
sw_train=[]

for i in range(1,len(files)):
        sw_train.append(files[i])
        
print(len(sw_train))<jupyter_output>1140<jupyter_text>**Create lists of Dialogs and their corresponding Dialog Acts from the Corpus**<jupyter_code>#Training on SwitchBoard Dialogs Act Corpus
import re
os.chdir("C://Users/PrishitaRay/Desktop/DSER_Movie_Subtitles/Switchboard Dialog Act Corpus/swda")
tokenized_dialogs=[]
dialog_acts=[]
dialogs=[]
for f in sw_train:
    file= pd.read_csv(f)
    d_act= list(file['act_tag'])
    text_with_pos= list(file['pos'])
    
    for d in d_act:
        dialog_acts.append(d)

    for i in range(0,len(d_act)):
        #Removing unneccessary delimiters from text
        pos= str(text_with_pos[i]).strip()
        word_tag = map((lambda x : tuple(x.split("/"))), re.split(r"\s+", pos))
        word_tag = filter((lambda x : len(x) == 2), word_tag)

        w= [x[0] for x in word_tag]
        tokenized_dialogs.append(w)
        separator=' '
        d= separator.join(w)
        dialogs.append(d)
print(len(dialogs))
print(len(dialog_acts))
print(dialogs[0:5])
print(dialog_acts[0:5])
#Testing for first movie only
test_data=[]
for key in pp_lines_dict.keys():
    #List of sentences with word embeddings for each movie
    swe=[]
    for line in pp_lines_dict[key]:
        tokenized = re.sub('[,?.]','', line).lower().split(' ')
        separator=' '
        line=separator.join(tokenized)
        test_data.append(line)
        
    break<jupyter_output><empty_output><jupyter_text>**Create vocabulary of words in the Corpus based on their occurence frequencies**<jupyter_code>#List of words
low=[]

for dialog in tokenized_dialogs:
    for token in dialog:
        low.append(token)  

low_cnt={k:v for v,k in enumerate(np.unique(low))}

sorted_lc=sorted(low_cnt.items(), key = lambda kv:(kv[1], kv[0]),reverse=True)[0:10000]
vocab=dict(sorted_lc)
print(len(vocab))
#print(vocab)<jupyter_output>10000<jupyter_text>**Encode words in a dialog to be trained using One Hot Encoding over the Vocabulary Length along with padding** **Generate a sparse matrix of the dialog acts after One Hot Encoding**<jupyter_code>from sklearn.preprocessing import LabelEncoder,OneHotEncoder
from keras.preprocessing.text import one_hot
from keras.preprocessing.sequence import pad_sequences
VOCAB_LEN=len(vocab.keys())
n_timesteps=20
k=5
encoded_docs = [one_hot(d, VOCAB_LEN) for d in dialogs]
max_length = 20
padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')

X= padded_docs[0:100000]
dialog_acts=np.array(dialog_acts)
label_encoder = LabelEncoder()
vec = label_encoder.fit_transform(dialog_acts)
enc= OneHotEncoder(sparse=False)
Y = enc.fit_transform(vec.reshape((vec.shape[0]),1))
Y= Y[0:100000]<jupyter_output>c:\users\prishitaray\appdata\local\programs\python\python35\lib\site-packages\sklearn\preprocessing\_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
  warnings.warn(msg, FutureWarning)<jupyter_text>**Create the Keras Bidirectional LSTMs Model (check the paper in the README file for reference)**<jupyter_code>from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers import Bidirectional
from keras.layers import TimeDistributed
from keras.layers import Dense
from keras import optimizers
from keras.layers.embeddings import Embedding

model= Sequential()
model.add(Embedding(VOCAB_LEN, 200, input_length=20))
model.add(Dropout(0.5))
model.add(Bidirectional(LSTM(100,return_sequences=True)))
#k=5 for previous 5 timesteps
model.add(LSTM(100))
#One hot encoding of Dialog Act class
model.add(Dense(303, activation='softmax'))
sgd = optimizers.SGD(lr=1, decay=0.8, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
print(model.summary())
model.fit(X, Y, epochs=50,batch_size=32,verbose=2)
# evaluate the model
loss, accuracy = model.evaluate(X, Y, verbose=2)
print('Accuracy: %f' % (accuracy*100))<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_1 (Embedding)      (None, 20, 200)           2000000   
_________________________________________________________________
dropout_1 (Dropout)          (None, 20, 200)           0         
_________________________________________________________________
bidirectional_1 (Bidirection (None, 20, 200)           240800    
_________________________________________________________________
lstm_2 (LSTM)                (None, 100)               120400    
_________________________________________________________________
dense_1 (Dense)              (None, 303)               30603     
=================================================================
Total params: 2,391,803
Trainable params: 2,391,803
Non-trainable params: 0
_________________________________________________________________
[...]<jupyter_text>**Predict Dialog acts over movie lines**<jupyter_code>encoded_docs = [one_hot(d, VOCAB_LEN) for d in test_data]
max_length = 20
X_test = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
ohe_Y_pred=model.predict(X_test)
da=list(set(dialog_acts))
pred_DA=[]
for i in range(0,len(ohe_Y_pred)):
    pred=list(ohe_Y_pred[i])
    maxpos=pred.index(max(pred))
    pred_DA.append(da[maxpos])

print(pred_DA)<jupyter_output>['ba', 'ba', 'ar^r', 'ba', 'ar^r', 'ar^r', 'ba', 'ba', 'ba', 'ar^r', 'qw', 'ba', 'ar^r', 'ba', 'ar^r', 'ar^r', 'ba', 'qw', 'ba', 'ba', 'ar^r', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ba', 'ba', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ba', 'ba', 'ba', 'ar^r', 'ar^r', 'ba', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ba', 'ar^r', 'ba', 'ba', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'qw', 'ba', 'ba', 'ar^r', 'ar^r', 'ar^r', 'ba', 'ba', 'ba', 'qw', 'ba', 'ba', 'ba', 'ar^r', 'ar^r', 'ar^r', 'ba', 'ba', 'ba', 'qw', 'ba', 'ar^r', 'ba', 'ba', 'qw', 'ba', 'ar^r', 'ar^r', 'ar^r', 'ar^r', 'ar^r', 'qw', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ba', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ba', 'qw', 'ba', 'ba', 'ar^r', 'ar^r', 'ba', 'ba', 'ba', 'ar^r', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ba', 'ba', 'ba', 'ba', 'ar^r', 'ar^r', 'ba', 'qw', 'ar^r[...]<jupyter_text>**Rules to determine speaker turns based on rules for a pair of dialog acts**<jupyter_code>turn_indexes=[]
for i in range(0,len(pred_DA)-1):
    #d is for the current dialog, nd is for the next dialog in the pair
    d=pred_DA[i]
    nd=pred_DA[i+1]
    if nd=='b' or nd=='sv' or nd=='aa' or nd=='ba' or nd=='bk' or nd=='fo_o_fw_by_bc' or nd=='bf' or nd=='ft' or nd=='bd':
        turn_indexes.append(i+1)
    elif d=='qo' or d=='qy^d' or d=='qy' or d=='qw' or d=='^g' or d=='qrr':
        if nd=='ny' or nd=='nn' or nd=='na' or nd=='ng' or nd=='no' or nd=='sd' or nd=='sv'or nd=='arp_nd' or nd=='aap_am' or nd=='ng' or nd=='oo_co_cc' or nd=='ar^r':
            turn_indexes.append(i+1)
    elif d=='ad' and nd=='no':
        turn_indexes.append(i+1)
    elif d=='fp' and nd=='sd':
        turn_indexes.append(i+1)
    elif d=='qw^d':
        if nd=='sd' or nd=='ng':
            turn_indexes.append(i+1)
    elif d=='fc' and nd=='aa':
        turn_indexes.append(i+1)
    elif d=='%' or d=='ba' or d=='sv' or d=='sd':
        if nd=='^q':
            turn_indexes.append(i+1)
    elif d=='qh':
        turn_indexes.append(i+1)
    elif d=='fa' or d=='ft' or d=='oo_co_cc' or d=='fc':
        if nd=='bd':
            turn_indexes.append(i+1)
    elif d=='bf' and nd=='aap_am':
        turn_indexes.append(i+1)
    elif d=='bh' or d=='^g':
        if nd=='aa':
            turn_indexes.append(i+1)
    elif d=='t3' and nd=='x':
        turn_indexes.append(i+1)
    elif d=='^q' and nd=='br':
        turn_indexes.append(i+1)
    elif d=='^h':
        if nd=='qw^d' or nd=='br' or nd=='bd' or nd=='^q':
            turn_indexes.append(i+1)
    elif d=='br' and nd=='ad':
        turn_indexes.append(i+1)
    elif d=='arp_nd':
        if nd=='t3' or nd=='oo_co_cc' or nd=='bd' or nd=='ft' or nd=='qw^d':
            turn_indexes.append(i+1)
    elif d=='^2':
        turn_indexes.append(i+1)
    elif d=='h':
        if nd=='bd' or nd=='^q':
            turn_indexes.append(i+1)
    elif d=='t1':
        if nd=='qw^d' or nd=='t3' or nd=='arp_nd' or nd=='no' or nd=='b^m':
            turn_indexes.append(i+1)
    elif d=='ad' and nd=='ft':
        turn_indexes.append(i+1)
    elif d=='%':
        turn_indexes.append(i+1)
    elif d=='fo_o_fw_by_bc' and nd=='bk':
        turn_indexes.append(i+1)<jupyter_output><empty_output><jupyter_text>**Sample Speaker IDs at predicted speaker turns in the movie and display to the user**<jupyter_code>print("Speaker turns for movie: ",movies_dict[' m0 '])
for i in range(0,len(lines_dict[' m0 '])):
    if i in turn_indexes:
        print(lines_dict[' m0 '][i],"(speaker turn): ",actual_speakers[' m0 '][i])
    else:
        print(lines_dict[' m0 '][i])<jupyter_output>Speaker turns for movie:  [' 10 things i hate about you ']
 They do not!
 They do to! (speaker turn):   CAMERON 
 I hope so.
 She okay? (speaker turn):   CAMERON 
 Let's go.
 Wow
 Okay -- you're gonna need to learn how to lie. (speaker turn):   BIANCA 
 No (speaker turn):   CAMERON 
" I'm kidding.  You know how sometimes you just become this ""persona""?  And you don't know how to quit?" (speaker turn):   BIANCA 
 Like my fear of wearing pastels?
" The ""real you""."
 What good stuff? (speaker turn):   BIANCA 
 I figured you'd get to the good stuff eventually.
 Thank God!  If I had to hear one more story about your coiffure... (speaker turn):   CAMERON 
" Me.  This endless ...blonde babble. I'm like
 What crap?
 do you listen to this crap? (speaker turn):   BIANCA 
 No...
" Then Guillermo says (speaker turn):   BIANCA 
 You always been this selfish? (speaker turn):   CAMERON 
 But
 Then that's all you had to say.
" Well (speaker turn):   BIANCA 
" You never wanted to go out with 'me (s[...]<jupyter_text>**Train Naive Bayes Classifier over ISEAR Dataset to predict emotions**<jupyter_code>import string
from textblob.classifiers import NaiveBayesClassifier

emotion_df=pd.read_csv("C://Users/PrishitaRay/Desktop/DSER_Movie_Subtitles/ISEAR.csv",header=None)

exclude=set(string.punctuation)
lemma=WordNetLemmatizer()

neg=['not', 'neither', 'nor', 'but', 'however', 'although', 'nonetheless', 'despite', 'except','even though', 'yet']

em_list = list(emotion_df[0][0:1000])
text_list = list(emotion_df[1][0:1000])

train = []
sum_text_list = []
e_score_dict = {}

for i in range(0,len(text_list)):
    stop_free = " ".join([x for x in text_list[i].lower().split() if x not in stop_words if x not in neg])
    punc_free = "".join([ch for ch in stop_free if ch not in exclude])
    text_list[i] = " ".join([lemmatizer.lemmatize(word) for word in punc_free.split()])

for i in range(0,len(em_list)):
    l=[]
    l.append(text_list[i])
    
    l.append(em_list[i])
    train.append(l)

cl = NaiveBayesClassifier(train)

emotion_data=dict.fromkeys(movie_ids)

cnt=0
for key in emotion_data.keys():
    l=[]
    for j in range(0,len(POS_Tags[cnt])):
        s=""
        for pair in POS_Tags[cnt][j]:
            if s=="":
                s=s+pair[0]
            else:
                s=s+" "+pair[0]
        l.append(s)
    emotion_data[key]=l
    cnt= cnt+1
    
test_em_mov0=emotion_data[' m0 ']

for line in test_em_mov0:
    em=cl.classify(line)
    print(line+" :"+em)<jupyter_output>they do not ! :guilt
they do to ! :shame
i hope so . :sadness
she okay ? :joy
let 's go . :guilt
wow :joy
okay -- you 're gon na need to learn how to lie . :disgust
no :joy
`` i 'm kidding . you know how sometimes you just become this `` '' persona '' '' ? and you do n't know how to quit ? '' :disgust
like my fear of wearing pastel ? :joy
`` the `` '' real you '' '' . '' :joy
what good stuff ? :joy
i figured you 'd get to the good stuff eventually . :joy
thank god ! if i had to hear one more story about your coiffure ... :joy
`` me . this endless ... blonde babble . i 'm like :sadness
what crap ? :joy
do you listen to this crap ? :anger
no ... :joy
`` then guillermo say :joy
you always been this selfish ? :joy
but :joy
then that 's all you had to say . :disgust
`` well :joy
`` you never wanted to go out with 'me :guilt
i wa ? :joy
`` i looked for you back at the party :joy
tons :joy
have fun tonight ? :joy
i believe we share an art instructor :sadness
you know chastity ? :joy
`` looks [...]
{'fraction_non_alphanumeric': 0.16616693016409528, 'fraction_numerical': 0.016355012811426702, 'mean_word_length': 4.1369364323718845, 'pattern_counts': {'":': 2, '<': 66, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '666488', 'license': 'MIT', 'n_tokens_mistral': 7014, 'n_tokens_neox': 6669, 'n_words': 1685, 'path': 'src/HCI_Project.ipynb', 'repo_name': 'PRISHIta123/DSER_Movie_Subtitles', 'size': '18343'}