<jupyter_start><jupyter_text>Open subtitles In this segement we are using opus opensubtitles 2016 parallel corpora  Helper methods<jupyter_code>import urllib2
from bs4 import BeautifulSoup

def getMovieName(docId):
    user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
    url = "http://www.opensubtitles.org/en/subtitles/{0}".format(docId)

    headers={'User-Agent':user_agent,} 

    req = urllib2.Request(url, headers=headers)

    try:
        page = urllib2.urlopen(req)
        
        content = page.read()
        soup = BeautifulSoup(content, 'html.parser')

        text = soup.find('span', {'itemprop': 'name'}).text
            
        return text[:text.rfind('subtitles')-1]
    
    except urllib2.HTTPError, e:
        print e.fp.read()
        return 'no name found'
print getMovieName(3127877)
pwd<jupyter_output><empty_output><jupyter_text>Mock Tree<jupyter_code>from babelnet.structures import Tree
import numpy as np

mockTree = Tree(np.array([0., 0.]), [
        Tree(np.array([2.,2.])), 
        Tree(np.array([3.,3.]), [
                Tree(np.array([4.,4.])),
                Tree(np.array([4.,4.])),
                Tree(np.array([4.,4.])), 
                Tree(np.array([5.,5.]), [
                            Tree(np.array([1., 1.0])),
                            Tree(np.array([1.,1.]))
                        ])
            ]), 
        Tree(np.array([1.,1.]))
    ])<jupyter_output><empty_output><jupyter_text>Distributed word representation and Semantic TreesHere we are taking advantage of semantic trees created with the help of BabelNet API and distributed word representation obtained by training skip-gram model with negative sampling on both polish and english wikipedia to automatically annotate movie subtitles.  Word2Vec<jupyter_code>import numpy as np
import os
import string

from display_sense_tree import load
from nltk import word_tokenize
from nltk.corpus import stopwords
from gensim.models import Word2Vec
from sklearn.metrics.pairwise import cosine_similarity
def getSentenceVector(sentence, params):    
    word_list = word_tokenize(sentence)
    
    model = params['model']
    blacklist = params['blacklist']
    
    filtered_words = [word.lower() for word in word_list if word.lower() not in blacklist]                       
    sentence_vector = sum([model[word] for word in filtered_words if word in model])
    
    return sentence_vector

def getVector(tree, params):
    word = tree.value.lemma
    model = params['model'] 
    return model[word] if word in model else np.zeros(model.layer1_size)

def tree2vector(tree, params, f = getVector):        
    
    def getvector(tree, current_weight, level):                                     
        
        tree_value = f(tree, params)
        
        node_value =  tree_value
        node_weight = 1.0 / len(tree.children) if tree.children else 1               

        if tree.children:        
            for leaf in tree.children:      
                node_value += node_weight*getvector(leaf, node_weight, level + 1)  
        
        return node_value / level
    
    return getvector(tree, 1, 1)

def getEmotionVectors(path, params):
    emotions = {}
    for file in os.listdir(path):
        if file.endswith(".pickle"):
            tree = load('{0}/{1}'.format(path, file))
            emotions[file[:file.find('.')]] = tree2vector(tree, params)

    return emotions

def closest(sentence_vector, emotions):
    
    closest = None
    best_distane = -float('inf')

    for name, emotion_vector in emotions.iteritems():        
        current_distance = cosine_similarity([sentence_vector], [emotion_vector])[0][0]    
        if best_distane < current_distance:
            best_distane = current_distance
            closest = name
    
    return closest
print mockTree
tree2vector(mockTree, {}, lambda tree, params: np.array(tree.value))
MODEL_PATH = 'C:/Users/hp/Anaconda3/envs/minor/project1/Lib/site-packages/'
import gensim 
en_model = gensim.models.KeyedVectors.load_word2vec_format(MODEL_PATH + 'GoogleNews-vectors-negative300.bin', binary=True)

#en_model = Word2Vec.load('/home/models/wiki.en.model')
#pl_model = Word2Vec.load('/home/models/wiki.pl.model')

en_params = {'model': en_model, 'blacklist': set(stopwords.words('english')).union(set(string.punctuation))}
#pl_params = {'model': pl_model, 'blacklist': set(stopwords.words('polish')).union(set(string.punctuation))}<jupyter_output><empty_output><jupyter_text>English test<jupyter_code>#import build_emotion_vectors
from build_emotion_vectors import getSentenceVector, getEmotionVectors, closest

sentence = getSentenceVector("To continue , you 've listened to a long and complex case , murder in the first degree .", en_params)
en_emotions = getEmotionVectors('/home/models/babelnet/en/', en_params)

closest(sentence, en_emotions)<jupyter_output><empty_output><jupyter_text>Polish test<jupyter_code>sentence = getSentenceVector('Kontynuując , wysłuchali panowie długiego i złożonego procesu dotyczącego morderstwa pierwszego stopnia .', pl_params)
pl_emotions = getEmotionVectors('/home/models/babelnet/pl/', pl_params)

closest(sentence, pl_emotions)<jupyter_output><empty_output><jupyter_text>Open subtitles 2016Now we will test wheter there is a correlation between emotions in emotions discovered in polish and english subtitles<jupyter_code>from babelnet import opensubtitles as o

# to reduce the amount of stuff downloaded from the internet we would like to cache as much as possible
class Memoize:
    def __init__(self, f):
        self.f = f
        self.memo = {}
    def __call__(self, *args):
        if not args in self.memo:
            self.memo[args] = self.f(*args)
        return self.memo[args]

o.getParalelSubtittles = Memoize(o.getParalelSubtittles)<jupyter_output><empty_output><jupyter_text>PlottingTo plot movie plot (pun intended) we would like to nicely accumulate emotions from given "subtitle stream"<jupyter_code>from collections import Counter

class DictCounter(object):
    def __init__(self, bufferSize, keys):
        self.bufferSize = bufferSize
        self.buffer = []
        self.results = []
        self.keys = keys
    
    def __flush__(self):
        self.results.append(Counter(self.buffer))
        self.buffer = []
    
    def add(self, item):
        self.buffer.append(item)
        if len(self.buffer) >= self.bufferSize:
            self.__flush__()
            
    def collect(self, normalize = True):
        if self.buffer:
            self.__flush__()
        
        data = {key:[] for key in self.keys}
        
        for counter in self.results:                       
            normalizing_factor = 1.0 / sum(counter.values()) if normalize else 1
            for key in self.keys:
                data[key].append(round(normalizing_factor * counter[key], 2))
        
        return data<jupyter_output><empty_output><jupyter_text>PlotThis is acctual code for plotting<jupyter_code>%matplotlib inline

import matplotlib.pyplot as plt

def plotMovie(data_1, data_2, title = 'Movie', color_map = 'plasma', titlesize = 18):

    cmap =  plt.cm.get_cmap(color_map)
    
    f, (ax1, ax2) = plt.subplots(2, 1, figsize=(20, 7))

    # positions of the left bar-boundaries
    keys_size = float(len(data_1.keys()))
    data_size = len(data_1[data_1.keys()[0]])
    bar_l = [i+1 for i in range(data_size)] 

    # positions of the x-axis ticks (center of the bars as bar labels)
    tick_pos = [i+(1/2) for i in bar_l] 
    
    def drawplot(data, suffix, ax):
        cum = None
        for (index, (label, series)) in enumerate(data.items()):

            # plot bar
            ax.bar(
                bar_l, series,
                bottom=cum, label=label,
                alpha=0.8, color = cmap(index / keys_size),
                width=1.0, linewidth = 0.0
            )    

            cum = series if not cum else [x + y for x, y in zip(cum, series)]

        plt.sca(ax)
        ax.set_ylabel("Frequency")
        ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
        ax.set_title('{0} {1}'.format(title, suffix), fontsize = titlesize)
        plt.xlim([min(tick_pos), max(tick_pos)])
        plt.ylim(0, 1)
        plt.grid()

    ###################
    ## First Data
    ###################    
    drawplot(data_1, "EN", ax1)
    
    ###################
    ## Second Data
    ###################
    drawplot(data_2, "PL", ax2)

    plt.show()<jupyter_output><empty_output><jupyter_text>This is it!<jupyter_code>import itertools

def colsestEmotion(text, emotion_vectors, params):
    sentence_vector = getSentenceVector(text, params)
    return closest(sentence_vector, emotion_vectors)

def extractEmotions(doc, buffLimit = 10, limit = None, normalize = True):
                    
    en_acc = DictCounter(buffLimit, en_emotions.keys())
    pl_acc = DictCounter(buffLimit, pl_emotions.keys())                    
                    
    for en_text, pl_text in itertools.islice(o.getParalelSubtittles(doc), 0, limit, 1):
        
        # Try not to die
        try:
            em_en = colsestEmotion(en_text, en_emotions, en_params)
            en_acc.add(em_en)
        except:
            print 'Fallback to emotionless [en]', en_text
            en_acc.add('emotionless')
        
        # Try not to die
        try:
            em_pl = colsestEmotion(pl_text, pl_emotions, pl_params)
            pl_acc.add(em_pl)
        except:
            print 'Fallback to emotionless [pl]', pl_text
            pl_acc.add('emotionless')                                                        

    return en_acc.collect(normalize), pl_acc.collect(normalize)<jupyter_output><empty_output><jupyter_text>12 Angry Men<jupyter_code>doc = "en/1957/50083/3127877.xml.gz"

data_en, data_pl = extractEmotions(doc, buffLimit = 10)

plotMovie(data_en, data_pl, '12 Angry Men', titlesize = 18)<jupyter_output><empty_output><jupyter_text>Casablanca<jupyter_code>doc = "en/1942/34583/3098261.xml.gz"

data_en, data_pl = extractEmotions(doc, buffLimit = 10)
plotMovie(data_en, data_pl, 'Casablanca', titlesize = 18)<jupyter_output><empty_output><jupyter_text>Pulp Fiction<jupyter_code>doc = "en/1994/110912/3133987.xml.gz"

data_en, data_pl = extractEmotions(doc, buffLimit = 10)
plotMovie(data_en, data_pl, 'Pulp Fiction', titlesize = 18)
data_en, data_pl = extractEmotions("en/1994/110912/3133987.xml.gz", buffLimit = 10, normalize= False)
sum([sum(x) for x in data_en.values()])
for k,v in data_en.iteritems():
    print k, sum(v)
sum([sum(x) for x in data_pl.values()])
for k,v in data_pl.iteritems():
    print k, sum(v)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11839521182081736, 'fraction_numerical': 0.020106611802113532, 'mean_word_length': 2.885901162790698, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 4}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '154708', 'license': 'MIT', 'n_tokens_mistral': 3659, 'n_tokens_neox': 3452, 'n_words': 777, 'path': 'Sentiment Analysis/envs/emotion_vectors.ipynb', 'repo_name': 'G-akki26/Automatic-Subtitle-Media-Player', 'size': '10693'}