<jupyter_start><jupyter_text>Data collection:  Get the list of movies: * Get a list of Wikipedia URLs from movies1.html/ movies2.html/ movies3.html<jupyter_code>from bs4 import BeautifulSoup
import requests
import time
import random
f1 = open("HW3 ADM/movies3.html")
f3 = open("HW3 ADM/movies1.html")
f2 = open("HW3 ADM/movies2.html")
soup = BeautifulSoup(f1)
soup1 = BeautifulSoup(f3)
soup2 = BeautifulSoup(f2)
listUrl_Movies1=[]
listUrl_Movies2=[]
listUrl_Movies3=[]
for link in soup.select('a'):
    listUrl_Movies3.append(link.text)
for link in soup2.select('a'):
    listUrl_Movies2.append(link.text)
for link in soup1.select('a'):
    listUrl_Movies1.append(link.text)<jupyter_output><empty_output><jupyter_text>* Merge these 3 lists in a single list that contains all 30 000 urls<jupyter_code>totalMovies=listUrl_Movies1+listUrl_Movies2+listUrl_Movies3<jupyter_output><empty_output><jupyter_text>Crawl Wikipedia: * Function 'downloadFile' that allows to download html files from URL listWe downloaded all movies. 10 000 per person. Every person downloaded file from:* listUrl_Movies1/ listUrl_Movies2/ listUrl_Movies3<jupyter_code>def downloadFile():

    for index in range(listUrl_Movies1):
        #set time=20 mintes
        t2=1200
        try:
            #wait 5 seconds bettween every request
            t1 = random.randint(1,5)
            time.sleep(t1)
            url=listUrl_Movies3[index]
            response = requests.get(url)
            name="article_"
            extension=".html"
            file="{}{}{}".format(name,index,extension)
            with open(file,'wb') as f: 
                f.write(response.content)  

        except response.status_code as e:
            print("exception")
            #error=492 is error that occurs when we have done a limit of request
            if e==492:
                #wait 20 minutes 
                time.sleep(t2)
                downloadFile(index+1)
            elif e==200:
                soup = BeautifulSoup(listUrl_Movies3[1])
                
                with open(file,'w') as f: 
                    f.write(soup.text)
                downloadFile(index+1)
            else:
                continue


downloadFile()<jupyter_output><empty_output><jupyter_text>Parse downloaded pages: * Create .tsv files for every html file and put them in tsv directory. These files contain data (title,intro,plot and infobox infos) as written in hw track. About starring in infobox, we saved every actor in a list (to do bonus section).<jupyter_code>import csv
import pandas as pd
import os.path
#define column of our dataframe
df=pd.DataFrame(columns=['title', 'intro', 'plot','film_name','producer','director','writer','starring','music','release date','runtime','country','language','budget'])


for index in range(len(totalMovies)):
    print(index)
    title=''
    plot=''
    intro=''
    title_name='NA'
    producer='NA'
    director='NA'
    writer='NA'
    starring=['NA']
    music='NA'
    release_date='NA'
    runtime='NA'
    country='NA'
    language='NA'
    budget='NA'
    
    
    #define name of the file that we want to find (in my case: in the same directory)
    name="article_"
    extension=".html"
    file="{}{}{}".format(name,index ,extension)
    
    #check if this file exists
    if not os.path.isfile("HW3 ADM/"+file):
        continue
        
    #open file   
    response2 = open("HW3 ADM/"+file)
    soup = BeautifulSoup(response2)
    #take title.
    title=soup.title.text.rsplit(' ', 2)[0]
    
    #take all p in intro(firt section)
    #print(soup.find('span', attrs={'class': 'mw-headline'}))
    if soup.find('span', attrs={'class': 'mw-headline'}):
        heading = soup.find('span', attrs={'class': 'mw-headline'})
        paragraphs = heading.find_all_previous('p')
        for p in paragraphs: 
            intro = p.text + intro
            
     
        #take all p in 'plot'(second section)
        b=True
        #print(soup.find('span', attrs={'class': 'mw-headline'}))
        if soup.find('span', attrs={'class': 'mw-headline'}): 
            
            heading = soup.find('span', attrs={'class': 'mw-headline'})
            
            for item in heading.parent.nextSiblingGenerator():
                
                if item.name=='h2':
                    break
                if hasattr(item, "text"):
                    
                    plot+=item.text

        else:
            plot="NAN"
    
    else:
        intro="NAN"
        plot="NAN"
    
    
    #Get info about infobox from every page and put them in respective sections in tsv file  
    if soup.find('table', attrs={'class': 'infobox vevent'}):
        
        table = soup.find('table', attrs={'class': 'infobox vevent'})  
    
        if table.find('th', attrs={'class': 'summary'}):
        
            x=table.find('th', attrs={'class': 'summary'})
            title_name=x.text.strip()
        
        for cell in table.find_all('th'):
        
            if cell.find_next_sibling('td'):
                a=cell.find_next_sibling('td')
                if cell.text.strip()=='Directed by':
                    director=a.text.strip()
                elif cell.text.strip()=='Produced by':
                
                    producer=a.text.strip()
                elif cell.text.strip()=='Written by':
                
                    writer=a.text.strip()
                elif cell.text.strip()=='Starring':
                    listStarring=[]
                    for link in a.select('a'):
                        
                        listStarring.append(link.text)
                    starring=listStarring
                    #print(starring)
                elif cell.text.strip()=='Music by':
                
                    music=a.text.strip()
                elif cell.text.strip()=='Release date':
                    release_date=a.text.strip()   
                elif cell.text.strip()=='Running time':
                
                    runtime=a.text.strip()
                elif cell.text.strip()=='Country':
              
                    country=a.text.strip()
                elif cell.text.strip()=='Language':
              
                    language=a.text.strip()
                elif cell.text.strip()=='Budget':
              
                    budget=a.text.strip()
            else:
                continue
            
    
    #put all infos in movie list
    movie=[title,intro,plot,title_name,producer,director,writer,starring,music,release_date,runtime,country,language,budget]
    #update dataframe with this list
    extension2=".tsv"
    file="{}{}{}".format(name,index,extension2)
   
    movieTitle=["title","intro","plot","title_name","producer","director","writer","starring","music","release_date","runtime","country","language","budget"]
    with open("HW3 ADM/tsv_new/"+file, 'w', newline='') as f_output:
        tsv_output = csv.writer(f_output, delimiter='\t')
        tsv_output.writerow(movieTitle)
        tsv_output.writerow(movie)
    df.loc[index] = movie<jupyter_output><empty_output><jupyter_text>Search Engine: Search engine 1: Conjunctive query  * Create tsv files with preporcessed text. We have preprocessed all texts in tsv file (that we have just created) and we have put them in other tsv files in tsv_correct directory. For every section we have a list of all words that are in respective section. We have also deleted duplicated words.<jupyter_code>import string
import csv

from shutil import move
from nltk.tokenize import RegexpTokenizer
#create tsv file in 'tsv_correct' directory wehere we have preprocessed the tsv file (just created in parser.py)
tokenizer = RegexpTokenizer(r'\w+')
name="article_"
extension2=".tsv"
exclude = string.punctuation
for index in range(len(totalMovies)):
    print(index)


    file="{}{}{}".format(name,index,extension2)
    with open("HW3 ADM/tsv/"+file,"r") as tsvfile, open("HW3 ADM/tsv_correct/"+file,"w") as outfile:
        tsvreader = csv.reader(tsvfile, delimiter="\t")
        tsvwriter = csv.writer(outfile, delimiter="\t")
        for row in tsvreader:
            for i in range(len(row)):
                #take every words, deleting ountuaction and other symbols
                row[i] = tokenizer.tokenize(row[i])
                #remove duplicate case-insensitive elements
                row[i]= list(set(map(str.lower, row[i])))
                #row[i] = row[i].translate({ord(c): None for c in string.punctuation})
                
            tsvwriter.writerow(row)<jupyter_output><empty_output><jupyter_text>* We created a vocabulary that we stored in vocabulary.tsv file. Here we have all words that we have in all tsv files (in intro and plot section as wirtten in hw track). Every word matches with an unique term_id. This term_is is a simple counter.<jupyter_code>import ast
from itertools import islice
import csv
#create vocabulary and save it on vocabulary.tsv
            

dict1 = dict()
term_id=0
present=False
with open('HW3 ADM/tsv/vocabulary.tsv', 'w', newline='') as f_output:
        tsv_vocabulary = csv.writer(f_output, delimiter='\t')
        tsv_vocabulary.writerow(['word','term_id'])
        name="article_"
        extension2=".tsv"
        
        for index in range(len(totalMovies)):
           
            print(index)
            file="{}{}{}".format(name,index,extension2)
            with open("HW3 ADM/tsv_correct/"+file,"r") as tsvfile:
                data_list = list(csv.reader(tsvfile, delimiter="\t"))
                tsvreader = csv.reader(tsvfile, delimiter="\t")
                #put in intro a list of all words that we have in intro of i-th page
                intro=data_list[1][2]
                intro = ast.literal_eval(intro)
                #put in plot a list of all words that we have in plot of i-th page
                plot=data_list[1][1]
                plot = ast.literal_eval(plot)
                
                #put in text, a list that contains all words that are in plot and word for every page (no duplicate)
                text=plot+intro
                text= list(set(map(str.lower, text)))
                
                #put in dict1 every words with its term_id (no duplicate)
                for i in text:
                    if i in dict1:    
                        continue
                    else:
                        dict1[i]=term_id
                        term_id+=1
                
        #put dict1 element in vocabulary.tsv file                
        for key, val in dict1.items():
                    tsv_vocabulary.writerow([key, val])<jupyter_output><empty_output><jupyter_text>Create index: * Then we created the index from vocabulary.tsv and from all preprocessed tsv files.  For every word (in plot and intro) in all prorocessed tsv files, we found their term_id (from vocabulary) and for evey term_id we match a list of document where repsective word is present. * This index was used to search query words in every document about 1st and 2nd search engines. We saved it into index.tsv<jupyter_code>import ast
from itertools import islice
import csv

#create index and save it on index.tsv                

dict2 = {}
count=0
present=False
with open('HW3 ADM/tsv/vocabulary.tsv', 'r', newline='') as f_output:
        tsv_vocabulary = list(csv.reader(f_output, delimiter='\t'))
        name="article_"
        extension2=".tsv"
        h=0
        for row in tsv_vocabulary:
            
            dict2[row[1]]=[]
        for index in range(len(totalMovies)):
            
            print(index)
            file="{}{}{}".format(name,index,extension2)
            with open("HW3 ADM/tsv_correct/"+file,"r") as tsvfile:
                data_list = list(csv.reader(tsvfile, delimiter="\t"))
                tsvreader = csv.reader(tsvfile, delimiter="\t")
                intro=data_list[1][1]
                intro = ast.literal_eval(intro)
                plot=data_list[1][2]
                plot = ast.literal_eval(plot)
                text=plot+intro
                text= list(set(map(str.lower, text)))
                
                #for evry words in plot adn intro (for every page) we get every word. From every word we get its term_id and put it whit their occurences (document_id) in dict2
                for i in text:
                    for row in tsv_vocabulary:
                        if i==row[0]:
                            doc="document_"
                            name2="{}{}".format(doc,index)
                            
                            dict2[row[1]].append(name2)
                            break
                        else:
                            continue
                            
        #put dict2 in index.tsv file. In. evry row we have a single term_id with occurences of respective word.
        with open('HW3 ADM/tsv/index1.tsv', 'w', newline='') as f_output:
            tsv_vocabulary = csv.writer(f_output, delimiter='\t')           
            for key, val in dict2.items():
                tsv_vocabulary.writerow([key, val])<jupyter_output><empty_output><jupyter_text>* Then we create getDocuments function that allows to find all documents where the input query is present.This fucntion has two parameters in input: words that is our query and index is the number of index that we consider to do specific search engine. This function returns a list of documents that contains input query based on index.* We use index1 about search engine 1 and search engine 2; index2 about search engine 3.* This function is used for all three different search engines<jupyter_code>import csv
import sys
import ast
#define function that allows us to calculate a list that is an intersection from two list
def intersection(lst1, lst2): 
    return list(set(lst1) & set(lst2)) 
def getDocuments(words,index):


    #we use dict3 to store term_id and its respective documents_id
    dict3={}
    ##we use dict4 to store evry word and its respective documents_id
    dict4={}
    csv.field_size_limit(sys.maxsize)
    #in listWords we have a list that contains all words about inout query
    listWords = words.split()
    listWords=[x.lower() for x in listWords]
    
    #with vocabulary.tsv we start to build a dict3 with term_id for every words in wordsList
    with open('HW3 ADM/tsv/vocabulary.tsv', 'r', newline='') as f_output:
        tsv_vocabulary = list(csv.reader(f_output, delimiter='\t'))
        for word in listWords:
            word=word.lower()
            present=False
            for row in tsv_vocabulary:
                if word.lower()==row[0]:
                    dict3[row[1]]=[]
                    present=True
            #case where word is not in vocabulary
            if present==False:
                dict4[word]=[]
        indexFile="index"+str(index)+".tsv"
        #we continue to match documnets_id to every term_id in dict3
        with open('HW3 ADM/tsv/'+indexFile, 'r', newline='') as f_output:
            tsv_index = list(csv.reader(f_output, delimiter='\t'))
            for k in dict3.keys():  
                for row in tsv_index:
                    if row[0]==k:
                        dict3[k]=row[1]
                        continue


        #finally we build dict4 where evry word matches to respective documents_id
        for k in dict3.keys():

            for row in tsv_vocabulary:
                if k==row[1]:
                    dict4[row[0]]=dict3[row[1]]

        document=ast.literal_eval(dict4[listWords[0]])         
        #return "no results" if any query words isn't at least in one document
        for i in dict4.values():
            if not i:
                error='No results'
                return error
        #interection between every list in values dict4. In this way we have documnets_id where all words (in query input) are present
        for value in dict4.values():
            document=intersection(document,ast.literal_eval(value))
        
    return document<jupyter_output><empty_output><jupyter_text>* searchEngine1 function allows to do the first search engine. It takes an input query and returns a dataframe that contains the result. In this function we call getDocuments and for every document that contains input query we get some info from not preprocessed tsv file in tsv directory.* This result is a list of movies and for every movies we show only intro, title and URL link.<jupyter_code>import pandas as pd



def searchEngine1(words):
        #build the dataframe with info for every documents_id
    document=getDocuments(words,1)
    if document=='No results':
        
        return document
    elif document==[]:
        return "No results"
    df=pd.DataFrame(columns=['title', 'intro', 'url'])
    for index in range(len(document)):
        #get id of documnets_is
        numberDocument=document[index][9:]
        #get wikipedia url
        
        url=totalMovies[int(numberDocument)]
        name="article_"
        extension2=".tsv"
        index=int(numberDocument)
        file="{}{}{}".format(name,index,extension2)
        #get info about title and intro for evert film that corresponds to every documents_id
        with open('HW3 ADM/tsv/'+file, 'r', newline='') as f_output:
            tsv_index = list(csv.reader(f_output, delimiter='\t'))
            title=tsv_index[1][3]
            intro=tsv_index[1][1]
            film=[title,intro,url]
            #put all info for every film in a single row of df dataframe
            df.loc[index] = film
    return df<jupyter_output><empty_output><jupyter_text>Execute the query:<jupyter_code>query='enormous damage unless something is done immediately'
searchEngine1(query)<jupyter_output><empty_output><jupyter_text>Search engine 2: Conjunctive query & Ranking score * Create tsv files from not preprocessed files. This is the same process that we've done for tsv file in tsv_correct dirctory. But in this case, we considerated also duplicate words. This it is important to calculate TfIdf about second search engine.* These tsv file are stored in tsv_correct2 directory<jupyter_code>import string
import csv
from shutil import move
from nltk.tokenize import RegexpTokenizer
#create tsv file in 'tsv_correct' directory wehere we have preprocessed the tsv file (just created in parser.py)
tokenizer = RegexpTokenizer(r'\w+')
name="article_"
extension2=".tsv"
exclude = string.punctuation
for index in range(len(totalMovies)):
    print(index)


    file="{}{}{}".format(name,index,extension2)
    with open("HW3 ADM/tsv/"+file,"r") as tsvfile, open("HW3 ADM/tsv_correct2/"+file,"w") as outfile:
        tsvreader = csv.reader(tsvfile, delimiter="\t")
        tsvwriter = csv.writer(outfile, delimiter="\t")
        for row in tsvreader:
            for i in range(len(row)):
                #take every words, deleting ountuaction and other symbols
                row[i] = tokenizer.tokenize(row[i])
                #remove duplicate case-insensitive elements
                row[i]= list(map(str.lower, row[i]))
                #row[i] = row[i].translate({ord(c): None for c in string.punctuation})
                
            tsvwriter.writerow(row)<jupyter_output><empty_output><jupyter_text>* From these tsv file that we have just created, we create a dataframe 'df2'.This dataframe contains the TfIdf value for every match between word-document. Its columns are all different words that we have preproccesed tsv file (in intro and plot). Its rows are different document that are identified by document_id (id stays for the number of movies, example:document_222 stays for article_222.html/article_222.tsv)<jupyter_code>import ast
from itertools import islice
import csv
import sys
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
csv.field_size_limit(sys.maxsize)
#create index and save it on index.tsv                

dict2 = {}
count=0
present=False
documents=[]
name='article_'
extension2='.tsv'
with open('HW3 ADM/tsv/vocabulary.tsv', 'r', newline='') as f_output:
        tsv_vocabulary = list(csv.reader(f_output, delimiter='\t'))
        for index in range(len(totalMovies)):
            print(index)
            file="{}{}{}".format(name,index,extension2)
            with open("HW3 ADM/tsv_correct2/"+file,"r") as tsvfile:
                data_list = list(csv.reader(tsvfile, delimiter="\t"))
                tsvreader = csv.reader(tsvfile, delimiter="\t")
                intro=data_list[1][1]
                intro = ast.literal_eval(intro)
                plot=data_list[1][2]
                plot = ast.literal_eval(plot)
                text=plot+intro
                text=list(map(str.lower, text))
                documents.append(' '.join(text))
                #print(documents)
                # text2= list(set(map(str.lower, text)))
                
                
        vectorizer = TfidfVectorizer(token_pattern=r"(?u)\b\w+\b")
      
        vectors = vectorizer.fit_transform(documents)
        #print(vectors)
        feature_names = vectorizer.get_feature_names()
        dense = vectors.todense()
        denselist = dense.tolist()
        df2 = pd.DataFrame(denselist, columns=feature_names)
df2<jupyter_output><empty_output><jupyter_text>Inverted index: * We created a new dictionary that has term_id as key (that we get from vocabulary by every word) and an array as value. This array contains a list of matching between document and respective TfIdf for respective word in this document. For example:key: "122", value:[[dcoument_12,0.02],[dcoument_18,0.22]]<jupyter_code>import ast
from itertools import islice
import csv
import sys
csv.field_size_limit(sys.maxsize)
#create index and save it on index.tsv                
dict={}
dict2 = {}
count=0
present=False


#print('step 2')
name="article_"
extension2=".tsv"
h=0
for row in tsv_vocabulary:
    h+=1
    dict[row[0]]=row[1]
    print(h)
    dict2[row[1]]=[]
for index in range(len(totalMovies)):
    #print(index)
    print("numero documento "+ str(index))
    file="{}{}{}".format(name,index,extension2)
    with open("HW3 ADM/tsv_correct2/"+file,"r") as tsvfile:
        data_list = list(csv.reader(tsvfile, delimiter="\t"))
        tsvreader = csv.reader(tsvfile, delimiter="\t")
        intro=data_list[1][1]
        intro = ast.literal_eval(intro)
        plot=data_list[1][2]
        plot = ast.literal_eval(plot)
        text=plot+intro
        text=list(map(str.lower, text))
    
        text2= list(set(map(str.lower, text)))
        #print(text2)
                        #for evry words in plot adn intro (for every page) we get every word. From every word we get its term_id and put it whit their occurences (document_id) in dict2
        for i in text2:

                            #print("word   "+str(i))
            # print("word "+str(i))
            # print("aaa" + str(df2.iloc[index][i]))
            res=df2.iloc[index][i]
                            #print("res   "+str(res))
            for term in dict:
                if i==term:
                    #print("key "+ str(term))
                    #print("value "+ str(dict[term]))
                    doc="document_"
                    name2="{}{}".format(doc,index)
                    result=[name2,res]
                    dict2[dict[term]].append(result)
                    break
                else:
                    continue<jupyter_output><empty_output><jupyter_text>* We stored this dictionary in index2.tsv file. Then we have the index needed for search engine 2 to get TfIdf for a specific word in a specific document.<jupyter_code>#put dict2 in index.tsv file. In. evry row we have a single term_id with occurences of respective word.


with open('HW3 ADM/tsv/index2.tsv', 'w', newline='') as f_output:
    tsv_index2 = csv.writer(f_output, delimiter='\t')           
    for key, val in dict2.items():
        tsv_index2.writerow([key, val])<jupyter_output><empty_output><jupyter_text>* This 'getTfidf_query' function allows to calculate TfIdf value about input query. It's needed because we must calculate coisine similiarity between result document and input query (based on TfIdf).It has input query as parameter and returns a dataframe that has tfIdf for every word in input query.<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
def getTfidf_query(query):


    vectorizer = TfidfVectorizer(token_pattern=r"(?u)\b\w+\b")
      
    vectors = vectorizer.fit_transform([query])
        #print(vectors)
    feature_names = vectorizer.get_feature_names()
    dense = vectors.todense()
    denselist = dense.tolist()
    df_query = pd.DataFrame(denselist, columns=feature_names)
    return df_query<jupyter_output><empty_output><jupyter_text>* This 'getTfidf_document' function allows to get TfIdf about every word in query for a specific document.This function searches this value from index2 (that we have just created with TfIdf values). It has as parameters: input query and document where we got TfIdf fro every word in query for this document.It returns a dataframe with these matchings.<jupyter_code>def getTfidf_document(words,document_id):
    dict={}
    
    listWords = words.split()
    listWords=[x.lower() for x in listWords]
    #print(listWords)
    df=pd.DataFrame(columns=listWords)
    tfIdf=[]
    with open('HW3 ADM/tsv/vocabulary.tsv', 'r', newline='') as f_output:
        tsv_vocabulary = list(csv.reader(f_output, delimiter='\t'))
        with open('HW3 ADM/tsv/index2.tsv', 'r', newline='') as f_output:
            tsv_index = list(csv.reader(f_output, delimiter='\t'))
            
        for word in listWords:
            listDoc=[]
            #print("word " + word)
            for row in tsv_vocabulary:
                if word.lower()==row[0]:
                    term=row[1]
                    #print("teerm" + str(term))
                    break
            for row in tsv_index:
                if term==row[0]:

                    listDoc=ast.literal_eval(row[1])

                    #print(listDoc)
                    break
            for index in listDoc:

                #print(index)
                if index[0]==document_id:
                    #print(index[0])

                    tfIdf.append(index[1])
                    #print(index[1])
                    break

        df.loc[0]=tfIdf
        df=df.reindex(sorted(df.columns), axis=1)
        return df<jupyter_output><empty_output><jupyter_text>'coisine' function allows to calculate coisine similairity from two list of TfIdf values. First list is about TfIdf values for input query calcuated by 'getTfidf_query' function. Second list is about TfIdf values for query input in document, calculated by 'getTfidf_document' function<jupyter_code>from sklearn.metrics.pairwise import cosine_similarity
def coisine(list_query,list_document):
    
    res=(cosine_similarity([list_query,list_document]))
    #print(res)
    return res[0][1]<jupyter_output><empty_output><jupyter_text>This 'searchEngine2' function allows to calculate the result for the second search engine.We call 'getDocuments(words,1)' to get all documents that contain input query.Then we call 'getTfidf_query(words)' to put in 'df' datframe all TfIdf values froi every word in input query.Then, for every document, we call 'getTfidf_document' and put its result in 'df_document' dataframe.Finally we call 'coisine' fucntion to get coisine value for every input and put its result in a final 'df' dataframe with other info(we get these infos from tsv file (intro, title, and url)). It's ordered by this similiarity value.<jupyter_code>import pandas as pd

def searchEngine2(words):
    #build the dataframe with info for every documents_id
    document=getDocuments(words,1)
    if document=='No results':
        
        return document
    elif document==[]:
        return "No results"
    df_query=getTfidf_query(words)
    df=pd.DataFrame(columns=['title', 'intro', 'url','similarity'])

    for index in range(len(document)):
        #get id of documnets_is
        df_document=getTfidf_document(words,document[index])
        
        
        #get Tfidf list from df_query dataframe
        list_query=list(df_query.loc[0])
        #get Tfidf list from df_document dataframe
        list_document=list(df_document.loc[0])
        
        similiarity=coisine(list_document,list_query)
        
        numberDocument=document[index][9:]
        #get wikipedia url
        
        url=totalMovies[int(numberDocument)]
        name="article_"
        extension2=".tsv"
        index=int(numberDocument)
        file="{}{}{}".format(name,index,extension2)
        #get info about title and intro for evert film that corresponds to every documents_id
        with open('HW3 ADM/tsv/'+file, 'r', newline='') as f_output:
            tsv_index = list(csv.reader(f_output, delimiter='\t'))
            title=tsv_index[1][3]
            intro=tsv_index[1][1]
            film=[title,intro,url,similiarity]
            #put all info for every film in a single row of df dataframe
            df.loc[index] = film
            df=df.sort_values(by=['similarity'],ascending=False)
    return df<jupyter_output><empty_output><jupyter_text>Execute query:<jupyter_code>query='In the United States 2019'

searchEngine2(query)
searchEngine1(query)<jupyter_output><empty_output><jupyter_text>Define a new score, Search Engine 3: We did 3rd search engine with "zone index" methods. We get info about this metohod from this link: https://moz.com/blog/search-engine-algorithm-basics.We conseidered the following sections for every movie: title, intro, plot and music. This method consists on assigning a fixed score. In our case we have:* title: 0.9* intro: 0.4* plot: 0.3* music: 0.6Then, we search all documents that contain input query (in title/intro/plot/music).For every document we get its score that we calculate in the following way:This score is a sum of different score in every section. You can sum the score for every section if and only if:* score about titile: if query contains the whole title;* score about intro/plot: if intro/plot contains the whole query;* score about music: if at least one word of the query is in music section.With this scoring we should give more importance to music and title section. We have choosen to give this score in the previous way about music because we want to sum this score when only name or surname (or both) of music compositor is in the input query. About title, we want to give this score becuase title must be totally in the input query.About intro and plot, we have given these score because we think that intro contains more significant words than words in plot. We created index for the search engine 3 and we put it in index3.tsv file. In this index, we considered also 'music' section to match document for every word. It's similiar to index1.tsv.<jupyter_code>import ast
from itertools import islice
import csv

#create index and save it on index.tsv                

dict2 = {}
count=0
present=False
with open('HW3 ADM/ind/vocabulary.tsv', 'r', newline='') as f_output:
        tsv_vocabulary = list(csv.reader(f_output, delimiter='\t'))
        name="article_"
        extension2=".tsv"
        h=0
        for row in tsv_vocabulary:
            
            dict2[row[1]]=[]
        for index in range(len(totalMovies)):
            
            print(index)
            file="{}{}{}".format(name,index,extension2)
            with open("HW3 ADM/tsv_correct/"+file,"r") as tsvfile:
                data_list = list(csv.reader(tsvfile, delimiter="\t"))
                tsvreader = csv.reader(tsvfile, delimiter="\t")
                intro=data_list[1][1]
                intro = ast.literal_eval(intro)
                plot=data_list[1][2]
                plot = ast.literal_eval(plot)
                music=data_list[1][8]
                music = ast.literal_eval(music)
                text=plot+intro+music
                text= list(set(map(str.lower, text)))
                
                #for evry words in plot adn intro (for every page) we get every word. From every word we get its term_id and put it whit their occurences (document_id) in dict2
                for i in text:
                    for row in tsv_vocabulary:
                        if i==row[0]:
                            doc="document_"
                            name2="{}{}".format(doc,index)
                            
                            dict2[row[1]].append(name2)
                            break
                        else:
                            continue
                            
        #put dict2 in index.tsv file. In. evry row we have a single term_id with occurences of respective word.
        with open('HW3 ADM/tsv/index3.tsv', 'w', newline='') as f_output:
            tsv_vocabulary = csv.writer(f_output, delimiter='\t')           
            for key, val in dict2.items():
                tsv_vocabulary.writerow([key, val])<jupyter_output><empty_output><jupyter_text>The following function 'heapSortK' allows us to use a heap data structure (using heapq library) for maintaining the top-k documents. This function has a list and a k value. This value is the number of ordered results that we want to show<jupyter_code>from operator import itemgetter
import heapq
def heappush(h, item, key=lambda x: x):
    heapq.heappush(h, (key(item), item))

def heappop(h):
    return heapq.heappop(h)[1]

def heapify(h, key=lambda x: x):
    for idx, item in enumerate(h):
        h[idx] = (key(item), item)
    heapq.heapify(h)
def heapFunction(a,k): 
    df=pd.DataFrame(columns=['title', 'intro', 'plot', 'music','starring', 'score'])
    result=[]
    h = []
    for item in a:
        heappush(h, item, key=itemgetter(-1))
    #print(h)
    while h:
        result.append(heappop(h))
    
    
    j=0    #print(result)
    result.reverse()
    
    for i in range(k):
      
        if len(a)>i:
            
            df.loc[j] = result[i]
            j+=1
        else:
            break
    return df<jupyter_output><empty_output><jupyter_text>In 'search3' we have calculated the result for the 3rd search engine.Initially we have created a 'df_score' dataframe to store fixed score for every section (taht we considered). Then we call 'getDocuments(query,3)' to get all dcouments where all words in query are present on them (based on index3.tsv).For every document, we check the following aspects:* if all query words are in title section, sum score about title in 'df_score' to 'score' (final score);* if all query words are in intro section, sum score about intro in 'df_score' to 'score' (final score);* if all 'title_section' words are in input query, sum score about title in 'df_score' to 'score' (final score);* if at least one word in inout query is in 'music' section, sum score about music in 'df_score' to 'score' (final score);Final dataframe 'df' contains all movies searched with these following data (title,intro,plot,music, score). It's ordered by this score from 'heapFunction' function and it contains only first k movies. K is a value that the user has choosen. If there are less results than k, only available results are shown.<jupyter_code>import pandas as pd
def searchEngine3(query,k):
    
    document=getDocuments(query,3)
    if document=='No results':
        
        return document
    elif document==[]:
        return "No results"
    listWords = query.split()
    listWords=[x.lower() for x in listWords]
    df=pd.DataFrame(columns=['title', 'intro', 'plot', 'music','starring', 'score'])
    #Create datarame that indicates score for every section.
    df_score=pd.DataFrame(columns=['title_score', 'intro_score', 'plot_score', 'music_score'])
    scores=[0.8,0.4,0.3,0.6]
    df_score.loc[0]=scores
    resultMovies=[]
    actors=[]
    
    for index in range(len(document)):
        score=0
        lista=[]
        #get id of documnets_is
        numberDocument=document[index][9:]
        #get wikipedia url
        url=totalMovies[int(numberDocument)]
        name="article_"
        extension2=".tsv"
        index=int(numberDocument)
        file="{}{}{}".format(name,index,extension2)
        #get info about title and intro for evert film that corresponds to every documents_id
        with open("HW3 ADM/tsv_correct/"+file,"r") as tsvfile:
                    tsv_index = list(csv.reader(tsvfile, delimiter='\t'))
                    title=ast.literal_eval(tsv_index[1][3])
                    
                    intro=ast.literal_eval(tsv_index[1][1])

                    plot=ast.literal_eval(tsv_index[1][2])
                    
                    music=ast.literal_eval(tsv_index[1][8])
                    #actors.append(tsv_index[1][7])
                    
                    if (all(elem in title  for elem in listWords)) or (all(elem in listWords  for elem in title)):
                            score+=df_score.loc[0]['title_score']
                            
                    if all(elem in intro  for elem in listWords)==True:
                            score+=df_score.loc[0]['intro_score']
                            
                    if all(elem in plot  for elem in listWords)==True:
                            score+=df_score.loc[0]['plot_score']
                            
                    if any(elem in music  for elem in listWords)==True:
                        
                        score+=df_score.loc[0]['music_score']
                    

        with open('HW3 ADM/tsv/'+file, 'r', newline='') as f_output:
            tsv_file = list(csv.reader(f_output, delimiter='\t'))
            title2=tsv_file[1][3]
            intro2=tsv_file[1][1]
            plot2=tsv_file[1][2]
            music2=tsv_file[1][8]
            listActors=ast.literal_eval(tsv_file[1][7])
            actors=listActors
            film=[title2,intro2,plot2,music2,actors,score]
            resultMovies.append(film)
            
            
       
    df=heapFunction(resultMovies,k)
    actorsGraph=[]
    for index, row in df.iterrows():
        actorsGraph.append(row['starring'])
    
    graph(actorsGraph)
    return df<jupyter_output><empty_output><jupyter_text>Bonus Step: Make a nice visualization! With this following function 'graph'we did the bonus section. This function has a nested list of actor, as parameter, about every film in results of search3. With this function we calculate evry couple of actors that are in (at least) two different film through film on the search3 result. Then we show these couple in a graph wehere every couple is represented by two nodes linked with an edge. If there is no actors couple, we don't show this graph.We used 'networkx' python library, as suggested in hw track.<jupyter_code>import networkx as nx 
import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
def graph(listMovies):
    G = nx.Graph() 
    couple=[]
    count=0
    for i in range(len(listMovies)):
        for j in range(len(listMovies[i])):
            for y in range(1,len(listMovies[i])):
                couple=[listMovies[i][j],listMovies[i][y]]
                count=0
                for k in listMovies:
                    if (couple[0] in k) and (couple[1] in k):
                        
                        count+=1
                        if count>=2 and not couple[0] == couple[1]:
                            
                            G.add_edge(couple[0],couple[1])
                            break
     
 
    
    if not nx.is_empty(G):
        pos = nx.spring_layout(G)   #<<<<<<<<<< Initialize this only once
        nx.draw(G,pos=pos, with_labels=True, node_size = 100, font_size=10)
        nx.draw_networkx_nodes(G,pos=pos, with_labels=True, node_size = 1500, font_size=10)
        nx.draw_networkx_edges(G, pos, alpha=0.3)#<<<<<<<<< pass the pos variable
        #plt.draw() 
        plt.figure(figsize=(8, 8))  # image is 8 x 8 inches
         # To plot the next graph in a new figure
        plt.show()<jupyter_output><empty_output><jupyter_text>Execute query:<jupyter_code>query='United States Richard heroes'
searchEngine3(query,5)
query='United States Strong Mark '
searchEngine3(query,11)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10144603458556947, 'fraction_numerical': 0.010484993043132579, 'mean_word_length': 2.3939623914326673, 'pattern_counts': {'":': 0, '<': 116, '<?xml version=': 0, '>': 99, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '666349', 'license': 'MIT', 'n_tokens_mistral': 12252, 'n_tokens_neox': 11712, 'n_words': 3520, 'path': 'main.ipynb', 'repo_name': 'elaesme/ADM-HW3', 'size': '40248'}