<jupyter_start><jupyter_text>0.1. Import Packages<jupyter_code># Run cell
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
import os
import csv
import re
import numpy as np
import pandas as pd
import json
from scipy import spatial
import spacy
import collections
import heapq
from IPython.display import HTML
from bs4 import BeautifulSoup
import time
from selenium import webdriver
from webdriver_manager.firefox import GeckoDriverManager
import webbrowser
from langdetect import detect
from multiprocessing import Pool
#import multi_processing_functions
import dask.dataframe as ddf
# Run cell
from __future__ import print_function
from dask.distributed import Client
client = Client()
client<jupyter_output>C:\Users\juanm\Anaconda3\envs\data_science_general\lib\site-packages\distributed\dashboard\core.py:79: UserWarning: 
Port 8787 is already in use. 
Perhaps you already have a cluster running?
Hosting the diagnostics dashboard on a random port instead.
  warnings.warn("\n" + msg)<jupyter_text>0.2. Useful Functions (read, write, etc.) We will define a few basic functions that will be used throughout the whole notebook<jupyter_code># Run cell
def write_file(filename, content):
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, "w", encoding='utf-8' ) as f:
        f.write(str(content))
# Run cell
def write_tsv(filename, content):
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, "wt", newline='', encoding='utf-8' ) as out_file:
        tsv_writer = csv.writer(out_file, delimiter='\t')
        tsv_writer.writerow(content)
# Run cell
def read_tsv(filename, type_='utf-8'):
    # cp850
    with open(filename, encoding = type_) as tsvfile:
        reader = csv.reader(tsvfile, delimiter='\t')
        for data in reader:
            return data
# Run cell
def write_json(file_name, content):
    os.makedirs(os.path.dirname(file_name), exist_ok=True)
    with open(file_name, 'w') as outfile:
        json.dump(content, outfile, sort_keys=True, indent=4)
# Run cell
def jsonKeys2int(x):
    if isinstance(x, dict):
            return {int(k):v for k,v in x.items()}
    return x
# Run cell
def read_json(file_name):
    with open(file_name) as json_file:
        data_dict = json.load(json_file, object_hook=jsonKeys2int)
        return data_dict
# Run cell
def read_json_simple(file_name):
    with open(file_name) as json_file:
        data_dict = json.load(json_file)
        return data_dict
# Run cell
def tsv_files_to_df(path, destiny='data/tsv_files/tsv_files.tsv'):
    """  
    Convert all tsv files in the given path into a dataframe, which will also be saved as a .tsv file
    """
    tsv_files = os.listdir(path)
    list_tsv = []
    for filename in tsv_files:
        d_id = int(re.findall(r'\d+', filename)[0])
        filename = path + filename
        book_data = read_tsv(filename)
        book_data.insert(0,d_id)
        list_tsv.append(book_data)
    df = pd.DataFrame(list_tsv)
        
    df.to_csv(destiny, sep='\t')
    return df<jupyter_output><empty_output><jupyter_text>1. Data collection  1.1. Get the list of books<jupyter_code># Initalize web browser for crawling
driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
def crawl_urls(href):
    """
    Given a specific url (href) crawl through the different list items and return the page url
    """
    driver.get(href)
    time.sleep(5)
    
    page_soup = BeautifulSoup(driver.page_source, features="lxml")
    links = page_soup.find_all('a',{'class': 'bookTitle'}, itemprop="url")
    
    urls = []
    # Loop over all links in the href page
    for link in links:
        url = link.get('href')
        url = 'https://www.goodreads.com' + url
        urls.append(url)
    
    urls = '\n'.join(urls)+'\n'
    
    return urls<jupyter_output><empty_output><jupyter_text>Apply previous function over all books that we are interested in downloading<jupyter_code>url = 'https://www.goodreads.com/list/show/1.Best_Books_Ever?page='
path = 'data/book_urls.txt'

urls = ''

for i in range(0,30000):
    href = url+str(i+1)
    urls += crawl_urls(href)

write_file(path, urls)<jupyter_output><empty_output><jupyter_text>1.2. Crawl books Given a set of links (generated by the previous function) download their html individually<jupyter_code>def scrap_book(href):
    driver.get(href)       
    time.sleep(5)
    return driver.page_source
filename = 'data/book_urls.txt'

book_urls = open(filename, 'r')
for url in book_urls:
    page_number = int((count-1)/100)+1
    html = scrap_book(url)
    path = 'data/page_'+str(page_number)+'/article_'+str(count)+'.html'
    write_file(path, html)<jupyter_output><empty_output><jupyter_text>1.3 Parse downloaded pages Given all the downloaded htmls, parse them individually<jupyter_code>def remove_html_tags(text):
    """Remove html tags from a string"""
    clean = re.compile('<.*?>')
    return re.sub(clean, '', text)
def parse_html_in_folder(path):
    for html_file in os.listdir(path):
        print(html_file)
        if os.path.exists('data/final_tsv_files/' + re.findall(r'\d+', html_file)[0] + '.tsv'):
            print('not parsed')
            continue
        else:
            with open(path + '/' + html_file, encoding='utf8') as infile:
                print('parsing...')
                soup = BeautifulSoup(infile, features="lxml")
                try:
                    Plot = ' '.join([remove_html_tags(str(c)) for c in soup.find_all('div', id="description")[0].contents[3].contents ])
                except Exception:
                    if not soup.find_all('div', id="description"):
                        Plot = ''
                    else:
                        Plot = ' '.join([remove_html_tags(str(c)) for c in soup.find_all('div', id="description")[0].contents[1].contents ])
                if Plot:
                    if detect(Plot) != 'en':
                        print('Article removed:', html_file)
                        continue
                try:
                    bookTitle = soup.find_all('h1')[0].contents[0].replace('\n', '').strip()
                except:
                    print('Wrong html file')
                    continue
                bookSeries = soup.find_all('h2', id='bookSeries')[0].text.replace('\n', '').strip()
                bookAuthors = ', '.join([soup.find_all('span', itemprop='name')[i].contents[0] for i in range(
                    len(soup.find_all('span', itemprop='name')))])
                ratingValue = soup.find_all('span', itemprop='ratingValue')[0].contents[0].replace('\n', '').strip()
                ratingCount = soup.find_all('meta', itemprop="ratingCount")[0]['content']
                reviewCount = soup.find_all('meta', itemprop="reviewCount")[0]['content']
                try:
                    NumberofPages = re.findall(r'\d+', soup.find_all('span', itemprop="numberOfPages")[0].contents[0])[0]
                except:
                    if not soup.find_all('span', itemprop="bookFormat"):
                        NumberofPages = ''
                    else:
                        NumberofPages = 0
                try:
                    temp_date = soup.find_all('div', id='details')[0].find_all('div', {"class": "row"})[1].text.split('\n')[
                        2].split()
                    if not temp_date:
                        temp_date = soup.find_all('div', id='details')[0].find_all('nobr', {"class": "greyText"})[0].contents[0].split('\n')[1].split()[-3:]
                except:
                    try:
                        temp_date = soup.find_all('div', id='details')[0].find_all('div', {"class": "row"})[0].contents[0].split('\n')[
                            2].split()
                    except:
                        temp_date = ''
                PublishingDate = ' '.join(temp_date)
                characters = []
                settings = []
                for i in range(1, len(soup.find_all('div', id="bookDataBox")[0].find_all('a'))):
                    if re.match(r'/characters/', soup.find_all('div', id="bookDataBox")[0].find_all('a')[i].attrs['href']):
                        characters.append(soup.find_all('div', id="bookDataBox")[0].find_all('a')[i].text)
                    elif re.match(r'/places/', soup.find_all('div', id="bookDataBox")[0].find_all('a')[i].attrs['href']):
                        settings.append(soup.find_all('div', id="bookDataBox")[0].find_all('a')[i].text)
                characters = ', '.join(characters)
                settings = ', '.join(settings)
                url = soup.find_all('link', rel='canonical')[0].attrs['href']

                final_list = [bookTitle, bookSeries, bookAuthors, ratingValue, ratingCount, reviewCount,
                              Plot, NumberofPages, PublishingDate, characters, settings, url]

                filename = 'data/final_tsv_files/' + re.findall(r'\d+', html_file)[0] + '.tsv'

                write_tsv(filename, final_list)<jupyter_output><empty_output><jupyter_text>Apply function over all folders containing the downloaded html files<jupyter_code># None parallel approach
for i in range(0,301):
    print(i)
    parse_html_in_folder('../data_html/page_' + str(i))
# Parallel approach
if __name__ == '__main__':
    with Pool(8) as p:
        print(p.map(multi_processing_functions.parse_html_in_folder, 
                    ['../data_html/' + i for i in os.listdir('../data_html')]))
tsv_files_to_df(path='data/final_tsv_files/', destiny='data/tsv_files/final_tsv_files.tsv')
df_dirty.to_csv('data/tsv_files/final_tsv_files.tsv', sep='\t', index=False)<jupyter_output><empty_output><jupyter_text>2. Search Engine  2.0. Pre-process of information<jupyter_code># Run cell
stopwords = set(stopwords.words('english'))
tokenizer = nltk.RegexpTokenizer(r"\w+")
nlp = spacy.load("en_core_web_sm")
# Run cell
def remove_stop_words(text):
    """
    This allow us to identify stop word in english and remove them. We are also removing character with single length (e.g. "s")
    """    
    word_tokens = word_tokenize(text)
    filtered_sentence = [w.lower() for w in word_tokens if w.lower() not in stopwords and not(len(w) == 1 and w.isalpha())]

    text = ' '.join(filtered_sentence)
    return text
# Run cell
def remove_punctuation(text): 
    """
    Remove puntuation from input string
    """
    text = tokenizer.tokenize(text)
    clean_punctuation = ' '.join(text)
    return clean_punctuation
# Run cell
def remove_stemming(text):
    """
    Apply stemming procedure over input text
    """
    ps = PorterStemmer()
    words = word_tokenize(text)
    stem_sentence=[]
    
    for w in words:
        stem_sentence.append(ps.stem(w))

    text = " ".join(stem_sentence)
    return text 
# Run cell
def remove_lemma(text):
    """
    Apply lemmanization procedure over input text
    """
    doc = nlp(text)
    lemma = []
    for token in doc:
        lemma.append(token.lemma_)
    text = ' '.join(lemma)
    return text
# Run cell
def parse_pulishing_date(publishingDate):
    """
    Only keep last 4 digits of publishing Date (Year of publication)
    """
    return publishingDate[-4:]
# Run cell
def global_pre_process(text):
    """ 
    Function to process everything at once 
    """
    text = remove_punctuation(text)
    text = remove_stop_words(text)
    text = remove_lemma(text)
    # This makes sure that we also remove strange letters that have not been removed with the previous packages 
    # (e.g. arabic letters)
    text = re.sub(r'[^a-zA-Z0-9]', ' ', text).strip()
    return text
# Run cell
def clean_text(df):
    """
    Take in a Dataframe, and clean based on the previously defined functions (each column is cleaned individually)
    """
    df['BookTitle'] = df.BookTitle.map(global_pre_process)
    df['BookSeries'] = df.BookSeries.map(global_pre_process)
    df['BookAuthors'] = df.BookAuthors.map(global_pre_process)
    df['Plot'] = df.Plot.map(global_pre_process)
    df['Characters'] = df.Characters.map(global_pre_process)
    df['PublishingDate'] = df.PublishingDate.map(parse_pulishing_date)
    return df<jupyter_output><empty_output><jupyter_text>Read data frame which still has not been pre-processed<jupyter_code># Run cell
df_dirty = pd.read_csv('data/tsv_files/final_tsv_files.tsv', sep='\t', keep_default_na=False)
df_dirty.head(5)<jupyter_output><empty_output><jupyter_text>Apply clean_text function over the "dirty" dataframe using dask to optimize running time<jupyter_code>dask_dataframe = ddf.from_pandas(df_dirty, npartitions=20)
%%time
df_clean = dask_dataframe.map_partitions(clean_text, meta=df_dirty).compute()
df_clean
df_clean.to_csv('data/clean_tsv_files/clean_final.tsv', sep='\t', index=False)<jupyter_output><empty_output><jupyter_text>2.1. Conjunctive query<jupyter_code># Run cell
# Read Clean tsv file
df_clean = pd.read_csv('data/clean_tsv_files/clean_final.tsv', sep='\t', keep_default_na=False)<jupyter_output><empty_output><jupyter_text>2.1.1. Create your index!<jupyter_code>def get_vocabulary_inverted_index(df, columns):
    """
    This function returns a dictionary with all the words in the dataframe (and specifically the provided columns) 
    and its inverted index
    Example:
    vocabulary_dict = {'river': 1, 'game': 2, 'friend': 3, ...}
    inverted_index = {1: [1, 4, 7], 2: [3, 6, 9], 3: [2, 7, 8]} where the list contains the documents in which 
    the word 1 (river) appears in
    
    df: Clean Dataframe
    columns: Columns over which the vocabulary and inverted index dictionaries will be generated
    """
    vocabulary = {}
    count = 1
    inverted_index = {}
    for index, row in df.iterrows():
        d_id = row['BookID']
        if isinstance(columns, list):
            text = (' '.join([row[i] for i in columns])).split(' ')
        else:
            raise('Column must be a list')
            
        for word in text:
            if word not in vocabulary: 
                vocabulary[word] = count
                inverted_index[count] = [d_id]
                count +=1
            else:
                key = vocabulary[word]
                if d_id not in inverted_index[key]:
                     inverted_index[key].append(d_id)
    return vocabulary, inverted_index
%%time
vocabulary, inverted_index = get_vocabulary_inverted_index(df_clean, columns=['Plot'])<jupyter_output>Wall time: 37.2 s<jupyter_text>Save dictionaries as json files for future use<jupyter_code>write_json('data/inverted_index.json', inverted_index)
write_json('data/vocabulary_dict.json', vocabulary)<jupyter_output><empty_output><jupyter_text>2.1.2. Execute the query Functionality to get intersection of documents in which query appears<jupyter_code># Run cell
def get_pointer_values(pointer, index_list):
    """ Based on a set of pointer values get the documents """
    values = []
    for i in range(len(pointer)):
        values.append(index_list[i][pointer[i]])
    return values
# Run cell
def update_pointer(values, pointer):
    """ Given the values, compute the minimum and update the pointer accordingly based on their minimum """
    mins = np.where(values == np.min(values))[0]
    for i in range(0, len(mins)):
        pointer[mins[i]] = pointer[mins[i]] + 1 
    return pointer
# Run cell
def query_function(query, index, vocabulary):
    """ 
    Given a query find the documents in which these appear based on the index 
    query: query string
    index: inverted index as dictionary
    vocabulary: vocabulary dictionary
    
    """
    
    # Pre-process query 
    query = global_pre_process(query)
    
    # Query to list of strings
    query_list = query.split()
    
    # Map strings to integer based on dict
    try:
        integer_list = [vocabulary[i] for i in query_list]
    except:
        return []
    
    # Start to look for the intersection of the query in the index
    total_query_documents = [sorted(index[i]) for i in integer_list]
    
    # Generate a list with the pointer values
    pointers = np.full(len(total_query_documents), 0)
    values = np.full(len(total_query_documents), 0)
    
    # List where intersection documents will be stored
    intersection = []

    # Compute the document in which the search should stop
    max_list = np.array([max(total_query_documents[i]) for i in range(len(total_query_documents))])

    try:
        # Loop over all elements stopping at the minimum between all documents
        while np.any(values != max_list):
            # Get the documents based on the pointer
            values = get_pointer_values(pointer = pointers, 
                                        index_list = total_query_documents)
            # If all values are equal we have found a match and all the pointer values are increased by one
            if len(set(values)) == 1:
                intersection.append(values[0])
                pointers += 1
            # If all values are not equal increase the values of the minimum pointers
            else:
                pointers = update_pointer(values, pointers)
    except:
        intersection = sorted(list(set.intersection(*map(set,total_query_documents))))
    
    assert intersection == sorted(list(set.intersection(*map(set,total_query_documents)))), 'Algorithm is not returning same result as python implementation'
    
    return intersection
# Run cell
def path_to_image_html(path):
    return '<img src="'+ path + '" style=max-height:124px;"/>'
# Run cell
def show_results(book_ids, df):
    """
    Get relevant information which will be shown in the final dataframe for the books in book_ids
    df: This dataframe should not be pre-processed
    book_ids: list of books
    """
    output = df[df['BookID'].isin(book_ids)][['BookTitle', 'Plot', 'Url']]
    return output
# Run cell
def search_engine_1(query, inverted_index, vocabulary, df):
    """
    Basic search engine which returns all books with the provided query
    query: Query of user
    df: This dataframe should not be pre-processed
    """
    query_results = query_function(query, inverted_index, vocabulary)
    if len(query_results) == 0:
        print('There are no results for the search')
    else:
        output = show_results(query_results, df)
        output = HTML(output.to_html(escape=False,
                                     formatters=dict(column_name_with_image_links=path_to_image_html)))
        return output<jupyter_output><empty_output><jupyter_text>Read json files (vocabulary and inverted index)<jupyter_code># Run cell
inverted_index = read_json('data/inverted_index.json')
vocabulary = read_json_simple('data/vocabulary_dict.json')<jupyter_output><empty_output><jupyter_text>Run First Search Engine<jupyter_code># Run cell
input_query = 'harry potter magic hogwarts'
# Run cell
search = search_engine_1(input_query, inverted_index, vocabulary, df_dirty)
search<jupyter_output><empty_output><jupyter_text>2.2. Conjunctive query & Ranking score  2.2.1. Inverted index<jupyter_code>def vectorize_tfidf(df, vocabulary, inverted_index, json_name='tfidf.json', columns=['Plot']):
    '''
    Vecterize Plots
    This function, given a vocabulary dictionary, inverted index and pre-processed dataframe return a dictionary with 
    tfidf scores
    Example: {1: {1: 0.7, 5: 3.7}, 2: {3: 1.7, 6: 5.7}} Where 1 and 2 denote the book_id and 1, 5, 3, 6 denote the word.
    df: Pre-processed data frame (df_clean)
    param column: If a list is provided the score will be computed over several columns
    '''

    no_of_documents = len(df)
    
    # number of words in vacabulary
    no_of_words_in_vocab = len(vocabulary)
    
    tfidfDicts = {}
    
    for index, row in df.iterrows():
        d_id = row['BookID']

        if isinstance(columns, list):
            text = (' '.join([row[i] for i in columns])).split(' ')
        else:
            raise('Column must be a list')
            
        no_of_words_in_plot = len(text)
        # Create a vector
        tfDict = dict.fromkeys((i for i in range(1, no_of_words_in_vocab+1)), 0)
        
        
        for word in text:
            index = vocabulary[word]
            tfDict[index] +=1
        
        tfidfDict = {}
        
        for key, value in tfDict.items():
            if value != 0:
                
                no_of_documents_appeared = len(inverted_index[key])

                tfidf = (value/no_of_words_in_plot) * np.log(no_of_documents/no_of_documents_appeared)

                tfidfDict[key] = float('{:.4f}'.format(tfidf))
                        
        tfidfDicts[d_id] = tfidfDict
        
    documents = collections.OrderedDict(sorted(tfidfDicts.items()))
    write_json('data/' + json_name, documents)
%%time
# Generate tfidf based only on the 'Plot' of the books (vocabulary and inverted index have been generated only over PLot as 
# well) 
vectorize_tfidf(df_clean, vocabulary, inverted_index, json_name='tfidf.json', columns=['Plot'])
# Run cell
def get_cosine(doc, query):
    """
    Given two vectors, return a float which is the cosine similarity score
    doc: dictionary vector 
    query: dictionary vector
    """
    intersection = set(doc.keys()) & set(query.keys())
    numerator = sum([doc[x] * query[x] for x in intersection])

    sum1 = sum([doc[x] ** 2 for x in list(doc.keys())])
    sum2 = sum([query[x] ** 2 for x in list(query.keys())])
    denominator = np.sqrt(sum1) * np.sqrt(sum2)

    if not denominator:
        return 0.0
    else:
        return float(numerator) / denominator
# Run cell
def show_results_cosine_similarity(book_id, cosine_similarity, df):
    """
    Generate a tuple with the relevant information about a book that will be displayed in the search engine, 
    and the cosine similarity score
    df: Not pre-processed data set
    """
    data = df[df.BookID == book_id][['BookTitle', 'Plot', 'Url']].values.tolist()[0]
    output = (cosine_similarity, data)
    return output
# Run cell
def search_engine_2(query, inverted_index, vocabulary, tfidf_scores_dict, df, k = 10):
    """
    Cosine similarity search engine which returns all books with the provided query in order of highest cosine 
    value, and displaying top k books
    
    query: Query of user
    df: This dataframe should not be pre-processed
    """
    output = pd.DataFrame(columns=['BookTitle', 'Plot', 'Url', 'Similarity'])
    documents_with_query_words = query_function(query, inverted_index, vocabulary)
    queryed_documents_tfidf = {key: value for key, value in tfidf_scores_dict.items() if key in documents_with_query_words}
    heap_data = []
    
    if len(documents_with_query_words) == 0:
        print('There are no results for the search')
    else:
    
        # pre-process query
        query = global_pre_process(query)

        # vectorize query
        vector_query = {}
        for word in query.split(' '):
            index = vocabulary[word]
            vector_query[index] = 1

        for i in queryed_documents_tfidf.keys():
            similarity = get_cosine(queryed_documents_tfidf[i], vector_query)
            x = show_results_cosine_similarity(i, similarity, df)
            if len(heap_data) < k:
                heapq.heappush(heap_data, x)
            else:
                heapq.heappushpop(heap_data, x)

        for i in range(len(heap_data)):
            output = output.append(pd.Series([heap_data[-(i+1)][1][0], heap_data[-(i+1)][1][1], 
                                              heap_data[-(i+1)][1][2], heap_data[-(i+1)][0]], 
                                             index=output.columns), ignore_index=True) 
        output = output.sort_values(by='Similarity', ascending=False)
        output = HTML(output.to_html(escape=False,
                                     formatters=dict(column_name_with_image_links=path_to_image_html)))
        return output
# Run cell
tfidfDicts = read_json('data/tfidf.json')
# Run cell
input_query = 'break heart'
# Run cell
search_engine_2(query = input_query, inverted_index = inverted_index, 
                vocabulary=vocabulary, tfidf_scores_dict=tfidfDicts,
                df = df_dirty, k = 10)<jupyter_output><empty_output><jupyter_text>3. Define a new score! Based on the information we have at our disposal, we have decided to explore two different approaches for our final search engine:1. Weighted average of the cosine_similarity, ratingValue, ratingCount and reviewCount. This is not the ideal scenario since weights are not justifiable without any data on the users search history (weights are completely based on human judgement). With more information on the user history we could continuously update the weights.\begin{align}score_{new} = \frac{CosineSimilarity \cdot \omega_{1} + ratingValue \cdot \omega_{2} + ratingCount \cdot \omega_{3} + reviewCount \cdot \omega_{4}}{max(ratingValue) + max(ratingCount) + max(reviewCount)}\end{align}2. Normalizing the variables ratingValue, ratingCount and reviewCount upon their maximum values of the queried book ids and multiply their sum against the cosine similarity value.\begin{align}score_{new} = CosineSimilarity \cdot \bigg(\frac{ratingValue}{max(ratingValue)} + \frac{ratingCount}{max(ratingCount)} + \frac{reviewCount}{max(reviewCount)} \bigg)\end{align}<jupyter_code>%%time
vocabulary_question3, inverted_index_question3 = get_vocabulary_inverted_index(df_clean,
                columns=['BookTitle', 'BookSeries', 'BookAuthors', 'Plot', 'PublishingDate', 'Characters'])

write_json('data/inverted_index_question3.json', inverted_index_question3)
write_json('data/vocabulary_dict_question3.json', vocabulary_question3)

vectorize_tfidf(df_clean, vocabulary_question3, 
                inverted_index_question3, json_name='tfidf_question3.json',
                columns=['BookTitle', 'BookSeries', 'BookAuthors', 'Plot', 'PublishingDate', 'Characters'])
# Run cell
def show_results_cosine_similarity_and_ratings(book_id, cosine_similarity, df):
    """
    Generate a tuple with the relevant information about a book that will be displayed in the search engine, 
    the cosine similarity score and relevant quantitative information about the book (rating, reviewcount, ratingcount)
    df: Not pre-processed data set
    """
    data = df[df.BookID == book_id][['BookTitle', 'Plot', 'Url']].values.tolist()[0]
    ratingValue = float(df[df.BookID == book_id][['RatingValue']].values[0])
    ratingCount = float(df[df.BookID == book_id][['RatingCount']].values[0])
    reviewCount = float(df[df.BookID == book_id][['ReviewCount']].values[0])
    output = (cosine_similarity, ratingValue, ratingCount, reviewCount, data)
    return output
# Run cell
inverted_index_question3 = read_json('data/inverted_index_question3.json')
vocabulary_question3 = read_json_simple('data/vocabulary_dict_question3.json')
tfidfDicts_question3 = read_json('data/tfidf_question3.json')
# Run cell
def search_engine_3(query, inverted_index, vocabulary, tfidf_scores_dict, df, k = 10,
                    new_score='cosine_normalizer', weights=None):
    """
    df: Not pre-processed data set!
    There are currently two alterantives for the computation of the new score:
    - cosine_normalizer: Normalize all quantitative values and multiply against cosine similiarity
    - weighted_average: Give weights to all features based on expert judgement (provide weights as list required!!)
    - weights: list with weights:
        weights[0]: weight for cosine_similiarity
        weights[1]: weight for ratingValue
        weights[2]: weight for ratingCount
        weights[3]: weight for reviewCount

    """
   
    output = pd.DataFrame(columns=['BookTitle', 'Plot', 'Url', 'Score'])
    documents_with_query_words = query_function(query, inverted_index, vocabulary)
    queryed_documents_tfidf = {key: value for key, value in tfidf_scores_dict.items() if key in documents_with_query_words}
    heap_data = []
    
    if len(documents_with_query_words) == 0:
        print('There are no results for the search')
    else:
        
        # pre-process query
        query = global_pre_process(query)

        # vectorize query
        vector_query = {}
        for word in query.split(' '):
            index = vocabulary[word]
            vector_query[index] = 1

        # Get max_ratingCount and max_reviewCount
        ratingValue_list = []
        ratingCount_list = []
        reviewCount_list = []
        ratings_df = df[df.BookID.isin(documents_with_query_words)]

        max_ratingValue = max(list(map(float, df.RatingValue)))
        max_ratingCount = max(list(map(int, df.RatingCount)))
        max_reviewCount = max(list(map(int, df.ReviewCount)))

        # Compute cosine over all intersected documents
        for i in queryed_documents_tfidf.keys():
            similarity = get_cosine(queryed_documents_tfidf[i], vector_query)
            temp = show_results_cosine_similarity_and_ratings(i, similarity, df)
            if new_score == 'weighted_average':
                # temp[0] = cosine_similarity
                # temp[1] = rating
                # temp[2] = ratingCount
                # temp[3] = reviewCount
                # temp[4] = Relevant book information ([booktitle, plot, url])
                score = (temp[0]*weights[0] + temp[1]*weights[1] + temp[2]*weights[2] + temp[3]*weights[3])/np.sum([max_ratingValue, 
                                                                                                                    max_ratingCount, 
                                                                                                                    max_reviewCount])
                x = (score, temp[4])
            elif new_score == 'cosine_normalizer':
                # temp[0] = cosine_similarity
                # temp[1] = rating
                # temp[2] = ratingCount
                # temp[3] = reviewCount
                # temp[4] = Relevant book information ([booktitle, plot, url])
                score = temp[0]*(temp[1]/max_ratingValue + temp[2]/max_ratingCount + temp[3]/max_reviewCount)
                x = (score, temp[4])
            else:
                raise('New score method is not implemented')

            if len(heap_data) < k:
                heapq.heappush(heap_data, x)
            else:
                heapq.heappushpop(heap_data, x)


        for i in range(len(heap_data)):
            output = output.append(pd.Series([heap_data[-(i+1)][1][0], heap_data[-(i+1)][1][1], 
                                              heap_data[-(i+1)][1][2], heap_data[-(i+1)][0]], 
                                             index=output.columns), ignore_index=True) 
        output = output.sort_values(by='Score', ascending=False)
        output = HTML(output.to_html(escape=False,
                                     formatters=dict(column_name_with_image_links=path_to_image_html)))
        return output
# Run cell
input_query = 'friends in love'<jupyter_output><empty_output><jupyter_text>First try using the cosine_normalizer approach<jupyter_code># Run cell
search_engine_3(query = input_query, inverted_index = inverted_index_question3, 
                vocabulary=vocabulary_question3, tfidf_scores_dict=tfidfDicts_question3,
                df = df_dirty, k = 10,
                new_score='cosine_normalizer')<jupyter_output><empty_output><jupyter_text>Second try using the weighted_average approach<jupyter_code># Run cell
search_engine_3(query = input_query, inverted_index = inverted_index_question3, 
                vocabulary=vocabulary_question3, tfidf_scores_dict=tfidfDicts_question3,
                df = df_dirty, k = 10,
                new_score='weighted_average', weights=[0.5, 0.2, 0.2, 0.1])<jupyter_output><empty_output><jupyter_text>4. Make a nice visualization!<jupyter_code># Run cell
def get_book_series(df, num_series=20, series_to_include=['<NAME>']):
    """ 
    Get first num_series Book Series based on the order of apperance (also including series in the list series_to_include)
    df: Not pre-processed data set!
    """
    bookSeries = {}
    for index, row in df.iterrows():
        d_id = row['BookID']
        book_data = row
        clean_series = re.sub(r'[^a-zA-Z0-9]', ' ', book_data['BookSeries']).split()
        series_name = re.sub(r'[^a-zA-Z]', ' ', book_data['BookSeries']).rstrip().lstrip()
        # If the book is part of a series and the series is one single book
        if (series_name != '') & (len([i for i in clean_series if bool(re.match(r'\d+', i))]) == 1):
            if series_name not in bookSeries:
                # Make sure we only take the first 20 series
                if (len(bookSeries.keys()) < num_series) | (series_name in series_to_include):
                    split_date = re.findall(r'\d+', book_data['PublishingDate'])
                    year = [i for i in split_date if len(i) == 4][0]
                    bookSeries[series_name] = [[' '.join(clean_series), year, book_data['NumberofPages'], book_data['Url']]]
            else:                
                split_date = re.findall(r'\d+', book_data['PublishingDate'])
                try:
                    year = [i for i in split_date if len(i) == 4][0]
                except:
                    year = book_data[8]
                bookSeries[series_name].append([' '.join(clean_series), year, book_data['NumberofPages'], book_data['Url']])

    return bookSeries
# Run cell
book_series = get_book_series(df_dirty)
# Run cell
from ipywidgets import interact, Dropdown

dropdown_bookSeries = Dropdown(options = list(book_series.keys()))

def plot_series(series_name, bookSeries_dict = book_series):
    publish_years = [i[1] for i in bookSeries_dict[series_name]]
    pages = [i[2] for i in bookSeries_dict[series_name]]
    df = pd.DataFrame(columns=['Year', 'Pages'])
    df['Year'] = [int(i) for i in publish_years]
    df['Pages'] = [int(i) for i in pages]
    df = df.sort_values(by='Year', ascending=True)
    df['Years Since Publishment'] = df.Year - df.Year.min()
    df['Pages of Book Series'] = df.Pages.cumsum()
    df.plot(x = 'Years Since Publishment', y = 'Pages of Book Series', title = series_name,
            figsize=(12,8))
    print(df)
    
@interact(series_name = dropdown_bookSeries)
def dropdown_series(series_name):
    plot_series(series_name)<jupyter_output><empty_output><jupyter_text>5. Algorithmic Question  Write a recursive program that, given a string, computes the length of the subsequence of maximum length that is in alphabetical order. Try some examples. Are the examples of short strings correct? Can you find examples that your algorithm does not terminate in reasonable time?<jupyter_code>def recursive_function_terminates_n(X, n):
    """
    Compute the longest subsequence in alphabetical order that terminates at position n of the string X
    X: string
    n: len string    
    """
    if n == 1:
        return 1
    else:
        return 1 + max([recursive_function_terminates_n(X[:-i], len(X[:-i])) for i in range(1, n) if X[n-1] > X[-(i+1)]], 
                       default=0)
    
def recursive_function(X):
    """
    Function to loop over all letter in the word X and compute the real longest subsequence in 
    alphabteical order of the whole string
    """
    len_list = []
    for i in range(len(X)):
        len_list.append(recursive_function_terminates_n(X[:i+1], len(X[:i+1])))
    return max(len_list)<jupyter_output><empty_output><jupyter_text>Write a program that computes the length of the subsequence of maximum length, using dynamic programming.<jupyter_code>def dynamic_function(X, print_max_len = False): 
    """ 
    X is a string
    """
    m = len(X)

    L = [X[0]]

    # We going letter by letter adding the new letter to all possible alphabetically ordered strings
    for i in range(1, m):
        L =  L + [X[i]] + [j + X[i] for j in L if j[-1] < X[i]]
        
    # Conditions t print all the max length strings or not
    L_len = [len(j) for j in L]
    if print_max_len:
        L_max = [j for j in L if len(j) == max(L_len)]
        return max(L_len), L_max
    else:
        return max(L_len)<jupyter_output><empty_output><jupyter_text>Quick proof that both functions return the same results<jupyter_code>import random 
import string

def get_string(n):
    
    return  ''.join(random.choice(string.ascii_uppercase) for _ in range(n))

X = get_string(25)

res_1 = recursive_function(X)
res_2 = dynamic_function(X)
assert res_1 == res_2<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11260200449093598, 'fraction_numerical': 0.009036639465469084, 'mean_word_length': 2.931847545219638, 'pattern_counts': {'":': 3, '<': 102, '<?xml version=': 0, '>': 99, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '209997', 'license': 'MIT', 'n_tokens_mistral': 11570, 'n_tokens_neox': 11063, 'n_words': 2866, 'path': '1 - Which book would you recomend?/main.ipynb', 'repo_name': 'qtt-alessandro/Machine-Learning-Projects-in-Python-', 'size': '36524'}