<jupyter_start><jupyter_text>Similarity matrices constructionConstruction of the similarity matricies to build the adjacency matrix. Different similarity are explored : * compute separatley the 4 cosine similarities (cast, crew, keywords, genres). * compute a cosine similarity of all the features togethers* compute a cosine similarity of all the features together exclusing the genreThen the matrices are saved as pickles<jupyter_code>import pandas as pd
import numpy as np
import pickle
import json
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.metrics import pairwise_distances
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt
imdb_movies = pd.read_csv('Data/merged_data.csv', index_col=0)
imdb_movies.columns
def get_list(x):
    """
    get the list represented by string x 
    -------
    INPUTS
        |---- x : string to be converted to list of dict 
    OUTPUTS
        |---- list_ : the converted string 
    """
    list_dict = json.loads(x)
    if(list_dict):
        list_ = []
        for d in list_dict:
            list_.append(d['name'].replace(' ',''))
        return list_
    else: 
        return []

def join_list(x):
    """
    Join the list elements of x
    """
    return ' '.join(x)
def get_similarity(column, dataset, dist_metric='cosine'):
    '''Function for computing the adjacency matrices
    ------
    INPUTS
        |---- column: name of the column we want the adjacency matrix for
        |---- dataset: imdb dataset containing the needed column (imdb_movies or imdb_credits)
        |---- dist_metric : the distance metric to use (from sklearn.metric.pairwise)
    OUTPUT
        |---- csim: sclaed cosine similarity distance adjacency matrix
    '''
    
    # Make a temporary copy for computation
    data_copy = dataset.copy()
    # Modify column so we can compute similarity
    data_copy[column] = data_copy[column].apply(get_list)
    data_copy[column] = data_copy[column].apply(join_list)
    
    # Vectorize and compute similarity
    vectorizer = TfidfVectorizer()
    vectors = vectorizer.fit_transform(data_copy[column]).toarray()
    
    #csim = cosine_similarity(vectors)
    csim = pairwise_distances(vectors, metric=dist_metric, n_jobs=-1)
    csim = csim.max() - csim
    # Min-max scaling
    csim[np.diag_indices(csim.shape[0])] = 0
    csim = csim/csim.ptp()
    # transformation to DataFrame
    csim = pd.DataFrame(csim, index=data_copy['title'], columns=data_copy['title'])
    
    return csim

def get_similarity_all(columns, dataset, dist_metric='cosine'):
    '''Function for computing the adjacency matrices
    ------
    INPUTS
        |---- columns: names of the columns we want the adjacency matrix for
        |---- dataset: imdb dataset containing the needed column (imdb_movies or imdb_credits)
        |---- dist_metric : the distance metric to use (from sklearn.metric.pairwise)
    OUTPUT
        |---- csim: sclaed cosine similarity distance adjacency matrix
    '''
    
    # Make a temporary copy for computation
    data_copy = dataset.copy()
    vect = []
    for column in columns:
        # Modify column so we can compute similarity
        data_copy[column] = data_copy[column].apply(get_list)
        #items = sorted(data_copy.explode(column=column)[column].unique())
        data_copy[column] = data_copy[column].apply(join_list)

        # Vectorize and compute similarity
        vectorizer = TfidfVectorizer()
        vect.append(vectorizer.fit_transform(data_copy[column]).toarray())
    # stack all vectors
    vectors = np.concatenate(vect, axis=1)
    
    csim = pairwise_distances(vectors, metric=dist_metric, n_jobs=-1)
    csim = csim.max() - csim # get similarity from distance
    # Min-max scaling
    csim[np.diag_indices(csim.shape[0])] = 0 # remove self similarity
    csim = csim/csim.ptp()
    
    # transformation to DataFrame
    csim = pd.DataFrame(csim, index=data_copy['title'], columns=data_copy['title'])
    
    return csim
# Similarity matrices generation
csim_genre = get_similarity('genres', imdb_movies, dist_metric='cosine') # Movie genre (e.g Action, Drama...)
csim_keywords = get_similarity('keywords', imdb_movies, dist_metric='cosine') # Movie kewords
csim_cast = get_similarity('cast', imdb_movies, dist_metric='cosine') # Movie cast (actors)
csim_crew = get_similarity('crew', imdb_movies, dist_metric='cosine') # Movie cast (realisator, productor...)
csim_all = get_similarity_all(['genres', 'keywords', 'cast', 'crew'], imdb_movies, dist_metric='cosine') # all columns together 
csim_not_genre = get_similarity_all(['keywords', 'cast', 'crew'], imdb_movies, dist_metric='cosine')  # all columns toghter but genre 
# visualize de similarities
fig, axs = plt.subplots(6, 1,figsize=(10, 14))
titles = ['Genre', 'Keywords', 'Cast', 'Crew', 'All', 'All but genre']
adj_mat = [csim_genre, csim_keywords, csim_cast, csim_crew, csim_all, csim_not_genre]
for name, mat, ax in zip(titles, adj_mat, axs.reshape(-1)):
    ax.hist(mat.values.ravel(), log=True, bins=100, color='Goldenrod', linewidth=1, edgecolor='black')
    ax.set_title(name)
    ax.set_xlabel('Scaled Cosine Similarity')
    ax.set_ylabel('Log scaled quantity')
    ax.set_xlim([0,1])
fig.tight_layout()
plt.show()
# save the similarity matrices as pickles
csim_genre.to_pickle('Data/csim_genre')
csim_keywords.to_pickle('Data/csim_keywords')
csim_cast.to_pickle('Data/csim_cast')
csim_crew.to_pickle('Data/csim_crew')
csim_all.to_pickle('Data/csim_all')
csim_not_genre.to_pickle('Data/csim_not_genre')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11319419879731164, 'fraction_numerical': 0.0038910505836575876, 'mean_word_length': 3.8582474226804124, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '660724', 'license': 'Apache-2.0', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1791, 'n_words': 465, 'path': 'Project/Similarity matrices construction.ipynb', 'repo_name': 'carparel/NTDS', 'size': '5654'}