<jupyter_start><jupyter_text>MCA: Multiple Correspondence Analysis<jupyter_code>import json
import pandas as pd
# ! pip install prince
import prince
from sklearn.preprocessing import OneHotEncoder
import itertools
omdb = json.load(open("../../../../data/parsed/omdb.json", "r") )
tmdb = json.load(open("../../../../data/parsed/tmdb.json", "r") )
categorical = {
    'omdb': ['Rated', 'Director', 'Genre', 'Language', 'Country', 'Type', 'Production'],
}
def apply_categorical(records, type, take):
    res = {i: {} for i in records.keys()}
    for row in records.keys():
        for col in records[row][type].keys():
            if col in take:
                res[row][col] = records[row][type][col]
    return res

def apply_split(records, split, limit):
    for row in records.keys():
        for col in split:
            records[row][col] = tuple(records[row][col].split(', '))
    return records
cat = apply_categorical(omdb, 'omdb', categorical['omdb'])
cat = apply_split(cat, ['Country', 'Language', 'Genre'], 3)
catdf = pd.DataFrame.from_dict(cat).T<jupyter_output><empty_output><jupyter_text>TopK One Hot<jupyter_code>def one_hot(arr, name, categories):
    return dict((name+i, i in arr) for i in categories)

def apply_one_hot(records, type, name, categories):
    for row in records.keys():
        records[row] = {**records[row], **one_hot(records[row][type], name, categories)}
        del records[row][type]
    return records

genres_cat = list(set(itertools.chain(*tuple(catdf.Genre.unique()))))
language_cat = pd.Series(list(itertools.chain(*catdf.Language))).value_counts()[:30].index
countries_cat = pd.Series(list(itertools.chain(*catdf.Country))).value_counts()[:30].index

cat = apply_one_hot(cat, 'Genre', 'g_', genres_cat)
cat = apply_one_hot(cat, 'Country', 'c_', countries_cat)
cat = apply_one_hot(cat, 'Language', 'l_', language_cat)
catdf = pd.DataFrame.from_dict(cat).T
catdf.Rated = catdf.Rated.fillna('Not rated')
catdf.Rated[catdf.Rated == 'N/A'] = 'Not rated'
catdf.Production.fillna('-')
catdf.Production[catdf.Production == 'N/A'] = '-'
catdf.Production[catdf.Production == 'NaN'] = '-'
catdf.Production[catdf.Production.isna()] = '-'
catdf.Director.fillna('-')
catdf.Director[catdf.Director == 'N/A'] = '-'
catdf
mca = prince.MCA(
        n_components=16,
        n_iter=20,
        copy=True,
        check_input=True,
        engine='auto',
)
mca = mca.fit(catdf)
%matplotlib inline
ax = mca.plot_coordinates(
        X=catdf,
        ax=None,
        figsize=(6, 6),
        show_row_points=True,
        row_points_size=10,
        show_row_labels=False,
        show_column_points=True,
        column_points_size=30,
        show_column_labels=False,
        legend_n_cols=1
)
mca.total_inertia_
mca.explained_inertia_
transformed = mca.transform(catdf)
transformed.head()
transformed.to_csv('../../../../data/engineering/mca.csv', index=True, index_label='idx')<jupyter_output><empty_output><jupyter_text>Probabilistic PCA with missing data<jupyter_code>import json
import pandas as pd
# ! pip install ppca
import ppca
import numpy as np
omdb = json.load(open("../../../../data/parsed/omdb.json", "r") )
tmdb = json.load(open("../../../../data/parsed/tmdb.json", "r") )
numerical = {
    'omdb': ['Year', 'Ratings', 'Metascore', 'imdbRating', 'imdbVotes'],
    'tmdb': ['budget', 'popularity', 'revenue', 'runtime', 'vote_average', 'vote_count']
}
def apply_numerical(records, type, take):
    res = {i: {} for i in records.keys()}
    for row in records.keys():
        for col in records[row][type].keys():
            if col in take:
                res[row][col] = records[row][type][col]
    return res

def apply_ratings(records):
    res = records.copy()
    for i in res.keys():
        for rating in res[i]['Ratings']:
            res[i]['r ' + rating['Source']] = rating['Value']
        del res[i]['Ratings']
    return res
numo = apply_numerical(omdb, 'omdb', numerical['omdb'])
numt = apply_numerical(tmdb, 'tmdb', numerical['tmdb'])
num = dict([(i, {**numo[i], **numt[i]}) for i in numo.keys()])
num = apply_ratings(num)
numdf = pd.DataFrame.from_dict(num).T
for col in numdf.columns:
    numdf[col].loc[numdf[col] == 'N/A'] = np.nan
numdf['budget'] = numdf['budget'].replace(to_replace=0, value=np.nan)
numdf['r Internet Movie Database'].loc[numdf['r Internet Movie Database'].notnull()] = \
numdf['r Internet Movie Database'].loc[numdf['r Internet Movie Database'].notnull()].apply(lambda x: x.split('/')[0])
numdf['r Metacritic'].loc[numdf['r Metacritic'].notnull()] = \
numdf['r Metacritic'].loc[numdf['r Metacritic'].notnull()].apply(lambda x: int(x.split('/')[0]))
numdf['r Rotten Tomatoes'].loc[numdf['r Rotten Tomatoes'].notnull()] = \
numdf['r Rotten Tomatoes'].loc[numdf['r Rotten Tomatoes'].notnull()].apply(lambda x: float(x.replace('%', '')))
numdf['revenue'] = numdf['revenue'].replace(to_replace=0, value=np.nan)
numdf['Year'].loc[numdf['Year'].notnull()] = numdf['Year'].loc[numdf['Year'].notnull()].apply(lambda x: int(x.replace('–', '')[0]))
numdf['imdbVotes'].loc[numdf['imdbVotes'].notnull()] = numdf['imdbVotes'].loc[numdf['imdbVotes'].notnull()].apply(lambda x: int(x.replace(',', ''))) 
numdf.head()
from ppca import PPCA
ppca = PPCA()
ppca.fit(data=numdf.values.astype(float), d=16, verbose=True)
ppca.var_exp
transformed = ppca.transform()
transformed = pd.DataFrame(transformed)
transformed['idx'] = pd.Series(list(omdb.keys()))
transformed = transformed.set_index('idx')
transformed.head()
transformed.to_csv('../../../../data/engineering/pca.csv', index=True, index_label='idx')
# Bonus: Prince's PCA visualization
# Unfortunately, Prince does not work with missing data
# So you need to use PCA Magic instead<jupyter_output><empty_output><jupyter_text>UMAP: Uniform Manifold Approximation and ProjectionP.S. As for now, I am using this as a proof of concept for numerical feature visualizationIt does not support NaN values, hence my main focus for numerics is Probabilistic PCA<jupyter_code># Get Rid of NaNs
from sklearn.impute import SimpleImputer
from sklearn import preprocessing
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
numdf = numdf.astype(float)
imputer = imputer.fit(numdf)
numdf = imputer.transform(numdf)
# pretty colors
color = []
for row in tmdb.keys():
    genres = tmdb[row]['tmdb'].get('genres', False)
    if genres:
        color.append(genres[0]['id'])
    else:
        color.append('0')
le = preprocessing.LabelEncoder()
color = le.fit_transform(color)
import warnings
warnings.filterwarnings('ignore')


import umap
embedding = umap.UMAP(n_neighbors=100, min_dist=0.3).fit_transform(numdf)
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.cm as cm
%matplotlib inline
sns.set(context="paper", style="white")


# color = cm.rainbow(color)
fig, ax = plt.subplots(figsize=(12, 10))
plt.scatter(
    embedding[:, 0], embedding[:, 1], c=color, cmap="Spectral", s=0.1
)
plt.setp(ax, xticks=[], yticks=[])
plt.title("2d umap Embeddings", fontsize=18)

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17772410853814463, 'fraction_numerical': 0.0058246910072453474, 'mean_word_length': 4.686591276252019, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '433358', 'license': 'Apache-2.0', 'n_tokens_mistral': 2552, 'n_tokens_neox': 2479, 'n_words': 518, 'path': 'examples/0. Embeddings Generation/Pipelines/ML20M/3. Feature Engineering .ipynb', 'repo_name': 'XinruLiu/RecNN', 'size': '7039'}