<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Description2&nbsp;&nbsp;Setup3&nbsp;&nbsp;Other functions3.1&nbsp;&nbsp;Filter4&nbsp;&nbsp;TSNE  Clustering  Description  - Cluster funds using Spektral Clustering  Setup<jupyter_code>import sys
sys.executable
import feather
from itertools import product
from multiprocessing import Pool

import numpy as np
import pandas as pd
import seaborn as sns
from scipy import sparse
from scipy.spatial.distance import euclidean

from minisom import MiniSom

import matplotlib.pyplot as plt
import matplotlib.cm as cm
from mpl_toolkits.mplot3d import Axes3D

from sklearn.preprocessing import normalize, LabelEncoder, minmax_scale
from sklearn.decomposition import TruncatedSVD
from sklearn.manifold import TSNE

from graph_tool.all import *
import cairo
### Returns
path = '../data/processed/returns.feather'
returns = feather.read_dataframe(path)
returns = returns.rename(columns = {'caldt' : 'report_dt'})
returns = returns.assign(year = returns['report_dt'].dt.year)

### row_info
path = '../data/processed/row_info_f.feather'
row_info = feather.read_dataframe(path)
le = LabelEncoder()
row_info['lipper_class_num'] = le.fit_transform(row_info['lipper_class'])

### col_info
path = '../data/processed/col_info_f.feather'
col_info = feather.read_dataframe(path)

### Holdings
path = '../data/processed/holdings_f.npz'
holdings = sparse.load_npz(path)<jupyter_output><empty_output><jupyter_text>Other functions  Filter<jupyter_code>def filter_data(year, verbose = True):
        
    row_info_f = row_info.copy()
    if (year != 'full'):    # If year = full take whole sample
        row_info_f = row_info_f.query('year == @year')

    begin_date = row_info_f.iloc[0,:]['report_dt']
    end_date = begin_date + pd.DateOffset(years=1) # 1 year offset
    row_info_f.reset_index(drop = True, inplace=True)

    # Filter returns
    crsp_fundno_unique = row_info_f['crsp_fundno'].unique()
    returns_f = returns.copy()
    query = '''report_dt >= @begin_date and report_dt <= @end_date and crsp_fundno in @crsp_fundno_unique'''
    returns_f = returns_f.query(query)

    # Change return of month for which holdings apply to 0
    returns_f = returns_f.copy()
    mask = returns_f['report_dt'] == begin_date
    returns_f.loc[mask,'mret'] = 0
    
    # Drop all funds with first return observation after starting date
    drop_fundnos = returns_f.drop_duplicates('crsp_fundno').query('mret != 0')['crsp_fundno']
    returns_f.query('crsp_fundno not in @drop_fundnos', inplace=True)
    row_info_f.query('crsp_fundno not in @drop_fundnos', inplace=True)
    
    # Filter holdings accordingly and delet all securities with less than two observations
    holdings_f = holdings.copy()
    holdings_f = holdings[row_info_f['row']]
    
    holdings_b = sparse.csr_matrix(holdings_f, copy=True)
    holdings_b.data = np.ones(len(holdings_f.data))

    sum_sec_boolean = holdings_b.toarray().sum(0)
    col_mask = (sum_sec_boolean >= 2).flatten()

    holdings_f = holdings_f.tocsc()
    holdings_f = holdings_f[:,col_mask]
    holdings_f = holdings_f.tocsr()
    
    ## Preprocessing

    holdings_ft = normalize(holdings_f, norm = 'l2')

    
    if (verbose):
        print('Numer of unique funds:           {:10,d}'.format(row_info_f.shape[0]))
        print('Numer of unique securities:      {:10,d}'.format(holdings_ft.shape[1]))
        print('Begin date:                      {}'.format(begin_date.date()))
        print('End date:                        {}'.format(end_date.date()))
    
    return(row_info_f, returns_f, holdings_ft, begin_date, end_date)<jupyter_output><empty_output><jupyter_text>TSNE<jupyter_code>row_info_f, returns_f, holdings_ft, begin_date, end_date = filter_data(2017)

SVD = TruncatedSVD(n_components=100, n_iter = 100).fit(holdings_ft)
SVD.explained_variance_

holdings_trans = SVD.transform(holdings_ft)

holdings_embed = TSNE(n_components=3, perplexity=50).fit_transform(holdings_trans)
x = holdings_embed[:,0]
y = holdings_embed[:,1]
z = holdings_embed[:,2]

c = row_info_f.lipper_class_num
fig = plt.figure(figsize=(20,20))
ax = fig.add_subplot(111, projection='3d')

ax.scatter(y, x,  z, c=c, zdir = (y))

ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11995359628770301, 'fraction_numerical': 0.010904872389791183, 'mean_word_length': 4.679841897233201, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '534203', 'license': 'FTL', 'n_tokens_mistral': 1574, 'n_tokens_neox': 1502, 'n_words': 299, 'path': 'notebooks/062 - tSNE.ipynb', 'repo_name': 'Amglex/Mutual-Funds', 'size': '4310'}