<jupyter_start><jupyter_text>Dimensionality ReductionThis notebook focuses on applying dimensionality reduction using TruncatedSVD.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.preprocessing import StandardScaler
fn = ('/mnt/processed/private/msds2021/lt6/chicago-dataset/sample/'
      'taxi_tnp_weather.csv.gz')
df = pd.read_csv(fn, low_memory=False,
                 parse_dates=['trip_end_timestamp', 'trip_start_timestamp'])<jupyter_output><empty_output><jupyter_text>Clustering Features    dropoff_centroid_latitude    dropoff_centroid_longitude    pickup_centroid_latitude    pickup_centroid_longitude    fare    tip    trip_miles    trip_seconds    trip_total    additional_charges<jupyter_code>non_cluster_feature = ['trip_id', 'company',
                       'dropoff_community_area', 'pickup_community_area',
                       'trip_end_timestamp', 'trip_start_timestamp',
                       'Type', 'Severity', 'TransportType',
                       'payment_is_cashless']
dfc = pd.get_dummies(df.drop(columns=non_cluster_feature))
dfc.head()
stdscale = StandardScaler()
dfc = stdscale.fit_transform(dfc)
pca = PCA()
pca.fit(dfc)
exp_var = pca.explained_variance_ratio_
plt.plot(range(1, len(exp_var)+1), exp_var.cumsum(), 'o-')
plt.gca().axhline(0.9, c='r', ls='--')
plt.gca().axhline(0.95, c='g', ls='--')
plt.ylim([0, 1])
print('Number of PCs for above 90% Explained Variance:',
      np.size(exp_var) - np.sum(exp_var.cumsum() > 0.90) + 1)
print('Number of PCs for above 95% Explained Variance:',
      np.size(exp_var) - np.sum(exp_var.cumsum() > 0.95) + 1)
np.save('exp_var.npy', exp_var)<jupyter_output><empty_output><jupyter_text>Setting the threshold of explained variance to 95%, we'll select **11 PCs** to retain prior to clustering. We'll save it to an `.npy` file for the clustering stage.<jupyter_code>pca.set_params(n_components = 11)
dfc = pca.fit_transform(dfc)
np.save('pca.npy', dfc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12199807877041306, 'fraction_numerical': 0.01536983669548511, 'mean_word_length': 4.584450402144772, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '603090', 'license': 'MIT', 'n_tokens_mistral': 780, 'n_tokens_neox': 721, 'n_words': 145, 'path': 'dimensionality-reduction.ipynb', 'repo_name': 'nkespiritu/dmw-final-project', 'size': '2082'}