<jupyter_start><jupyter_text>PCA<jupyter_code># PCA
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

pca = PCA()
print(pca)

print(df.shape)
    
dummies_cols = ['Category', 'Descript', 'DayOfWeek', 'PdDistrict', 'Resolution']
drop_cols = ['Date', 'Time', 'Address', 'Location'] #Date and Tme can be handled better
df1 = pd.get_dummies(df, columns=dummies_cols)
df1 = df1.drop(drop_cols, axis=1)
df1 = df1.dropna(axis=0)
X = df1.loc[:,:].values
X_cols = df1.loc[:,:].columns

scaler=StandardScaler()
X_transformed = scaler.fit_transform(X)

pca.fit(X_transformed)
X_pca = pca.fit(X_transformed).transform(X_transformed)
# Explained Variance 
plt.figure(figsize=(15,8))
ax = sns.barplot(x=np.arange(100), y=pca.explained_variance_[0:100]/sum(pca.explained_variance_))
ax.set_xlabel('Principal Component')
ax.set_ylabel('Explained Variance')
# Histogram of explained variance
vals, bins, patches = plt.hist(pca.explained_variance_/sum(pca.explained_variance_), log=True, bins=50)
plt.title('Histogram of explained variance')
plt.xlabel('Explained Variance')
plt.ylabel('Number of components')

# Observations
num_exp_var = np.sum(vals)-vals[0] #
print(num_exp_var, 'PCA components explain all variance.')
print(np.sum(vals), 'total features')
print(num_exp_var/np.sum(vals), '% of all features')<jupyter_output>519.0 PCA components explain all variance.
1012.0 total features
0.5128458498023716 % of all features
{'fraction_non_alphanumeric': 0.14216701173222912, 'fraction_numerical': 0.033126293995859216, 'mean_word_length': 7.238636363636363, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '545341', 'license': 'MIT', 'n_tokens_mistral': 575, 'n_tokens_neox': 539, 'n_words': 107, 'path': 'notebooks/police_dept/.ipynb_checkpoints/initial_exploration-PCA-checkpoint.ipynb', 'repo_name': 'daericks/datasf', 'size': '1449'}