<jupyter_start><jupyter_text>--- Dimensionality Reduction<jupyter_code>from sklearn.decomposition import PCA, TruncatedSVD


svd = TruncatedSVD(n_components=100)
svdmat = svd.fit_transform(mat.T)

4598195/(871445*94464), 871445*94464
svdmat.shape, mat.shape, mat.size, (svdmat > 0.00001).sum()/svdmat.size, mat.size/(mat.shape[0]*mat.shape[1])
len(tfidf.get_feature_names())
np.savetxt("svd_mat_0.15.tsv", svdmat, delimiter="\t")
from umap import UMAP


umap = UMAP(
        n_neighbors=15, min_dist=0.1, n_components=2, metric='euclidean',
        verbose=True
    )

orig_n = mat.T.shape[0]
rand_inds = np.random.randint(orig_n, size=orig_n//10)
umat = umap.fit_transform(mat.T)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15352112676056337, 'fraction_numerical': 0.07183098591549296, 'mean_word_length': 5.644859813084112, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '590720', 'license': 'CC-BY-2.0', 'n_tokens_mistral': 341, 'n_tokens_neox': 302, 'n_words': 41, 'path': 'K-Cap_2021/regressions/data.ipynb', 'repo_name': 'cultural-ai/ConConCor', 'size': '710'}