<jupyter_start><jupyter_text>Preprocessing<jupyter_code>X_array = data.loc[:, data.columns].values
X_array.shape
# Feature seletion:
sel = VarianceThreshold(threshold=0)
X_array_sel = sel.fit_transform(X_array)
X_array_sel.shape
# Normalizing data:
x_array_norm = MinMaxScaler().fit_transform(X_array_sel)
pd.DataFrame(x_array_norm)
# Standardizing data:
x_array_std = StandardScaler().fit_transform(X_array_sel)
pd.DataFrame(x_array_std)<jupyter_output><empty_output><jupyter_text>Transformations:<jupyter_code>n_components = 48

# PCA:
pca_obj1 = PCA(n_components=n_components)
x_pca = pca_obj1.fit_transform(X_array_sel)

pca_obj2 = PCA(n_components=n_components)
x_pca_norm = pca_obj2.fit_transform(x_array_norm)

pca_obj3 = PCA(n_components=n_components)
x_pca_strd = pca_obj3.fit_transform(x_array_std)
plt.figure(figsize=(12, 6))
plt.title('PCA: Normalize/Standardize or not?')
plt.bar(range(1, pca_obj1.n_components_+1, 1), pca_obj1.explained_variance_ratio_, alpha=0.5, label='un-normalized data')
plt.bar(range(1, pca_obj2.n_components_+1, 1), pca_obj2.explained_variance_ratio_, alpha=0.5, label='normalized data')
plt.bar(range(1, pca_obj3.n_components_+1, 1), pca_obj3.explained_variance_ratio_, alpha=0.5, label='standardized data')
plt.xlabel('nº pca components')
plt.ylabel('explained variance ratio')
plt.legend()
plt.show()
cum_evr1 = [0]
cum_evr2 = [0]
cum_evr3 = [0]
for i, (val1, val2, val3) in enumerate(zip(pca_obj1.explained_variance_ratio_,
                                           pca_obj2.explained_variance_ratio_,
                                           pca_obj3.explained_variance_ratio_,)):
    cum_evr1.append(val1 + cum_evr1[-1])
    cum_evr2.append(val2 + cum_evr2[-1])
    cum_evr3.append(val3 + cum_evr3[-1])
    print(f'{i+1} component(s) {round(100*cum_evr1[-1],2)}% (not norm.) {round(100*cum_evr2[-1],2)}% (norm.) {round(100*cum_evr3[-1],2)}% (strd.)' )
plt.figure(figsize=(12, 6))
plt.title('PCA: Normalize/Standardize or not?')
plt.bar(range(0, pca_obj1.n_components_+1, 1), cum_evr1, alpha=0.5, label='un-normalized data')
plt.bar(range(0, pca_obj2.n_components_+1, 1), cum_evr2, alpha=0.5, label='normalized data')
plt.bar(range(0, pca_obj3.n_components_+1, 1), cum_evr3, alpha=0.5, label='standardized data')
plt.xlabel('nº pca components')
plt.ylabel('explained variance ratio')
plt.legend()
plt.show()
def choose_n_components(pca_obj, n_components = None, min_evr = 0.95):
    cum_evr = [0]
    for val in pca_obj.explained_variance_ratio_:
        cum_evr.append(val + cum_evr[-1])
    if n_components is None:
        n_components = pca_obj.n_components_
        for j, val in enumerate(cum_evr):
            if val > min_evr:
                n_components = j
                break
    plt.figure(figsize=(10, 6))
    plt.title(f'{n_components} explain at least {min_evr} of variance ratio')
    plt.plot(range(0, pca_obj.n_components_+1, 1), cum_evr, alpha=0.5, marker='s')    
    plt.scatter([n_components], [cum_evr[n_components]], marker='o', s=150, facecolors="None", edgecolors='r')
    plt.plot([0, pca_obj.n_components_+1], [1,1], linestyle=':', linewidth=1, color='k')
    plt.xlabel('nº pca components')
    plt.ylabel('explained variance ratio')
    plt.xlim([0, pca_obj.n_components_+1])
    plt.show()
choose_n_components(pca_obj1)
choose_n_components(pca_obj2)
choose_n_components(pca_obj3)
# DBScan test:
# eps_values = np.arange(0.1, 100, 0.1)
# n_pca_components = np.arange(1, x_pca_norm.shape[1]+1, 1)

n_pca_components = np.arange(1, 48+1, 1)
eps_values = np.arange(0.1, 80, 0.1)

opt_silhoucall = -np.inf
opt_config = {}
opt_cluster_labels = []

plt.figure(figsize=(18, 8))
for n in n_pca_components:
    X = x_pca_norm[:, 0:n]
    sil_array = []
    eps_array = []
    for eps in eps_values:
        dbscan_model = DBSCAN(eps=eps, min_samples=2).fit(X)
        cluster_labels = dbscan_model.labels_

        if len(np.unique(cluster_labels)) > 1 and -1 in cluster_labels:
            sample_silhouette_values = silhouette_samples(X, cluster_labels)
            silhouette_avg = sample_silhouette_values[np.where(cluster_labels >= 0)[0]].mean()
            cluster_recall = len(np.where(cluster_labels >= 0)[0])/len(cluster_labels)
            silhoucall = 2 * (silhouette_avg * cluster_recall) / (silhouette_avg + cluster_recall)
            sil_array.append(silhoucall)
            eps_array.append(eps)
            
            if silhoucall > opt_silhoucall:
                opt_silhoucall = silhoucall
                opt_config = {'n_pca_components': n, 'eps': eps, 'n_clusters': len(np.unique(cluster_labels))-1,
                              'silhouette_avg': silhouette_avg, 'cluster_recall': cluster_recall, 'silhoucall': silhoucall}
                opt_cluster_labels = cluster_labels
                print('opt_config:', opt_config)

    if n > 10 and n <= 20:
        plt.plot(eps_array, sil_array, label=f'{n} component(s)', linestyle=':', linewidth=2)
    elif n > 20 and n <= 30:
        plt.plot(eps_array, sil_array, label=f'{n} component(s)', linestyle='--', linewidth=2)
    elif n > 30:
        plt.plot(eps_array, sil_array, label=f'{n} component(s)', linestyle='-.', linewidth=2)
    else:
        plt.plot(eps_array, sil_array, label=f'{n} component(s)', linestyle='-', linewidth=2)

plt.xlabel('eps')
plt.ylabel('silhouette_avg')
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), ncol=3)
plt.tight_layout()
plt.show()
opt_cluster_labels
fig = plt.figure(figsize=(20, 8))
ax = fig.add_subplot(121, projection='3d')
plt.title('DBSCAN/PCA clustering results')
for k in np.unique(opt_cluster_labels):
    indexes = np.where(opt_cluster_labels == k)[0]
    if k == -1:
        ax.scatter(x_pca_norm[indexes, 0], x_pca_norm[indexes, 1], x_pca_norm[indexes, 2], alpha=0.5, s=80, label=k, c='k')
    else:
        ax.scatter(x_pca_norm[indexes, 0], x_pca_norm[indexes, 1], x_pca_norm[indexes, 2], alpha=0.5, s=80, label=k)
ax.set_xlabel('Component 1')
ax.set_ylabel('Component 2')
ax.set_zlabel('Component 3')
plt.legend()
plt.tight_layout()

ax = fig.add_subplot(122)
plt.title('DBSCAN/PCA clustering results')
for k in np.unique(opt_cluster_labels):
    indexes = np.where(opt_cluster_labels == k)[0]
    if k == -1:
        plt.scatter(x_pca_norm[indexes, 0], x_pca_norm[indexes, 1], alpha=0.5, s=80, label=k, c='k')
    else:
        plt.scatter(x_pca_norm[indexes, 0], x_pca_norm[indexes, 1], alpha=0.5, s=80, label=k)
ax.set_xlabel('Component 1')
ax.set_ylabel('Component 2')
plt.legend()
plt.tight_layout()

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14804639804639805, 'fraction_numerical': 0.03418803418803419, 'mean_word_length': 4.331977217249797, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '264443', 'license': 'MIT', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2560, 'n_words': 429, 'path': 'notebooks/nbk_02_pca.ipynb', 'repo_name': 'dwelcaslu/i2a2-genomic-analysis-challenge', 'size': '6552'}