<jupyter_start><jupyter_text>Step 2: Dimensionality Reduction<jupyter_code>fig = plt.figure(figsize=(15,12))

data = []
cluster_algo = 'KMeans'
for i, (d, x) in enumerate({'Wine': x1, 'Pima': x2}.items()):
    ax = plt.subplot(421+i)
    pca = PCA(whiten=True, random_state=42)
    pca.fit(x)
    ax.plot(np.arange(1, len(pca.explained_variance_ratio_)+1), np.cumsum(pca.explained_variance_ratio_), 'bs--', label='PCA-'+d+' Expl.Var')
    ax_rev = ax.twinx()
    ax_rev.plot(np.arange(1, len(pca.singular_values_)+1), pca.singular_values_, 'ro-', label='PCA-'+d+' Singl. Vals')
    #print('PCA-'+d+'Avg Log Likelihood=', pca.score(x))
    ax.legend()
    ax_rev.legend(loc='center right')
    ax.grid()
    ax_rev.grid()
    ax = plt.subplot(422+i)
    
    kurt_values = []
    
    #print(x.shape)
    
    for n in range(1, x.shape[1]+1):
        ica = FastICA(n_components=n, whiten=True, random_state=42)
        ica.fit(x)
        x_ica = ica.transform(x)
        kurt_mean = np.mean(np.abs((kurtosis(x_ica, axis=0, fisher=True))))
        kurt_std = np.std(np.abs((kurtosis(x_ica, axis=0, fisher=True))))
    
        #print('n=', n, 'kurt=', np.mean(kurtosis(x_ica, axis=0, fisher=True),axis=0))
        
        kurt_values.append([kurt_mean, kurt_std])
    
    kurt_values = np.array(kurt_values)
    #print(kurt_values)
    
    kurt_before_transform = kurtosis(x, axis=0, fisher=True)
    kurt_values = np.absolute(np.array(kurt_values))
    kurt_values_mean = np.mean(kurt_values, axis=0)
    kurt_values_std = np.std(kurt_values, axis=0)
    ax = plt.subplot(423+i)
    ax.plot(range(1,x_ica.shape[1]+1), kurt_values[:,0], 'bs--', label='ICA-'+d+' Kurt-Transformed')
    ax.fill_between(range(1,x_ica.shape[1]+1), kurt_values[:,0]+kurt_values[:,1], kurt_values[:,0], alpha=0.3)
    #ax.plot(range(1,x.shape[1]+1), kurt_before_transform, 'ro--', label='ICA-'+d+' Kurt-Original')
    #print('ICA-'+d+'=', pca.score(x))
    ax.legend()
    
    ax = plt.subplot(425+i)
    ax_rev = ax.twinx()
    
    for i in range(x_ica.shape[1]):
        sns.kdeplot(x_ica[:, i], color='b',  linestyle='dashed', 
                    ax=ax, 
                   alpha=0.2)
        xi = x[:,i] if type(x)==np.ndarray else x[x.columns[i]]
        sns.kdeplot(xi, color='r', linestyle='dashed', 
                    ax=ax_rev, 
                    alpha=0.2)
    
    ax.ticklabel_format(useOffset=False, style='plain')
    ax.set_ylabel('Density of recovered signals', color='b')  # we already handled the x-label with ax1
    ax.tick_params(axis='y', labelcolor='b')
    ax.grid()
    ax.set_xlabel('Value')
    #ax.legend()

    ax_rev.ticklabel_format(useOffset=False, style='plain')
    ax_rev.set_ylabel('Density for raw data', color='r')  # we already handled the x-label with ax1
    ax_rev.tick_params(axis='y', labelcolor='r')
    ax_rev.grid()
    ax_rev.set_xlabel('Value')
    #ax_rev.legend()

    
    
    #print(kurt_values)
    #data.append([pca.score(x)])


x1, y1 = get_data.get_dataset1(False)
x2, y2 = get_data.get_dataset2(False)

data = []
for i, (d, x) in enumerate({'Wine': x1, 'Pima': x2}.items()):   
    
    for s in [42, 1, 360, 2020, 1000]:
        for n in range(1, x1.shape[1]+10):
            random_projection = SparseRandomProjection(random_state=s, n_components=n)
            # data has this shape:  row, col = 4898, 11 

            random_projection.fit(x)
            components =  random_projection.components_.toarray() # shape=(5, 11) 
            p_inverse = np.linalg.pinv(components.T) # shape=(5, 11) 

            #now get the transformed data using the projection components
            reduced_data = random_projection.transform(x)
            reconstructed= reduced_data.dot(p_inverse) 

            #print(reduced_data.shape) #(4898, 5)
            #print(reconstructed.shape) #(4898, 11), back in original shape

            assert  x.shape ==  reconstructed.shape
            error = mean_squared_error(x, reconstructed)
            
            data.append([d, s, n, error])
            


data = pd.DataFrame(data, columns=['data', 'seed', '#Projections', 'Reconstruction Error'])
data
recons_error_mean = data.groupby(by=['data', '#Projections'])['Reconstruction Error'].mean().reset_index()
print(recons_error_mean)
recons_error_std = data.groupby(by=['data', '#Projections']).std()['Reconstruction Error'].reset_index()
print(recons_error_std)
figure = fig = plt.figure(figsize=(6,4))
ax1 = plt.subplot(111)
d = recons_error_mean[recons_error_mean['data']=='Wine']
ax1.plot(d["#Projections"], d['Reconstruction Error'], 'bo--', label='Wine')
ax1.fill_between(d["#Projections"], d['Reconstruction Error']-recons_error_std[recons_error_std['data']=='Wine']['Reconstruction Error'],
                d['Reconstruction Error']+recons_error_std[recons_error_std['data']=='Wine']['Reconstruction Error'], alpha=0.1)


d = recons_error_mean[recons_error_mean['data']=='Pima']
ax1.plot(d["#Projections"], d['Reconstruction Error'], 'rs-', label='Pima')
ax1.fill_between(d["#Projections"], d['Reconstruction Error']-recons_error_std[recons_error_std['data']=='Pima']['Reconstruction Error'],
                d['Reconstruction Error']+recons_error_std[recons_error_std['data']=='Pima']['Reconstruction Error'], alpha=0.1)
ax1.grid()
ax1.set_ylabel('Reconstruction Error')
ax1.set_xlabel('#Projections')
ax1.set_title('Reconstruction Error for Random Projection')
ax1.legend()
plot_dir = os.path.join(os.pardir, r'plot\step2\RP')
os.makedirs(plot_dir, exist_ok=True)
plt.savefig(plot_dir+'/step2_random_proj_recons_error.png')
os.getcwd()<jupyter_output><empty_output><jupyter_text>RFC<jupyter_code>x1, y1 = get_data.get_dataset1(False)
x2, y2 = get_data.get_dataset2(False)

figure = fig = plt.figure(figsize=(8,5))

for i, (d, (x, y)) in enumerate({'Wine': (x1,y1), 'Pima': (x2,y2)}.items()):  
    rfc = RFC(n_estimators=500, min_samples_leaf=round(0.1*x.shape[0]), n_jobs=-1, random_state=5)
    feat_imp = rfc.fit(x,y).feature_importances_ 
    feat_imp = pd.DataFrame(feat_imp,columns=['Feature Importance'], index=x.columns)
    
    feat_imp = feat_imp.sort_values(by=['Feature Importance'], ascending=False)
    feat_imp['Cum Sum Importance'] = feat_imp['Feature Importance'].cumsum()
    feat_imp = feat_imp[feat_imp['Cum Sum Importance'] <= 0.95]
    top_cols = feat_imp.index.tolist()
    print(top_cols)
    ax = plt.subplot(121+i)
    ax.plot(feat_imp['Cum Sum Importance'], label=d)
    plt.xticks(rotation=90)
    ax.set_ylabel('Cum. Importance')
    ax.legend()
    ax.grid()

plt.suptitle('Random Forest Classifier Feature Importance')
plt.tight_layout(pad=1.0)
plot_dir = os.path.join(os.pardir, 'plot', 'step2', 'RFC')
os.makedirs(plot_dir, exist_ok=True)
plt.savefig(os.path.join(plot_dir, 'step2_rfc_feature_importance.png'))<jupyter_output>count     4898
unique       2
top       True
freq      3258
Name: quality, dtype: object
['alcohol', 'density', 'volatile acidity', 'chlorides', 'total sulfur dioxide', 'citric acid']
['Glucose', 'BMI', 'Age', 'Pregnancies', 'Insulin']
{'fraction_non_alphanumeric': 0.1714245135634143, 'fraction_numerical': 0.024286322965487855, 'mean_word_length': 3.983722576079264, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '214414', 'license': 'Apache-2.0', 'n_tokens_mistral': 2778, 'n_tokens_neox': 2659, 'n_words': 428, 'path': 'unsupervised_learning/src/Unsupervised learning dimensionality reduction.ipynb', 'repo_name': 'asif-rehan/CS7641-Machine-Learning', 'size': '7041'}