<jupyter_start><jupyter_text>Correlated Data with PCADemonstrate how PCA works with dataset containing related featuresInput Features: 10 columns are related to each other. x,x^2,x^3,...Target Feature: y is sum of all input features for that sampleObjective: 1. Demonstrate how PCA is able to reduce the input dimensions2. Train two Linear Regression models one with original features and another with PCA components.Compare the outcome<jupyter_code># 1000 rows x 10 columns
np.random.seed(5)
# Generate 1000 random values between 0 and 1
random_data = np.random.rand(1000)
random_data.shape
random_data[:5]
# Add 10 columns. 
# x = random_data
# Columns: x,x^2,x^3,....
col_data = {}
for i in range(10):
    col_data[i] = random_data ** (i+1)
df = pd.DataFrame(col_data)
df.shape
df.head()
df.corr()
# Normalize the dataset before running PCA
# x = (x-mean)/stddev
scaler = StandardScaler()
def transform_data(scaler, df, columns):
    transformed_data = scaler.transform(df[columns])
    df_transformed = pd.DataFrame(transformed_data, columns=columns)
    
    for col in df_transformed.columns:
        df[col] = df_transformed[col]
scaler.fit(df)
transform_data(scaler, df, df.columns)
df.head()
df.describe()
y = df.sum(axis=1)
y.head()
df_pca = df.copy()
# Test PCA
# Two modes to test with PCA
# How many components we need in final output or how much variance do we need to capture as a percentage

pca = PCA(n_components=0.9) # percentage of variance to capture
#pca = PCA(n_components=2) # number of components
pca.fit(df_pca)

# number of components PCA came up with
pca.n_components_
def transform_with_pca(pca, df, columns):
    transformed_data = pca.transform(df[columns])
    
    tcols = []
    for i in range(pca.n_components_):       
        tcols.append('component_' + str(i))
    
    print ('components:',tcols)
    df_transformed = pd.DataFrame(transformed_data, columns=tcols)
    
    for col in df_transformed.columns:
        df[col] = df_transformed[col]
    
    df.drop(columns, inplace=True, axis=1)
    
    return tcols
transform_with_pca(pca,df_pca,df_pca.columns)
df_pca.head()
# Compare X and y
# Compare First Component and y
plt.scatter(x=df[0],y=y,label='ideal fit')
plt.grid(True)
plt.xlabel('X')
plt.ylabel('Target')
plt.title('X vs y')
plt.legend()
plt.scatter(x=df_pca['component_0'],y=y,label='component 0')
plt.grid(True)
plt.xlabel('First Component')
plt.ylabel('Target')
plt.title('First Component vs y')
plt.legend()
# Train with actual data using Linear Regression Model
import sklearn.linear_model as lm

regressor = lm.LinearRegression()
regressor.fit(df,y)
y_predicted = regressor.predict(df)
plt.scatter(x=df[0],y=y,label='Actual')
plt.scatter(x=df[0],y=y_predicted,label='Predicted')
plt.grid(True)
plt.xlabel('Input Feature')
plt.ylabel('Target')
plt.legend()
import sklearn.metrics as metrics
print("RMSE: {0}".format(metrics.mean_squared_error(y,y_predicted)**.5))
plt.boxplot([y,y_predicted], labels=['actual','predicted'])
plt.title('Box Plot - Actual, Predicted')
plt.ylabel('Target')
plt.grid(True)
# Train with PCA Components using Linear Regression Model
plt.scatter(x=df_pca['component_0'],y=y,label='component 0')
plt.grid(True)
plt.xlabel('Input Feature')
plt.ylabel('Target')
plt.legend()
regressor.fit(df_pca,y)
y_predicted_pca = regressor.predict(df_pca)
plt.scatter(x=df_pca['component_0'],y=y,label='Actual')
plt.scatter(x=df_pca['component_0'],y=y_predicted_pca,label='Predicted')
plt.grid(True)
plt.xlabel('Component')
plt.ylabel('Target')
plt.legend()
plt.scatter(x=df[0],y=y_predicted,label='X vs y')
plt.scatter(x=df[0],y=y_predicted_pca,label='X vs PCA Predicted')
plt.grid(True)
plt.xlabel('X')
plt.ylabel('Target')
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13777421080791868, 'fraction_numerical': 0.013108614232209739, 'mean_word_length': 5.629432624113475, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '274149', 'license': 'Apache-2.0', 'n_tokens_mistral': 1436, 'n_tokens_neox': 1331, 'n_words': 315, 'path': 'fm-pca-xgboost/pca/ExplorePCA/correlated_data_pca_exploration.ipynb', 'repo_name': 'prasanjit-/ml_notebooks', 'size': '3738'}