<jupyter_start><jupyter_text>Connect to Database<jupyter_code>import psycopg2
import pandas as pd
import pandas.io.sql as pd_sql
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import os
import yellowbrick as yb

from yellowbrick.features import RadViz
from sklearn.metrics import accuracy_score
from yellowbrick.features import Rank1D
from yellowbrick.features import Rank2D
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from yellowbrick.features import FeatureImportances
from sklearn.feature_selection import SelectFromModel
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet, RidgeCV, LogisticRegression

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
def connectDB(DB):
    # connect to the PostgreSQL server
    return psycopg2.connect(
        database=DB,
        user="postgres",
        password="<PASSWORD>",
        host="database-1.c5vispb5ezxg.us-east-1.rds.amazonaws.com",
        port='5432')

def disconnectDB():
    cur.close()
    conn.close()
conn = connectDB("Dataset")

# extract everything from 'table_name' into a dataframe
df = pd_sql.read_sql(f"select * from public.\"FinalData_ForFeatures\" ", con=conn)
#code to be able to view all the columns in pandas dataframe
pd.set_option('display.max_columns', 50)<jupyter_output><empty_output><jupyter_text>Feature Analysis for STEM 2013<jupyter_code>df = df[['code_module', 
         'module_domain', 
         'module_presentation_length',
         'year',
         'term', 
         'num_of_prev_attempts',
         'final_result', 
         'date_registration', 
         'disability', 
         'gender', 
         'age_band', 
         'region', 
         'highest_education', 
         'imd_band', 
         'studied_credits', 
         'b4_sum_clicks', 
         'half_sum_clicks', 
         'std_half_score'
        ]]
feature_list = df
#remove null values
#feature_list = feature_list.dropna()
feature_list.head()
feature_list = df.loc[(df.year == 0)&(df.module_domain==1)]
feature_list = feature_list[feature_list.columns.drop(['module_domain', 'year', 'term'])]
feature_list.head()<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code>#pick out the 5 features to examine relationship
X = feature_list[["half_sum_clicks","gender", "highest_education", "code_module", "studied_credits"]].values
        
#define target variable
y = feature_list["final_result"].values<jupyter_output><empty_output><jupyter_text>Radviz Visualization<jupyter_code>classes = ["fail", "pass"]
features = ["half_sum_clicks", "gender", "highest_education", "code_module", "studied_credits"]
        
sns.set_palette("colorblind")
# Instantiate the visualizer
visualizer = RadViz (
    classes=classes,
    features = features
)

visualizer.fit(X, y)           # Fit the data to the visualizer
visualizer.transform(X)        # Transform the data
visualizer.show()              # Draw the data<jupyter_output><empty_output><jupyter_text>Pairplots: note that the hue is final_result which is now mapped to a binary classification<jupyter_code>df_plot = df[["final_result","gender", "term", "highest_education", "half_sum_clicks","num_of_prev_attempts", "studied_credits"]]
g = sns.pairplot(df_plot, hue="final_result")
plt.show()
#printing out to check the y variable was filled
y<jupyter_output><empty_output><jupyter_text>Feature Analysis Ranking Shapiro Feature Ranking<jupyter_code>sns.set_palette("cubehelix")

X = feature_list[feature_list.columns.drop(['final_result'])]

y = feature_list['final_result']

# Instantiate the 1D visualizer with the Sharpiro ranking algorithm
visualizer = Rank1D(algorithm='shapiro',size=(1080, 720))

visualizer.fit(X, y)           # Fit the data to the visualizer
visualizer.transform(X)        # Transform the data
visualizer.show()              # Finalize and render the figure<jupyter_output>//anaconda3/lib/python3.7/site-packages/scipy/stats/morestats.py:1653: UserWarning: p-value may not be accurate for N > 5000.
  warnings.warn("p-value may not be accurate for N > 5000.")
//anaconda3/lib/python3.7/site-packages/scipy/stats/morestats.py:1653: UserWarning: p-value may not be accurate for N > 5000.
  warnings.warn("p-value may not be accurate for N > 5000.")
//anaconda3/lib/python3.7/site-packages/scipy/stats/morestats.py:1653: UserWarning: p-value may not be accurate for N > 5000.
  warnings.warn("p-value may not be accurate for N > 5000.")
//anaconda3/lib/python3.7/site-packages/scipy/stats/morestats.py:1653: UserWarning: p-value may not be accurate for N > 5000.
  warnings.warn("p-value may not be accurate for N > 5000.")
//anaconda3/lib/python3.7/site-packages/scipy/stats/morestats.py:1653: UserWarning: p-value may not be accurate for N > 5000.
  warnings.warn("p-value may not be accurate for N > 5000.")
//anaconda3/lib/python3.7/site-packages/scipy/stats/morestats.py:[...]<jupyter_text>RandomForest Classifier Feature Ranking<jupyter_code>model = RandomForestClassifier(n_estimators=10)
feature_list = FeatureImportances(model, size=(1080, 720))
feature_list.fit(X, y)

feature_list.show()<jupyter_output><empty_output><jupyter_text>LogisticRegression Feature Ranking<jupyter_code>model = LogisticRegression()
viz = FeatureImportances(model, size=(1080, 720))

viz.fit(X, y)

viz.show()<jupyter_output>//anaconda3/lib/python3.7/site-packages/yellowbrick/model_selection/importances.py:195: YellowbrickWarning: detected multi-dimensional feature importances but stack=False, using mean to aggregate them.
  YellowbrickWarning,<jupyter_text>DecisionTreeClassifier Feature Ranking<jupyter_code>model = DecisionTreeClassifier()
viz = FeatureImportances(model, size=(1080, 720))

viz.fit(X, y)
viz.show()
feature_list = RandomForestClassifier()
feature_list.fit(X, y)
predictions = feature_list.predict(X)
    
accuracy_score(y, predictions)<jupyter_output><empty_output><jupyter_text>GradientBoostingClassifier Feature Ranking<jupyter_code>from sklearn.ensemble import GradientBoostingClassifier

model = GradientBoostingClassifier()
viz = FeatureImportances(model, size=(1080, 720))
viz.fit(X, y)
viz.show()<jupyter_output><empty_output><jupyter_text>SVM Feature Ranking<jupyter_code>from sklearn import linear_model
model = linear_model.SGDClassifier()
viz = FeatureImportances(model, size=(1080, 720))
viz.fit(X, y)
viz.show()<jupyter_output>//anaconda3/lib/python3.7/site-packages/yellowbrick/model_selection/importances.py:195: YellowbrickWarning: detected multi-dimensional feature importances but stack=False, using mean to aggregate them.
  YellowbrickWarning,<jupyter_text>Lasso Feature Ranking<jupyter_code>model = Lasso()
viz = FeatureImportances(model, size=(1080, 720))

viz.fit(X, y)
viz.show()<jupyter_output><empty_output><jupyter_text>Additional Feature Analysis Pearson Squares Feature Ranking (Excludes final_result)<jupyter_code># Instantiate the visualizer with the Pearson ranking algorithm
visualizer = Rank2D(algorithm='pearson', size=(1080, 720))

visualizer.fit(X, y)
visualizer.transform(X)
visualizer.show()
#features = feature_list[feature_list.columns.drop(['final_result'])]
#label = feature_list['final_result']<jupyter_output><empty_output><jupyter_text>PCA Decomposition Analysis<jupyter_code>from yellowbrick.features.pca import PCADecomposition

# Create a list of colors to assign to points in the plot
colors = np.array(['r' if yi else 'b' for yi in y])

visualizer = PCADecomposition(
    scale=True, color=colors, size=(1080, 720)
)
visualizer.fit(X, y)
visualizer.transform(X, y)
visualizer.show()<jupyter_output>//anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/data.py:645: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.
  return self.partial_fit(X, y)
//anaconda3/lib/python3.7/site-packages/sklearn/base.py:464: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.
  return self.fit(X, **fit_params).transform(X)
//anaconda3/lib/python3.7/site-packages/sklearn/pipeline.py:451: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.
  Xt = transform.transform(Xt)<jupyter_text>Manifold from Yellowbrick<jupyter_code>from yellowbrick.features.manifold import Manifold

visualizer = Manifold(
    manifold='isomap', n_neighbors=5, target='continuous', size=(1080, 720)
)
visualizer.fit(X, y)
visualizer.transform(X, y)
visualizer.show()<jupyter_output><empty_output><jupyter_text>Regularization Techniques<jupyter_code>features = X
labels   = y
list(features)
model = LinearRegression()
model.fit(features, labels)
print(list(zip(features, model.coef_.tolist())))
model = Lasso()
model.fit(features, labels)
print(list(zip(features, model.coef_.tolist())))
model = Ridge()
model.fit(features, labels)
print(list(zip(features, model.coef_.tolist())))
model = ElasticNet()
model.fit(features, labels)
print(list(zip(features, model.coef_.tolist())))
model = LogisticRegression()
model.fit(features, labels)
print(list(zip(features, model.coef_.tolist())))
model = linear_model.SGDClassifier()
model.fit(features, labels)
print(list(zip(features, model.coef_.tolist())))<jupyter_output>[('code_module', [-224.89919607303278, -2026.0963209697218, 5.0088907811366035, -404.9688196548927, -15.778005960580135, -34.310901850785385, -39.0693480928651, -265.4712114002368, 78.6395852638449, 392.9474817801629, -160.28450499636392, -190.83873876130505, 161.7871722307124, 9205.92403123733])]<jupyter_text>Transformer Techniques<jupyter_code>model = Lasso()
sfm = SelectFromModel(model)
sfm.fit(features, labels)
print(list(features.iloc[:, sfm.get_support(indices=True)]))
model = Ridge()
sfm = SelectFromModel(model)
sfm.fit(features, labels)
print(list(features.iloc[:, sfm.get_support(indices=True)]))
model = ElasticNet()
sfm = SelectFromModel(model)
sfm.fit(features, labels)
print(list(features.iloc[:, sfm.get_support(indices=True)]))
model = RandomForestClassifier()
sfm = SelectFromModel(model)
sfm.fit(features, labels)
print(list(features.iloc[:, sfm.get_support(indices=True)]))
model = DecisionTreeClassifier()
sfm = SelectFromModel(model)
sfm.fit(features, labels)
print(list(features.iloc[:, sfm.get_support(indices=True)]))
model = linear_model.SGDClassifier()
sfm = SelectFromModel(model)
sfm.fit(features, labels)
print(list(features.iloc[:, sfm.get_support(indices=True)]))
model = LogisticRegression()
sfm = SelectFromModel(model)
sfm.fit(features, labels)
print(list(features.iloc[:, sfm.get_support(indices=True)]))
model = GradientBoostingClassifier()
sfm = SelectFromModel(model)
sfm.fit(features, labels)
print(list(features.iloc[:, sfm.get_support(indices=True)]))<jupyter_output>['std_half_score']
{'fraction_non_alphanumeric': 0.12122052704576976, 'fraction_numerical': 0.041793804900601014, 'mean_word_length': 6.201065246338215, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '176983', 'license': 'MIT', 'n_tokens_mistral': 3997, 'n_tokens_neox': 3643, 'n_words': 730, 'path': 'code/feature analysis/Feature Analysis STEM 2013.ipynb', 'repo_name': 'yifeitung/University-Learning-Analytics', 'size': '10817'}