<jupyter_start><jupyter_text>Data Cleaning<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import pickle
import requests
import json
from sklearn.datasets import make_regression
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
df = pd.read_sas('epcg19.sas7bdat')
df.head()
df.shape
df.isnull().sum()
df1 = df[["AGE", "AGEGR", "ASDGRI", "CLIC", "CTZN", "GENDER", "MRDG","NBAMEMG","NDGMEMG", "N2OCPRMG", "RACETHM", "SALARY"]].copy()
df1.head()
df1[["AGE", "AGEGR", "ASDGRI", "CLIC", "CTZN", "GENDER", "MRDG","NBAMEMG", "NDGMEMG", "N2OCPRMG", "RACETHM", "SALARY"]].shape<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code>df1["ASDGRI"].value_counts()
df1['ASDGRI'].replace({bytes('N', 'ascii'):0, bytes('Y', 'ascii'):1}, inplace= True)
df1['ASDGRI'].value_counts()
df1["CLIC"].replace({bytes('N', 'ascii'):0, bytes('Y', 'ascii'):1}, inplace= True)
df1["CLIC"].value_counts()
skip_vals = df1[(df1['CLIC'] == bytes('L', 'ascii'))].index
df1.drop(skip_vals, inplace = True)
df1['CLIC'].value_counts()
df1["CLIC"] = df1["CLIC"].astype(int)
df1["CTZN"].replace({bytes('1', 'ascii'):1, bytes('2', 'ascii'):2, bytes('3', 'ascii'):3,
                    bytes('4', 'ascii'):4}, inplace= True)
df1["CTZN"].value_counts()
df1["GENDER"].replace({bytes('M', 'ascii'):0, bytes('F', 'ascii'):1}, inplace = True)
df1["GENDER"].value_counts()
df1["NBAMEMG"].value_counts()
skip_vals2 = df1[(df1["NBAMEMG"] == bytes('8', 'ascii'))].index
df1.drop(skip_vals2, inplace = True)
df1["NBAMEMG"].value_counts()
df1["NBAMEMG"].replace({bytes('1', 'ascii'):1, bytes('2', 'ascii'):2, bytes('3','ascii'):3,
                        bytes('4', 'ascii'):4, bytes('5', 'ascii'):5, bytes('6','ascii'):6,
                        bytes('7', 'ascii'):7}, inplace = True)
df1["NDGMEMG"].value_counts()
df1["NDGMEMG"].replace({bytes('1', 'ascii'):1, bytes('2', 'ascii'):2, bytes('3','ascii'):3,
                        bytes('4', 'ascii'):4, bytes('5', 'ascii'):5, bytes('6','ascii'):6,
                        bytes('7', 'ascii'):7}, inplace = True)
df1["NDGMEMG"].value_counts()
df1["N2OCPRMG"].head()
skip_vals3 = df1[(df1['N2OCPRMG'] == bytes('8', 'ascii'))].index
df1.drop(skip_vals3, inplace = True)
df1["N2OCPRMG"].value_counts()
df1["N2OCPRMG"].replace({bytes('1', 'ascii'):1, bytes('2', 'ascii'):2, bytes('3','ascii'):3,
                        bytes('4', 'ascii'):4, bytes('5', 'ascii'):5, bytes('6','ascii'):6,
                        bytes('7', 'ascii'):7}, inplace = True)
df1["N2OCPRMG"].value_counts()

df1["MRDG"].replace({bytes('1', 'ascii'):1, bytes('2', 'ascii'):2, bytes('3', 'ascii'):3,
                    bytes('4', 'ascii'):4}, inplace = True)
df1["MRDG"].value_counts()
df1["RACETHM"].replace({bytes('1', 'ascii'):1, bytes('2', 'ascii'):2, bytes('3','ascii'):3,
                        bytes('4', 'ascii'):4, bytes('5', 'ascii'):5, bytes('6','ascii'):6,
                        bytes('7', 'ascii'):7}, inplace = True)

df1["RACETHM"].value_counts()
cut_salary_labels_8 = ['1', '2', '3', '4', '5', '6', '7', '8']
cut_bins = [-1, 19999, 34999, 49999, 64999, 79999, 99999, 119999, 689578]
df1['SALARYG'] = pd.cut(df1['SALARY'], bins=cut_bins, labels=cut_salary_labels_8)
df1['SALARYG'].value_counts()
cut_age_labels_7 = ['1', '2', '3', '4', '5', '6', '7']
cut_bins = [0, 25, 29, 39, 49, 59, 69, 75]
df1['AGEG'] = pd.cut(df1['AGE'], bins=cut_bins, labels=cut_age_labels_7)
df1['AGEG'].value_counts()
#df1 = df1.drop(columns=['AGEGR'])
df1_first20=df1.head(20)
print(df1_first20)<jupyter_output>AGE  ASDGRI  CLIC  CTZN  GENDER  MRDG  NBAMEMG  NDGMEMG  N2OCPRMG  \
0   36.0       0     0     1       0     2        1        1         1   
1   38.0       1     1     1       1     2        4        7         7   
2   42.0       0     0     1       0     2        4        7         7   
3   48.0       0     0     1       1     1        2        2         2   
4   37.0       0     1     1       1     4        4        7         7   
5   52.0       0     1     1       1     1        4        4         7   
6   34.0       0     1     1       0     3        4        4         7   
7   35.0       0     1     1       0     3        4        4         4   
8   61.0       0     1     1       0     4        2        6         6   
9   47.0       1     1     1       1     1        7        7         7   
10  43.0       0     1     1       0     1        7        7         7   
11  46.0       1     0     2       0     2        1        1         1   
12  37.0       1     0     1       0  [...]<jupyter_text>Exploratory Data Analysis<jupyter_code>df1.shape
df1.dtypes
df1.head()
df1.describe()
# no missing values
df1.isna().mean()*100
df1['SALARYG'] = df1['SALARYG'].astype(int)
df1['AGEG'] = df1['AGEG'].astype(int)
df1.to_csv('NSCG_clean_encoded.csv', index=False)
df1.dtypes
need_labels = df1.copy(deep=True)
print(need_labels)
need_labels['GENDER'] = need_labels['GENDER'].replace([0,1], ['Male', 'Female'])
need_labels['GENDER'].head()
need_labels['ASDGRI'] = need_labels['ASDGRI'].replace([0,1], ['No Assoc Deg', 'Assoc Deg'])
need_labels['ASDGRI'].head()
need_labels['CLIC'] = need_labels['CLIC'].replace([0,1], ['No Certn_Lic', 'Certn_Lic'])
need_labels['CLIC'].head()
need_labels['CTZN'] = need_labels['CTZN'].replace([1,2,3,4], ['US Citizen', 'US Citizen-Nat', 'Non-US Perm', 'Non-US Temp'])
need_labels['CTZN'].head()
need_labels['MRDG'] = need_labels['MRDG'].replace([1,2,3,4], ['Bachelors', 'Masters', 'Doctorate', 'Professional'])
need_labels['MRDG'].head()
need_labels['NBAMEMG'] = need_labels['NBAMEMG'].replace([1,2,3,4,5,6,7], ['Comp & Math', 'Bio, Agr & Envrn', 'Phys Sci', 'Soc Sci', 'Engr', 'S&E-related', 'Non S&E'])
need_labels['NBAMEMG'].head()
need_labels['NDGMEMG'] = need_labels['NDGMEMG'].replace([1,2,3,4,5,6,7], ['Comp & Math', 'Bio, Agr & Envrn', 'Phys Sci', 'Soc Sci', 'Engr', 'S&E-related', 'Non S&E'])
need_labels['NDGMEMG'].head()
need_labels['N2OCPRMG'] = need_labels['N2OCPRMG'].replace([1,2,3,4,5,6,7], ['Comp & Math', 'Bio, Agr & Envrn', 'Phys Sci', 'Soc Sci', 'Engr', 'S&E-related', 'Non S&E'])
need_labels['N2OCPRMG'].head()
need_labels['RACETHM'] = need_labels['RACETHM'].replace([1,2,3,4,5,6,7], ['Asian', 'AI/AN', 'Black', 'Hispanic', 'White', 'NH/PI', 'Multi-Race'])
need_labels['RACETHM'].head()
need_labels['SALARYG'] = need_labels['SALARYG'].replace([1,2,3,4,5,6,7,8], ['Less than $20,000', '$20,000-34,999', '$35,000-49,999', '$50,000-64,999', '$65,000-79,999', '$80,000-99,999', '$100,000-119,999', '$120,000 or more'])
need_labels['SALARYG'].head()
need_labels['AGEG'] = need_labels['AGEG'].replace([1,2,3,4,5,6,7], ['Ages 24 or younger', '25-29', '30-39', '40-49', '50-59', '60-69', '70-75'])
need_labels['AGEG'].value_counts()
need_labels.head()
need_labels.to_csv('NSCG_clean_labeled.csv', index=False)
need_labels.groupby('GENDER').SALARY.median().plot(kind = 'bar')
# 0 = male 1 = female
sns.relplot(x='AGE', y='SALARY', data=need_labels)<jupyter_output><empty_output><jupyter_text>Multiple Linear Regression<jupyter_code>df1.corr().style.background_gradient(cmap='RdBu', vmin=-1, vmax=1)
X = df1[["AGE", "ASDGRI", "CLIC", "CTZN", "GENDER", "MRDG", "NBAMEMG", "NDGMEMG", "N2OCPRMG", "RACETHM"]]
y = df1["SALARY"].values.reshape(-1, 1)
print(X.shape, y.shape)
# Use train_test_split to create training and testing data

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, random_state=0)
# Create a StandardScater model and fit it to the training data

X_scaler = StandardScaler().fit(X_train)
y_scaler = StandardScaler().fit(y_train)
# Transform the training and testing data using the X_scaler and y_scaler models

X_train_scaled = X_scaler.transform(X_train)
X_test_scaled = X_scaler.transform(X_test)
y_train_scaled = y_scaler.transform(y_train)
y_test_scaled = y_scaler.transform(y_test)
# Create the model using LinearRegression

model = LinearRegression()
model.fit(X_train_scaled, y_train_scaled)
# Make predictions using a fitted model
# Plot the difference between the model predicted values and actual y values, versus the model predicted values
# Hint: You can predict values of X training and testing data using the model.predict() method on a fitted model

predictions = model.predict(X_test_scaled)
model.fit(X_train_scaled, y_train_scaled)
plt.scatter(model.predict(X_train_scaled), model.predict(X_train_scaled) - y_train_scaled, c="blue", label="Training Data")
plt.scatter(model.predict(X_test_scaled), model.predict(X_test_scaled) - y_test_scaled, c="orange", label="Testing Data")
plt.legend()
plt.hlines(y=0, xmin=y_test_scaled.min(), xmax=y_test_scaled.max())
plt.title("Residual Plot")
plt.show()
# Used X_test_scaled, y_test_scaled, and model.predict(X_test_scaled) to calculate MSE and R2

from sklearn.metrics import mean_squared_error

MSE = mean_squared_error(y_test_scaled, predictions)
r2 = model.score(X_test_scaled, y_test_scaled)


print(f"MSE: {MSE}, R2: {r2}")
# LASSO model
# Note: Use an alpha of .01 when creating the model for this activity
from sklearn.linear_model import Lasso

### BEGIN SOLUTION
lasso = Lasso(alpha=.01).fit(X_train_scaled, y_train_scaled)

predictions = lasso.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, predictions)
r2 = lasso.score(X_test_scaled, y_test_scaled)
### END SOLUTION

print(f"MSE: {MSE}, R2: {r2}")
# Ridge model
# a method of estimating the coefficients of multiple-regression models 
# in scenarios where independent variables are highly correlated.
# Note: Use an alpha of .01 when creating the model for this activity
from sklearn.linear_model import Ridge

### BEGIN SOLUTION
ridge = Ridge(alpha=.01).fit(X_train_scaled, y_train_scaled)

predictions = ridge.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, predictions)
r2 = ridge.score(X_test_scaled, y_test_scaled)
### END SOLUTION

print(f"MSE: {MSE}, R2: {r2}")
# ElasticNet model
# Note: Use an alpha of .01 when creating the model for this activity
from sklearn.linear_model import ElasticNet

### BEGIN SOLUTION
elasticnet = ElasticNet(alpha=.01).fit(X_train_scaled, y_train_scaled)

predictions = elasticnet.predict(X_test_scaled)

MSE = mean_squared_error(y_test_scaled, predictions)
r2 = elasticnet.score(X_test_scaled, y_test_scaled)
### END SOLUTION

print(f"MSE: {MSE}, R2: {r2}")<jupyter_output>MSE: 0.8453287733792106, R2: 0.10009191476078627<jupyter_text>Machine Learning Code From Stanford Class PCA Clustering<jupyter_code>import pydotplus
from scipy.cluster import hierarchy
from sklearn import decomposition, preprocessing, cluster, tree
from yellowbrick.cluster.silhouette import SilhouetteVisualizer
#PCA
X = df1
std = preprocessing.StandardScaler()
X_std = pd.DataFrame(std.fit_transform(X), columns=X.columns)
X_std
pca = decomposition.PCA()
pca_X = pd.DataFrame(pca.fit_transform(X_std), columns=[f'PC{i+1}' for i in range(len(X.columns))])
pca_X
pca.explained_variance_ratio_
(pd.DataFrame(pca.components_, columns=X.columns)
 .iloc[:4]
 .plot.bar()
 .legend(bbox_to_anchor=(1,1)))
sns.scatterplot(x='PC1', y='PC2', 
                data=pca_X.assign(SALARY=X.SALARY), 
                hue='SALARY')
sns.scatterplot(x='PC2', y='PC3', 
                data=pca_X.assign(SALARY=X.SALARY), 
                hue='SALARY')
inerts = []
for i in range(2, 20):
    k = cluster.KMeans(n_clusters=i, random_state=0)
    k.fit(X_std)
    inerts.append(k.inertia_)
    
pd.Series(inerts).plot()
# start, end = 4, 8
# cols = 2
# rows = ((end - start) // cols)
# fix, axes = plt.subplots(rows, cols, figsize=(12,8))
# axes = axes.reshape(cols * rows)
# for i, k in enumerate(range(start, end), 0):
#     ax = axes[i]
#     sil = SilhouetteVisualizer(cluster.KMeans(n_clusters=k, random_state=0), ax=ax)
#     sil.fit(X_std)
#     sil.finalize()
# plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18082605775688382, 'fraction_numerical': 0.05473472128945601, 'mean_word_length': 4.170572916666667, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '564942', 'license': 'MIT', 'n_tokens_mistral': 4961, 'n_tokens_neox': 4623, 'n_words': 943, 'path': 'Final_Project_Gigi_Jones_Cleaning_EDA_SomeML.ipynb', 'repo_name': 'GGGOJO/US-Scientists-PartII', 'size': '11912'}