<jupyter_start><jupyter_text>https://www.kdnuggets.com/2019/07/data-pre-processing-optimizing-regression-model-performance.html<jupyter_code>!pip install --user scikit-learn
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OrdinalEncoder
import sys
np.set_printoptions(threshold=sys.maxsize)
from sklearn import svm
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Import the data and show the top 5 rows<jupyter_code>data = pd.read_csv('data_darrel.csv')
data.head()
# check shape
data.shape
# check for nan values
data[data.isna().any(axis=1)]
# set independent variables (the first 12 columns)
X = data.iloc[:,:-2]
X
# set dependent variable
y = data.iloc[:, 12:13]
y
# columns that contain yes or no and numerical inputs
data_numeric = data[['Do you believe multi-factor authentication improves security?',
                 'How important is online security to you?', 
                 'How important is quick log in authentication to you?',
                 'How important is usability during log in authentication to you?',
                 'How important is convenience during log in authentication to you?',
                 'How satisfied are you with current log in systems?',
                 'How often do you opt to use multi-factor authentication when given the choice?']]


# columns that contain categories 
data_categorical = data[['What is your age?', 
                     'Which gender do you identify most with?', 
                     'What is the highest grade (or year) of regular school you have completed?', 
                     'Which of the following best describes you?', 
                     'Which of the following, if any, images are you familiar with?',
                     'How many of the following, if any, images are you familiar with?']]

#data_output = data[['Which of the following examples, if any, of multi-factor authentication systems have you used']]
data_categorical.head()
data_numeric.head()<jupyter_output><empty_output><jupyter_text>Print the category options<jupyter_code>#unique values for categorical headers 
#will category encode these 
print(data['What is your age?'].unique())
print(data['Which gender do you identify most with?'].unique())
print(data['What is the highest grade (or year) of regular school you have completed?'].unique())
print(data['Which of the following best describes you?'].unique())
print(data['Which of the following, if any, images are you familiar with?'].unique())
print(data['How many of the following, if any, images are you familiar with?'].unique())<jupyter_output>['35-44' '25-34' '18-24' '45-54' '55-64' '65']
['Female' 'Gender Variant/Non-Conforming' 'Male']
['Graduate School' 'High School' 'Prefer Not to Answer' 'College'
 'Elementary School']
['Two or more races' 'White or Caucasian' 'Black or African American'
 'Hispanic or Latino' 'Asian or Pacific Islander' 'Indian']
['SMS text-message, Software Token or Pin Passcode, Biometric (or Fingerprint) 2FA verification, Push Notification'
 'SMS text-message, Software Token or Pin Passcode, Biometric (or Fingerprint) 2FA verification, Push Notification, Hardware Token, Facial Recognition'
 'SMS text-message, Software Token or Pin Passcode, Biometric (or Fingerprint) 2FA verification, Facial Recognition'
 'SMS text-message, Software Token or Pin Passcode, Biometric (or Fingerprint) 2FA verification, Push Notification, Facial Recognition'
 'SMS text-message, Software Token or Pin Passcode, Push Notification, Hardware Token'
 'SMS text-message, Biometric (or Fingerprint) 2FA verification, Push Notific[...]<jupyter_text>Look at the final column, dependent variable<jupyter_code># Dependent Variable 
y = data.iloc[:,12].astype(str)
print(y)
# ordinal encode target variable
label_encoder = LabelEncoder()
y = label_encoder.fit_transform(y)<jupyter_output>0          Often
1      Sometimes
2      Sometimes
3          Often
4          Often
         ...    
181       Always
182        Often
183        Often
184       Always
185    Sometimes
Name: How often do you opt to use multi-factor authentication when given the choice?, Length: 186, dtype: object<jupyter_text>Encode the categorical columns<jupyter_code>#Age Encoder

from sklearn.preprocessing import LabelEncoder

age_encoder = LabelEncoder()
age_encoder.fit(data_categorical['What is your age?'])
age_values = age_encoder.transform(data_categorical['What is your age?'])
age_values
#Gender Encoder

from sklearn.preprocessing import LabelEncoder

gender_encoder = LabelEncoder()
gender_encoder.fit(data_categorical['Which gender do you identify most with?'])
gender_values = gender_encoder.transform(data_categorical['Which gender do you identify most with?'])
gender_values
#Education Level Encoder

#does education categories really matter? If ranked? I read it is categorical data.
#But I can assign values later.

from sklearn.preprocessing import LabelEncoder

education_encoder = LabelEncoder()
education_encoder.fit(data_categorical['What is the highest grade (or year) of regular school you have completed?'])
education_values = education_encoder.transform(data_categorical['What is the highest grade (or year) of regular school you have completed?'])
education_values
#Ethnicity Encoder

from sklearn.preprocessing import LabelEncoder

ethnicity_encoder = LabelEncoder()
ethnicity_encoder.fit(data_categorical['Which of the following best describes you?'])
ethnicity_values = ethnicity_encoder.transform(data_categorical['Which of the following best describes you?'])
ethnicity_values
#Image Familiarity Encoder

from sklearn.preprocessing import LabelEncoder

image_encoder = LabelEncoder()
image_encoder.fit(data_categorical['Which of the following, if any, images are you familiar with?'])
image_values = image_encoder.transform(data_categorical['Which of the following, if any, images are you familiar with?'])
image_values
# Number of Images Familiarity Encoder

num_image_encoder = LabelEncoder()
num_image_encoder.fit(data_categorical['How many of the following, if any, images are you familiar with?'])
num_image_values = num_image_encoder.transform(data_categorical['How many of the following, if any, images are you familiar with?'])
num_image_values
#unique values for numeric headers 
# will ordinal encode these 

print(data['Do you believe multi-factor authentication improves security?'].unique())
print(data['How important is online security to you?'].unique())
print(data['How important is quick log in authentication to you?'].unique())
print(data['How important is usability during log in authentication to you?'].unique())
print(data['How important is convenience during log in authentication to you?'].unique())
print(data['How satisfied are you with current log in systems?'].unique())
print(data['How often do you opt to use multi-factor authentication when given the choice?'].unique())
#Improves Security Encoder

from sklearn.preprocessing import OrdinalEncoder

security_encoder = OrdinalEncoder()
security_encoder.fit(data_numeric[['Do you believe multi-factor authentication improves security?']])
security_values = security_encoder.transform(data_numeric[['Do you believe multi-factor authentication improves security?']])
security_values
#Online Security Encoder 

from sklearn.preprocessing import OrdinalEncoder

online_encoder = OrdinalEncoder()
online_encoder.fit(data_numeric[['How important is online security to you?']])
online_values = online_encoder.transform(data_numeric[['How important is online security to you?']])
online_values
#Quick Log In Encoder

from sklearn.preprocessing import OrdinalEncoder

quick_encoder = OrdinalEncoder()
quick_encoder.fit(data_numeric[['How important is quick log in authentication to you?']])
quick_values = quick_encoder.transform(data_numeric[['How important is quick log in authentication to you?']])
quick_values                          
#Usability Encoder

from sklearn.preprocessing import OrdinalEncoder

usability_encoder = OrdinalEncoder()
usability_encoder.fit(data_numeric[['How important is usability during log in authentication to you?']])
usability_values = usability_encoder.transform(data_numeric[['How important is usability during log in authentication to you?']])
usability_values               
#Convenience Encoder

from sklearn.preprocessing import OrdinalEncoder

convenience_encoder = OrdinalEncoder()
convenience_encoder.fit(data_numeric[['How important is convenience during log in authentication to you?']])
convenience_values = convenience_encoder.transform(data_numeric[['How important is convenience during log in authentication to you?']])
convenience_values               
#Current Systems Encoder

from sklearn.preprocessing import OrdinalEncoder

current_encoder = OrdinalEncoder()
current_encoder.fit(data_numeric[['How satisfied are you with current log in systems?']])
current_values = usability_encoder.transform(data_numeric[['How satisfied are you with current log in systems?']])
current_values              
#Often Encoder

from sklearn.preprocessing import OrdinalEncoder

often_encoder = OrdinalEncoder()
often_encoder.fit(data_numeric[['How often do you opt to use multi-factor authentication when given the choice?']])
often_values = often_encoder.transform(data_numeric[['How often do you opt to use multi-factor authentication when given the choice?']])
often_values  
# columns to combine: age_values, current_values, convenience_values, usability_values,
# quick_values, online_values, often_values  

# for 'Age Group' column
age = pd.DataFrame(age_values, columns=['Age Group'])

# for 'Gender' column
gender = pd.DataFrame(gender_values, columns=['Gender'])

# for 'Education' column
education = pd.DataFrame(education_values, columns=['Education'])

# for 'Ethnicity' column
ethnicity = pd.DataFrame(ethnicity_values, columns=['Ethnicity'])

# for 'Image Familiarity' column
image_familiarity = pd.DataFrame(image_values, columns=['Image Familiarity'])

# for 'Number of Image Familiarity' column
num_image_familiarity = pd.DataFrame(num_image_values, columns=['Number of Image Familiarity'])

# for 'Security Values' column
security_values = pd.DataFrame(security_values, columns=['Security Values'])

# for 'Online Security Importance' column
online_security_importance = pd.DataFrame(online_values, columns=['Online Security Importance'])

# for 'Quick Log In Importance' column
quick_login_importance = pd.DataFrame(quick_values, columns=['Quick Log In Importance'])

# for 'Usability Importance' column
usability_importance = pd.DataFrame(usability_values, columns=['Usability Importance'])

# for 'Convenience Importance' column
convenience_importance = pd.DataFrame(convenience_values, columns=['Convenience Importance'])

# for 'Log In Satisfaction' column
login_satisfaction = pd.DataFrame(current_values, columns=['Log In Satisfaction'])

# for 'MFA Use' column
# mfa_use = pd.DataFrame(often_values, columns=['MFA Use'])

# FOR DARREL: COMBINE COLUMNS THAT CONTAIN PSYCHOLOGICAL FACTORS
df_psych_factors = pd.concat((num_image_familiarity, security_values, online_security_importance, quick_login_importance, usability_importance, convenience_importance, login_satisfaction, mfa_use), axis = 1)

# combine all categorical columns as one dataframe
# df_encoded = pd.concat((age, gender, education, ethnicity, image_familiarity, security_values, online_security_importance, quick_login_importance, usability_importance, convenience_importance, login_satisfaction, mfa_use), axis=1)

# the preview
print(df_psych_factors.shape)
df_psych_factors
print(df_psych_factors)
# Export encoded data as csv file
df_psych_factors.to_csv('df_psych_factors_data.csv')
#independent variables

X_input_var = df_psych_factors.iloc[:,:7].astype(str)
                                                         
# target variable
y_target = df_psych_factors.iloc[:,7].astype(str)

# summarize 
print('Input', X_input_var.shape)
print('Output', y_target.shape)<jupyter_output>Input (186, 7)
Output (186,)<jupyter_text>Use the LabelEncoder & OneHotEncoder to work on categorical featureshttps://towardsdatascience.com/encoding-categorical-features-21a2651a065c<jupyter_code># Categorical boolean mask
categorical_feature_mask = X_input_var.dtypes == object
categorical_feature_mask

# filter categorical columns using mask and turn it into a list
categorical_cols = X_input_var.columns[categorical_feature_mask].tolist()
categorical_cols
# One Hot Encoder
# from sklearn.preprocessing import OneHotEncoder
# ohe = OneHotEncoder(categorical_features = )
# LabelEncoder converts each class under specificed feature to a numerical value.

# Instantiate a LabelEncoder object:

# Import labelencoder
from sklearn.preprocessing import LabelEncoder

# instantiate labelencoder object
le = LabelEncoder()

# Apply the LabelEncoder on each of the categorical columns:
X_input_var[categorical_cols] = X_input_var[categorical_cols].apply(lambda col: le.fit_transform(col))

X_input_var[categorical_cols].head(10)<jupyter_output><empty_output><jupyter_text>Our categorical features are multi class, which makes it not a good choice for LabelEncoder since it bring a natural ordering for different classes. This can lead to certain inputs appearing to be 'greater' than others and poor performance.Therefore, for our dataframe with multi class features, we will use OneHotEncoder as an additional step.<jupyter_code># import OneHotEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.compose import ColumnTransformer
# prepare the categorical columns for OneHotEncoder
categorical_ix = X_input_var.select_dtypes(include=['object','bool']).columns
categorical_ix
# define the data preparation for the columns
t = [('cat', OneHotEncoder(), categorical_ix)]
col_transform = ColumnTransformer(transformers=t)
from numpy import mean
from numpy import std
from numpy import absolute

from sklearn.svm import SVR
from sklearn.pipeline import Pipeline
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score

# define the SVR model
model = SVR(kernel='rbf', gamma = 'scale', C=100)

# define the data preparation and modeling pipeline
pipeline = Pipeline(steps=[('prep', col_transform), ('m',model)])

# define the model cross-validation configuration
cv = KFold(n_splits=10, shuffle=True, random_state=1)
# evaluate the pipeline using cross validation and calculate MAE
scores= cross_val_score(pipeline, X, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1)

# summarize the model performance
print('MAE: %.3f (%.3f)' % (mean(scores),std(scores)))<jupyter_output>MAE: nan (nan)<jupyter_text>Split the dataset<jupyter_code># split the dataset into train and test sets
x, X_test, y, y_test = train_test_split(X_input_var, y_target, test_size=0.1, random_state=1)
X_test.shape

#X_train, X_test, y_train, y_test = train_test_split(df_encoded.drop(['Image Familiarity'], axis=1), df_encoded['Image Familiarity'], test_size =0.33, random_state=1)
X_input_var, X_cv, y_train, y_cv = train_test_split(x,y,test_size = 0.2, train_size=0.8)
# Ordinal encode input variables
"""
ordinal_encoder = OrdinalEncoder()
# X_test.transpose(.reshape((X_test))
print(x)
ordinal_encoder._fit(x)
print('########################')
X_train = ordinal_encoder.transform(x)
print(X_train)
X_test = ordinal_encoder._transform(X_test)
"""
#Feature Scaling
from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)
print(X_test)
#Demonstrating feature scaling on a graph

import pandas as pd
import numpy as np
from sklearn import preprocessing
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
matplotlib.style.use('ggplot')
np.random.seed(1)
df = pd.DataFrame({
    'Age Group': np.random.normal(0, 1, 186),
    'Gender': np.random.normal(4, 2, 186),
    'Education': np.random.normal(-8, 4, 186),
    'Ethnicity': np.random.normal(0, 1, 186),
    'Image Familiarity': np.random.normal(4, 2, 186),
    'Security Values': np.random.normal(-8, 4, 186),
    'Online Security Importance': np.random.normal(0, 1, 186),
    'Quick Log In Importance': np.random.normal(4, 2, 186),
    'Usability Importance': np.random.normal(-8, 4, 186),
    'Convenience Importance': np.random.normal(-8, 4, 186),
    'Log In Satisfaction': np.random.normal(-8, 4, 186)
    
})
    
scaler = preprocessing.StandardScaler()
scaled_df = scaler.fit_transform(df)
scaled_df = pd.DataFrame(scaled_df, columns=['Age Group', 'Gender', 'Education', 
                                             'Ethnicity', 'Image Familiarity', 'Security Values',
                                             'Online Security Importance', 'Quick Log In Importance', 
                                             'Usability Importance', 'Convenience Importance', 
                                             'Log In Satisfaction'
])
fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(6, 5))
ax1.set_title('Before feature Scaling')
sns.kdeplot(df['Age Group'], ax=ax1)
sns.kdeplot(df['Gender'], ax=ax1)
sns.kdeplot(df['Education'], ax=ax1)
sns.kdeplot(df['Ethnicity'], ax=ax1)
sns.kdeplot(df['Image Familiarity'], ax=ax1)
sns.kdeplot(df['Security Values'], ax=ax1)
sns.kdeplot(df['Online Security Importance'], ax=ax1)
sns.kdeplot(df['Quick Log In Importance'], ax=ax1)
sns.kdeplot(df['Usability Importance'], ax=ax1)
sns.kdeplot(df['Convenience Importance'], ax=ax1)
sns.kdeplot(df['Log In Satisfaction'], ax=ax1)
ax2.set_title('After feature Scaling')
sns.kdeplot(scaled_df['Age Group'], ax=ax2)
sns.kdeplot(scaled_df['Gender'], ax=ax2)
sns.kdeplot(scaled_df['Education'], ax=ax2)
sns.kdeplot(scaled_df['Ethnicity'], ax=ax2)
sns.kdeplot(scaled_df['Image Familiarity'], ax=ax2)
sns.kdeplot(scaled_df['Security Values'], ax=ax2)
sns.kdeplot(scaled_df['Online Security Importance'], ax=ax2)
sns.kdeplot(scaled_df['Usability Importance'], ax=ax2)
sns.kdeplot(scaled_df['Convenience Importance'], ax=ax2)
sns.kdeplot(scaled_df['Log In Satisfaction'], ax=ax2)
plt.show()
scaled_df
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(X_train, y_train)#predicting the test set results
y_pred = regressor.predict(X_test)
y_pred

# df1 = pd.DataFrame({'Actual': y.values.flatten(), 'Predicted': y_pred.flatten()})
# df1 = pd.DataFrame({'Actual': y.flatten(), 'Predicted': y_pred.flatten()})
# df1.plot(kind='bar')
# plt.grid(which='major', linestyle='-', linewidth='0.5', color='green')
# plt.grid(which='minor', linestyle=':', linewidth='0.5', color='black')
# plt.show()
df = ({'Actual': y.flatten(), 'Predicted': y_pred.flatten()})
df1 = pd.DataFrame.from_dict(df, orient ='index')
df1 = df1.transpose()
df1.plot(kind='bar')
plt.grid(which='major', linestyle='-', linewidth='0.5', color='green')
plt.grid(which='minor', linestyle=':', linewidth='0.5', color='black')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1047439126784215, 'fraction_numerical': 0.010967674223341729, 'mean_word_length': 5.6726190476190474, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '453314', 'license': 'MIT', 'n_tokens_mistral': 5901, 'n_tokens_neox': 5621, 'n_words': 1687, 'path': 'NEU-Masters-Darrel.ipynb', 'repo_name': 'domschrein/NEU-MSCS-Thesis', 'size': '19056'}