<jupyter_start><jupyter_text>Ethnicity Data Hierarchical Clustering<jupyter_code>data = pd.read_csv('Present Bias Conditions DF.csv')
data.columns

data_eth = data.iloc[:,14:21]
data = data_eth
from sklearn.preprocessing import normalize
data_scaled = normalize(data)
data_scaled = pd.DataFrame(data_scaled, columns=data.columns)
data_scaled.head()
import scipy.cluster.hierarchy as shc
plt.figure(figsize=(10, 7))  
plt.title("Dendrograms")  
dend = shc.dendrogram(shc.linkage(data_scaled, method='ward'))
plt.figure(figsize=(10, 7))  
plt.title("Dendrograms")  
dend = shc.dendrogram(shc.linkage(data_scaled, method='ward'))
plt.axhline(y=2, color='r', linestyle='--')
from sklearn.cluster import AgglomerativeClustering
cluster = AgglomerativeClustering(n_clusters=3, affinity='euclidean', linkage='ward')  
cluster.fit_predict(data_scaled)
plt.figure(figsize=(10, 7))  
plt.scatter(data_scaled['White or Caucasian'], data_scaled['MFA Familiarity Count'], c=cluster.labels_)
plt.figure(figsize=(10, 7))  
plt.scatter(data_scaled['Asian or Pacific Islander'], data_scaled['MFA Familiarity Count'], c=cluster.labels_)
data_ = data.iloc[:,14:21]
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
# education: col 9-14
"""
# Using DataFrame.drop
df.drop(df.columns[[1, 2]], axis=1, inplace=True)

# drop by Name
df1 = df1.drop(['B', 'C'], axis=1)

# Select the ones you want
df1 = df[['a','d']]
"""
data_ed = pd.read_csv('demo_factors_ohe.csv')<jupyter_output><empty_output><jupyter_text>Running Education against Opt In Frequency<jupyter_code>data_ed_col = data_ed[['Prefer Not to Answer','MFA Familiarity Count', 'Security Beliefs', 'Security Importance', 'Quick Log In Importance','Log In Usability Importance', 'Log In Convenience Importance', 'Log In Satisfaction','MFA Opt In Frequency']]
data_ed_col
from sklearn.preprocessing import normalize
data_scaled = normalize(data_ed_col)
data_scaled = pd.DataFrame(data_scaled, columns=data_ed_col.columns)
data_scaled.head()
import scipy.cluster.hierarchy as shc
plt.figure(figsize=(10, 7))  
plt.title("Prefer Not To Answer Dendrograms")  
dend = shc.dendrogram(shc.linkage(data_scaled, method='ward'))
# 1. Select the data
data_ed_col = data_ed[['College', 'Elementary School', 'Graduate School', 'High School', 'Prefer Not to Answer','Log In Usability Importance']]
data_ed_col
# 2. Normalize the data
from sklearn.preprocessing import normalize
data_scaled = normalize(data_ed_col)
data_scaled = pd.DataFrame(data_scaled, columns=data_ed_col.columns)
data_scaled.head()
import scipy.cluster.hierarchy as shc
plt.figure(figsize=(10, 7))  
plt.title("Dendrograms")  
dend = shc.dendrogram(shc.linkage(data_scaled, method='ward'))
# reset the threshold
plt.figure(figsize=(10, 7))  
plt.title("Dendrograms")  
dend = shc.dendrogram(shc.linkage(data_scaled, method='ward'))
plt.axhline(y=1.0, color='r', linestyle='--')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1391629806056482, 'fraction_numerical': 0.015311330384484519, 'mean_word_length': 7.3760683760683765, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '184042', 'license': 'MIT', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1063, 'n_words': 224, 'path': "Master'sThesisModels+PreparedData/Collab- Hierarchical clustering-using PB DF.ipynb", 'repo_name': 'domschrein/NEU-MSCS-Thesis', 'size': '2939'}