<jupyter_start><jupyter_text>Replace Undergrad major values with Stem/non-stem. Delete 'i never Declared a major'<jupyter_code>check_values()
a = df.query('JobSatisfaction=="Neither satisfied nor dissatisfied"')
# a = a.groupby('Country')['Respondent'].count()
# b = df.groupby('Country')['Respondent'].count()
# a/b
# df.query('Country=="Italy"').shape
# df.shape
df.query('Employment =="Employed full-time"').shape[0]/df.shape[0]
# df = df[df['JobSatisfaction']!="Neither satisfied nor dissatisfied"]

contigency_pct = pd.crosstab(df['Country'], df['FormalEducation'], normalize='all')
contigency_pct
plt.figure(figsize=(12,8))
sns.heatmap(contigency_pct, annot=True, cmap="YlGnBu")
c, p, dof, expected = chi2_contingency(contigency_pct)
p
a = 0
b = 0
mat=np.zeros([7,7])
for i in df.columns:
    for j in df.columns:
        contigency_pct = pd.crosstab(df[i], df[j])
        c, p, dof, expected = chi2_contingency(contigency_pct)
        print(i,j,":",p)
        mat[a][b]=p
        b=b+1
    a = a+1
    b=0
sns.heatmap(mat, annot=True, cmap="YlGnBu")<jupyter_output><empty_output><jupyter_text>(>=0.05 Independent)<jupyter_code># alpha = 0.0005 #should be 0.05, used to prune even more dependent values so that the graph has less edges
alpha = 5e-7
sns.heatmap((lambda x: x>=alpha)(mat), annot=True, cmap="YlGnBu")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17037037037037037, 'fraction_numerical': 0.02, 'mean_word_length': 5.558252427184466, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '653710', 'license': 'MIT', 'n_tokens_mistral': 561, 'n_tokens_neox': 527, 'n_words': 104, 'path': 'demo.ipynb', 'repo_name': 'drchapman-17/faikr-bn', 'size': '1350'}