<jupyter_start><jupyter_text>Univariate Frequency Table<jupyter_code>df_chl = df_chl.drop(columns=["Unnamed: 0", "P_ID"])
df_pat = df_pat.drop(columns=["Unnamed: 0", "P_ID"])
merged_freq_chl=pd.DataFrame() 
for index in range(128):
   result = pd.crosstab(index=df_chl.iloc[: , index], columns=df_chl.columns[index])
   merged_freq_chl = pd.merge(left=merged_freq_chl, right=result, how='outer', left_index=True, right_index=True)
merged_freq_chl
df_chl.loc[df_chl.C_CHOPI == 0, "C_CHOPI"] = 2

df_chl.loc[df_chl.C_CHOPB == 1, "C_CHOPB"] = 0
df_chl.loc[df_chl.C_CHOPB == 2, "C_CHOPB"] = 0

df_chl.loc[df_chl.C_CHOPF == 0, "C_CHOPF"] = 1

df_chl.loc[df_chl.C_SCHMISS == 0, "C_SCHMISS"] = 1

df_chl.loc[df_chl.C_CHOPH == 1, "C_CHOPH"] = 0
df_chl.loc[df_chl.C_CHOPH == 2, "C_CHOPH"] = 0

df_chl.loc[df_chl.C_CHOPG == 0, "C_CHOPG"] = 2

df_chl.loc[df_chl.C_CHOPK == 0, "C_CHOPK"] = 1
df_chl.loc[df_chl.C_CHOPK == 2, "C_CHOPK"] = 1

df_chl.loc[df_chl.C_CHOPL == 0, "C_CHOPL"] = 3

df_chl.loc[df_chl.C_CHOPC == 2, "C_CHOPC"] = 0

df_chl.loc[df_chl.C_CHOPD == 0, "C_CHOPD"] = 1

df_chl.loc[df_chl.C_CHOPE == 1, "C_CHOPE"] = 0

df_chl.loc[df_chl.C_CHOPA == 1, "C_CHOPA"] = 0
df_chl.loc[df_chl.C_CHOPA == 2, "C_CHOPA"] = 0

df_chl.loc[df_chl.C_PAIN == 1, "C_PAIN"] = 0

df_chl.loc[df_chl.C_PAINDISC == 0, "C_PAINDISC"] = 1

df_chl.loc[df_chl.C_TEELOOK == 0, "C_TEELOOK"] = 1

df_chl.loc[df_chl.C_BRUSH == 0, "C_BRUSH"] = 1

df_chl.loc[df_chl.C_CHOPJ == 1, "C_CHOPJ"] = 0
df_chl.loc[df_chl.C_CHOPJ == 2, "C_CHOPJ"] = 0

df_chl.loc[df_chl.C_VISIT == 1, "C_VISIT"] = 0
df_chl.loc[df_chl.C_VISIT == 2, "C_VISIT"] = 0

df_chl.loc[df_chl.C_PROFILE == 2, "C_PROFILE"] = 1

df_chl.loc[df_chl.C_SNACK == 4, "C_SNACK"] = 3
df_chl.loc[df_chl.C_SNACK == 5, "C_SNACK"] = 3

df_chl.loc[df_chl.C_SUGAR1 == 4, "C_SUGAR1"] = 3

df_chl.loc[df_chl.C_SUGAR2 == 12, "C_SUGAR2"] = 5
df_chl.loc[df_chl.C_SUGAR2 == 21, "C_SUGAR2"] = 5
df_chl.loc[df_chl.C_SUGAR2 == 52, "C_SUGAR2"] = 5
df_chl.loc[df_chl.E_D == 2, "E_D"] = 1
df_chl.loc[df_chl.E_D == 3, "E_D"] = 1
df_chl.loc[df_chl.E_D == 4, "E_D"] = 1
df_chl.loc[df_chl.E_D == 5, "E_D"] = 1
df_chl.loc[df_chl.E_D == 6, "E_D"] = 1
df_chl.loc[df_chl.E_D == 7, "E_D"] = 1

df_pat.loc[df_pat.E_D == 2, "E_D"] = 1
df_pat.loc[df_pat.E_D == 3, "E_D"] = 1
df_pat.loc[df_pat.E_D == 4, "E_D"] = 1
df_pat.loc[df_pat.E_D == 5, "E_D"] = 1
df_pat.loc[df_pat.E_D == 6, "E_D"] = 1
df_pat.loc[df_pat.E_D == 7, "E_D"] = 1
merged_freq_chl2=pd.DataFrame() 
for index in range(128):
   result = pd.crosstab(index=df_chl.iloc[: , index], columns=df_chl.columns[index])
   merged_freq_chl2 = pd.merge(left=merged_freq_chl2, right=result, how='outer', left_index=True, right_index=True)
merged_freq_chl2
merged_freq_pat=pd.DataFrame() 
for index in range(106):
   result = pd.crosstab(index=df_pat.iloc[: , index], columns=df_pat.columns[index])
   merged_freq_pat = pd.merge(left=merged_freq_pat, right=result, how='outer', left_index=True, right_index=True)
merged_freq_pat
df_pat.loc[df_pat.P_PHYFUNC == 1, "P_PHYFUNC"] = 0

df_pat.loc[df_pat.P_SWALLOW == 3, "P_SWALLOW"] = 2

df_pat.loc[df_pat.P_CLDHEALT == 0, "P_CLDHEALT"] = 1

df_pat.loc[df_pat.P_CHOPI == 0, "P_CHOPI"] = 2
df_pat.loc[df_pat.P_CHOPI == 1, "P_CHOPI"] = 2

df_pat.loc[df_pat.P_CHOPK == 0, "P_CHOPK"] = 3
df_pat.loc[df_pat.P_CHOPK == 2, "P_CHOPK"] = 3

df_pat.loc[df_pat.P_CHOPL == 0, "P_CHOPL"] = 3
df_pat.loc[df_pat.P_CHOPL == 2, "P_CHOPL"] = 3

df_pat.loc[df_pat.P_CHOPB == 0, "P_CHOPB"] = 2
df_pat.loc[df_pat.P_CHOPB == 1, "P_CHOPB"] = 2

df_pat.loc[df_pat.P_CHOPF == 0, "P_CHOPF"] = 2
df_pat.loc[df_pat.P_CHOPF == 1, "P_CHOPF"] = 2

df_pat.loc[df_pat.P_CHOPG == 0, "P_CHOPG"] = 2
df_pat.loc[df_pat.P_CHOPG == 1, "P_CHOPG"] = 2

df_pat.loc[df_pat.P_CHOPH == 0, "P_CHOPH"] = 1

df_pat.loc[df_pat.P_PAIN == 0, "P_PAIN"] = 1

df_pat.loc[df_pat.P_CHOPC == 0, "P_CHOPC"] = 2
df_pat.loc[df_pat.P_CHOPC == 1, "P_CHOPC"] = 2

df_pat.loc[df_pat.P_CHOPD == 0, "P_CHOPD"] = 1

df_pat.loc[df_pat.P_CHOPE == 0, "P_CHOPE"] = 2
df_pat.loc[df_pat.P_CHOPE == 1, "P_CHOPE"] = 2

df_pat.loc[df_pat.P_WORRY == 1, "P_WORRY"] = 0

df_pat.loc[df_pat.P_DEVELOP == 1, "P_DEVELOP"] = 0

df_pat.loc[df_pat.P_DEVELOP1 == 1, "P_DEVELOP1"] = 0

df_pat.loc[df_pat.P_ACCESS1 == 2, "P_ACCESS1"] = 0

df_pat.loc[df_pat.P_ACCESS5 == 1, "P_ACCESS5"] = 0

df_pat.loc[df_pat.P_CHOPJ == 1, "P_CHOPJ"] = 0
df_pat.loc[df_pat.P_CHOPJ == 2, "P_CHOPJ"] = 0

df_pat.loc[df_pat.P_SELFEFF == 0, "P_SELFEFF"] = 1

df_pat.loc[df_pat.P_SELFEFF1 == 0, "P_SELFEFF1"] = 1

df_pat.loc[df_pat.P_VISIT == 0, "P_VISIT"] = 3
df_pat.loc[df_pat.P_VISIT == 2, "P_VISIT"] = 3

df_pat.loc[df_pat.P_CAVITYTD == 2, "P_CAVITYTD"] = 1

df_pat.loc[df_pat.P_FILLINGA == 2, "P_FILLINGA"] = 1

df_pat.loc[df_pat.P_FILLINGT == 2, "P_FILLINGT"] = 1

df_pat.loc[df_pat.P_CROWNA == 2, "P_CROWNA"] = 1

df_pat.loc[df_pat.P_CROWNT == 2, "P_CROWNT"] = 1

df_pat.loc[df_pat.P_PROFILE == 2, "P_PROFILE"] = 1

df_pat.loc[df_pat.P_SNACK == 4, "P_SNACK"] = 3
df_pat.loc[df_pat.P_SNACK == 5, "P_SNACK"] = 3

df_pat.loc[df_pat.P_SUGAR1 == 3, "P_SUGAR1"] = 2
df_pat.loc[df_pat.P_SUGAR1 == 8, "P_SUGAR1"] = 2

df_pat.loc[df_pat.P_SUGAR2 == 3, "P_SUGAR2"] = 2
df_pat.loc[df_pat.P_SUGAR2 == 4, "P_SUGAR2"] = 2
df_pat.loc[df_pat.P_SUGAR2 == 11, "P_SUGAR2"] = 2

df_pat.loc[df_pat.P_BRUSH == 0, "P_BRUSH"] = 1

df_pat.loc[df_pat.P_INS == 2, "P_INS"] = 1
merged_freq_pat2=pd.DataFrame() 
for index in range(106):
   result = pd.crosstab(index=df_pat.iloc[: , index], columns=df_pat.columns[index])
   merged_freq_pat2 = pd.merge(left=merged_freq_pat2, right=result, how='outer', left_index=True, right_index=True)
merged_freq_pat2
df_chl.to_csv('CHL_scaledC.csv')
df_pat.to_csv('PAT_scaledC.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19396702911259206, 'fraction_numerical': 0.04103823219922834, 'mean_word_length': 5.15210355987055, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '180226', 'license': 'MIT', 'n_tokens_mistral': 3394, 'n_tokens_neox': 3159, 'n_words': 440, 'path': '01data_preparation/PROMIS_freq_larger_than_4.ipynb', 'repo_name': 'dixiong777/dixiong777-COH_SF_IRTML', 'size': '5702'}