<jupyter_start><jupyter_text>Take a Quick Look at the Data Structure<jupyter_code>train_df.head()
train_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 223414 entries, 0 to 223413
Data columns (total 19 columns):
 #   Column                      Non-Null Count   Dtype  
---  ------                      --------------   -----  
 0   Path                        223414 non-null  object 
 1   Sex                         223414 non-null  object 
 2   Age                         223414 non-null  int64  
 3   Frontal/Lateral             223414 non-null  object 
 4   AP/PA                       191027 non-null  object 
 5   No Finding                  22381 non-null   float64
 6   Enlarged Cardiomediastinum  44839 non-null   float64
 7   Cardiomegaly                46203 non-null   float64
 8   Lung Opacity                117778 non-null  float64
 9   Lung Lesion                 11944 non-null   float64
 10  Edema                       85956 non-null   float64
 11  Consolidation               70622 non-null   float64
 12  Pneumonia                   27608 non-null   float64
 13  Atelectasis   [...]<jupyter_text>check non-numeric clumn values<jupyter_code>for (column, t) in zip(train_df.columns, train_df.dtypes):
    if column != 'Path' and t == object:
        print(column)
        a = train_df[column].unique()
        print(a)
for fl in ['Frontal', 'Lateral']:
    print(fl)
    print(train_df[train_df['Frontal/Lateral']==fl]['AP/PA'].unique())
def encode_categorial_features(df):
    df['Frontal/Lateral'] = df['Frontal/Lateral'].map({'Frontal': 1, 'Lateral': 0})
    df['Sex'] = df['Sex'].map({'Male': 1, 'Female': 0, 'Unknown': np.nan})
    df['AP/PA'] = df['AP/PA'].map({'AP': 1, 'PA': 2, 'LL': 3,  'RL':4})
    return df
train_df = encode_categorial_features(train_df)
test_df = encode_categorial_features(test_df)
train_df.describe()
train_df['Age'].hist()<jupyter_output><empty_output><jupyter_text>Looking for Corelations<jupyter_code>obs_columns = train_df.columns[5::]
obs_columns
corr = train_df.corr()
# drop No Finding because it only has 1 calsses
corr = corr.drop("No Finding", axis=0).drop("No Finding", axis=1)

fig = plt.figure(figsize=(30, 15))
sns.set(font_scale=3)
xticklabels = [
    'Sex', 'Age', 'Frontal/Lateral', 'AP/PA', 'Enlarged Card', 'Cardiomegaly',
    'Lung Opacity', 'Lung Lesion', 'Edema', 'Consolidation', 'Pneumonia',
    'Atelectasis', 'Pneumothorax', 'Pleural Effusion', 'Pleural Other',
    'Fracture', 'Support Devices'
]
ax = sns.heatmap(corr.iloc[4:, :],
                 xticklabels=xticklabels,
                 vmin=-1,
                 vmax=1,
                 cmap='PuOr',
                 annot=True,
                 fmt=".2f",
                 annot_kws={"fontsize": "small"})<jupyter_output><empty_output><jupyter_text>Classes weight<jupyter_code>obs_df = train_df[obs_columns]
value_count_df = obs_df.apply(lambda c: c.value_counts()).T
value_count_df
weight_df = value_count_df[1.0] / (value_count_df[1.0] + value_count_df[0])
weight_df.apply(lambda x: round(x, 2)).to_frame()
def calculate_classes_distribution(df, sort=True):
    df_size = len(df)
    classes = [('Positive(%)', 1.0), ("Uncertain(%)", -1.0), ('Negative(%)', 0)]
    result = {}

    for o in obs_columns:
        result[o] = {}
        for n, v in classes:
            count = df[df[o]==v][o].count()
            result[o][n] = (count, round(count/df_size * 100, 2))

    result_df = pd.DataFrame(result)
    if sort:
        result_df = result_df.sort_values(by='Positive(%)', axis=1, ascending=False).applymap(lambda v: f"{v[0]}({v[1]})")
    return result_df.T


print(len(train_df))
calculate_classes_distribution(train_df, sort=True)
print(len(test_df))
calculate_classes_distribution(test_df, sort=True)<jupyter_output>234
{'fraction_non_alphanumeric': 0.14391143911439114, 'fraction_numerical': 0.04322614654717976, 'mean_word_length': 2.7096774193548385, 'pattern_counts': {'":': 1, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '43623', 'license': 'MIT', 'n_tokens_mistral': 1463, 'n_tokens_neox': 1311, 'n_words': 274, 'path': 'notebooks/eda_chexpert.ipynb', 'repo_name': 'jiahui890/chexpert-aml', 'size': '3794'}