<jupyter_start><jupyter_text>EDA  Importing the libraries<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os
from glob import glob
from skimage.io import imread
from collections import defaultdict
from matplotlib.collections import PatchCollection
from matplotlib.patches import Rectangle
%matplotlib inline
def create_boxes(in_row):
    box_list = []
    for (start_x, start_y, end_x, end_y) in in_row['bbox']:
        box_list += [Rectangle((start_x, start_y), 
                         np.abs(end_x-start_x),
                         np.abs(end_y-start_y)
                         )]
    return box_list<jupyter_output><empty_output><jupyter_text>Directory of File<jupyter_code>image_dir = "minideeplesion/"<jupyter_output><empty_output><jupyter_text>Importing and cleaning the dataset<jupyter_code>df = pd.read_csv("DL_info.csv")
df.head()
df.info()
df['def_path'] = df.apply(lambda c_row: os.path.join(image_dir, '{Patient_index:06d}_{Study_index:02d}_{Series_ID:02d}'.format(**c_row),
                                                                        '{Key_slice_index:03d}.png'.format(**c_row)), 1)
# df.head()
print('Loaded', df.shape[0], 'cases')
# Getting the boundaries
df['bbox'] = df['Bounding_boxes'].map(lambda x: np.reshape([float(y) for y in x.split(',')], (-1, 4)))
df['norm_loc'] = df['Normalized_lesion_location'].map(lambda x: np.reshape([float(y) for y in x.split(',')], (-1)))
df['Spacing_mm_px_'] = df['Spacing_mm_px_'].map(lambda x: np.reshape([float(y) for y in x.split(',')], (-1)))
df['Lesion_diameters_Pixel_'] = df['Lesion_diameters_Pixel_'].map(lambda x: np.reshape([float(y) for y in x.split(',')], (-1)))
df['Radius_x'] = df.apply(lambda x: x['Lesion_diameters_Pixel_'][0]*x['Spacing_mm_px_'][0], 1)]
lesion_type_dict = dict(enumerate('Bone,Abdomen,Mediastinum,Liver,Lung,Kidney,Soft tissue,Pelvis'.split(','), 1))

df['Coarse_lesion_name'] = df['Coarse_lesion_type'].map(lambda x: lesion_type_dict.get(x, 'Unknown'))
for i, ax in enumerate('xyz'):
    df[f'{ax}_loc'] = df['norm_loc'].map(lambda x: x[i])
print('Found', df.shape[0], 'patients with images')
df.head(3)<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code>sns.pairplot(hue='Coarse_lesion_name', 
             data = df[['Patient_age', 'Coarse_lesion_name', 'Key_slice_index', 'Radius_x']])
freq_flyers_df = df.groupby('Patient_index')[['Patient_age']].apply(
    lambda x: pd.Series({'counts': x.shape[0], 
                         'Age_Range': np.max(x['Patient_age'])-np.min(x['Patient_age'])})).reset_index().sort_values('Age_Range', ascending = False)
sns.pairplot(freq_flyers_df[['counts', 'Age_Range']])

join_df = pd.merge(df, freq_flyers_df.head(15))
ax = sns.lmplot(x='Patient_age', y='Radius_x', ci=50,
                hue = 'Coarse_lesion_name',
                sharex=False, sharey=False, x_jitter=0.5,
                 col='Patient_index', col_wrap=5,
                data = join_df)
sns.lmplot(x='Study_index', y='Radius_x', ci=50,
                hue = 'Coarse_lesion_name',
                sharex=False, sharey=False, x_jitter=0.5,
                 col='Patient_index', col_wrap=5,
                data = join_df)
def count_and_check_studies(in_patient_df):
    gr_df = in_patient_df.groupby('Study_index').size().reset_index(name='counts')
    match_df = gr_df[gr_df['counts']==gr_df['counts'].max()]
    
    if (gr_df['counts'].max()>1) and (match_df.shape[0]>1): # more than one study and more than one series 
        return in_patient_df[in_patient_df['Study_index'].isin(match_df['Study_index'])]
    else:
        return in_patient_df.head(0)
grp_patient_df = df.groupby(['Patient_index']).apply(count_and_check_studies).reset_index(drop = True)
print(grp_patient_df.shape[0], 'scans available')
print(len(grp_patient_df['Patient_index'].value_counts()), 'patients')
print(len(grp_patient_df.groupby(['Patient_index', 'Study_index'])), 'studies')
grp_patient_df.head(5)<jupyter_output>3313 scans available
419 patients
962 studies
{'fraction_non_alphanumeric': 0.16323492929794095, 'fraction_numerical': 0.012403870007442322, 'mean_word_length': 4.515731874145007, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '616451', 'license': 'MIT', 'n_tokens_mistral': 1533, 'n_tokens_neox': 1472, 'n_words': 237, 'path': 'CT_analysis_modelling/eda.ipynb', 'repo_name': 'shaswat01/Deep-XT', 'size': '4031'}