<jupyter_start><jupyter_text>Config<jupyter_code># config
DEBUG = False # set False to do all preprocess
input_dir = "../input/orig"
preprocess_dir = "../input/preprocess"
df_train_path = "{}/train.csv".format(input_dir)
train_dir_path = "{}/train".format(input_dir)
train_npy_dir_path = "{}/train_law_npy".format(preprocess_dir)
os.makedirs(preprocess_dir, exist_ok=True)
os.makedirs(train_npy_dir_path, exist_ok=True)
col_index = 'SOPInstanceUID'
col_groupby = 'StudyInstanceUID'
col_targets = [
    'negative_exam_for_pe',
    'indeterminate',
    'chronic_pe',
    'acute_and_chronic_pe',
    'central_pe',
    'leftsided_pe',
    'rightsided_pe',
    'rv_lv_ratio_gte_1',
    'rv_lv_ratio_lt_1',
    'pe_present_on_image',
]
NUM_CLASS = len(col_targets)
print('NUM_CLASS: {}'.format(NUM_CLASS))<jupyter_output>NUM_CLASS: 10<jupyter_text>Data loading<jupyter_code># load train data
df_train = pd.read_csv(df_train_path)
if DEBUG:
    df_train = ri(df_train.iloc[:1000])
print(df_train.shape)
df_train.head()<jupyter_output>(1000, 17)<jupyter_text>Dicom Data Extraction<jupyter_code># make exam-level train data
df_train_exam = ri(df_train[df_train[col_groupby].duplicated()==False])
print(df_train_exam.shape)
df_train_exam.head()
# add dicom path column
df_train['dicom_path'] = train_dir_path + "/" + \
                         df_train[col_groupby].values + "/" + \
                         df_train['SeriesInstanceUID'].values + "/" + \
                         df_train[col_index].values + ".dcm"
print(df_train['dicom_path'][0])
df_train.head()
# load dicom and save meta data and images
starttime = time.time()

def task(i):
    if (i+1)%100000==0:
        print("{}/{} {:.1f}".format(i+1, len(df_train), time.time()-starttime))
        
    # load dicom
    path = df_train['dicom_path'][i]
    tmp_dcm = pydicom.dcmread(path)
    
    # extract meta data
    image_id = df_train[col_index][i]
    z_pos = tmp_dcm.ImagePositionPatient[-1]
    RescaleSlope = tmp_dcm['RescaleSlope'].value
    RescaleIntercept = tmp_dcm['RescaleIntercept'].value
    PatientPosition = tmp_dcm['PatientPosition'].value
    
    # save npy
    new_path = "{}/{}.npy".format(train_npy_dir_path, image_id)
    tmp_npy = np.asarray(tmp_dcm.pixel_array)
    np.save(new_path, tmp_npy)
    
    return [image_id, z_pos, RescaleSlope, RescaleIntercept, PatientPosition]

executor = ProcessPoolExecutor(max_workers=multiprocessing.cpu_count())
# executor = ProcessPoolExecutor(max_workers=2)
futures = [executor.submit(task, i) for i in range(len(df_train.iloc[:]))]
z_pos= []
RescaleSlope = []
RescaleIntercept = []
PatientPosition = []
for i in range(len(futures)):
    z_pos.append(futures[i].result()[0])
    RescaleSlope.append(futures[i].result()[1])
    RescaleIntercept.append(futures[i].result()[2])
    PatientPosition.append(futures[i].result()[3])
df_meta = df_train[[col_index]]
df_meta['z_pos'] = z_pos
df_meta['RescaleSlope'] = RescaleSlope
df_meta['RescaleIntercept'] = RescaleIntercept
df_meta['PatientPosition'] = PatientPosition
df_meta.head()<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>df_train = pd.merge(df_train, df_meta, on=col_index, how='left')
df_train.head()
# get series index of each exam
df_tmps = []
for i in range(len(df_train_exam)):
    exam = df_train_exam[col_groupby][i]
    df_tmp = ri(df_train[df_train[col_groupby]==exam])
    df_tmp = ri(df_tmp.sort_values('z_pos'))
    df_tmp['series_index'] = np.arange(len(df_tmp))
    df_tmps.append(df_tmp)
df_tmps = pd.concat(df_tmps)
df_train = pd.merge(df_train, df_tmps[[col_index, 'series_index']], on=col_index, how='left')
df_train.head()
# sort data by series index
df_train = pd.merge(df_train_exam[[col_groupby]], df_train.sort_values('series_index'), how='left')
print(df_train.shape)
df_train.head()
# calc the number of series
df_tmp = df_train.groupby(col_groupby)[col_index].agg(len).reset_index()
df_tmp.columns = [col_groupby, 'num_series']
df_train = pd.merge(df_train, df_tmp, on=col_groupby)
df_train.head()
# calc q_i
df_tmp = df_train.groupby(col_groupby)['pe_present_on_image'].agg('sum').reset_index()
df_tmp.columns = [col_groupby, 'm_i']
df_train = pd.merge(df_train, df_tmp, on=col_groupby, how='left')
df_train['q_i'] = df_train['m_i'] / df_train['num_series'] 
df_train.head()
# get npy path
df_train['npy_path'] = (
      train_npy_dir_path + '/'
    + df_train['SOPInstanceUID'] + ".npy"
)
print(df_train['npy_path'][0])
df_train.head()
# get exam index
df_train_exam['exam_index'] = np.arange(len(df_train_exam))
df_train = pd.merge(df_train, df_train_exam[[col_groupby, 'exam_index']], on=col_groupby, how='left')
df_train.head()
# save
df_train.to_csv("{}/df_train.csv".format(preprocess_dir), index=None)
for col in df_train.columns:
    print("'{}',".format(col))<jupyter_output>'StudyInstanceUID',
'SeriesInstanceUID',
'SOPInstanceUID',
'pe_present_on_image',
'negative_exam_for_pe',
'qa_motion',
'qa_contrast',
'flow_artifact',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'true_filling_defect_not_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate',
'dicom_path',
'z_pos',
'RescaleSlope',
'RescaleIntercept',
'PatientPosition',
'series_index',
'num_series',
'm_i',
'q_i',
'npy_path',
'exam_index',
{'fraction_non_alphanumeric': 0.1547641598787649, 'fraction_numerical': 0.0064406137526046596, 'mean_word_length': 5.947368421052632, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '243811', 'license': 'MIT', 'n_tokens_mistral': 2159, 'n_tokens_neox': 2097, 'n_words': 297, 'path': 'notebook/preprocess.ipynb', 'repo_name': 'OsciiArt/Kaggle_RSNA2020_6th_Solution', 'size': '5279'}