<jupyter_start><jupyter_text>Config<jupyter_code># config
DEBUG = False # set False to do all preprocess
input_dir = "/mnt/disks/data5/rsna2020"
df_train_path = "{}/train.csv".format(input_dir)
train_dir_path = "{}/train".format(input_dir)
# input_dir = "../input/orig"
preprocess_dir = "/mnt/disks/data6/rsna2020/preprocessed"
# preprocess_dir = "../input/preprocess"
train_npy_dir_path = "{}/train_law_npy".format(preprocess_dir)
os.makedirs(preprocess_dir, exist_ok=True)
os.makedirs(train_npy_dir_path, exist_ok=True)
col_index = 'SOPInstanceUID'
col_groupby = 'StudyInstanceUID'
col_targets = [
    'negative_exam_for_pe',
    'indeterminate',
    'chronic_pe',
    'acute_and_chronic_pe',
    'central_pe',
    'leftsided_pe',
    'rightsided_pe',
    'rv_lv_ratio_gte_1',
    'rv_lv_ratio_lt_1',
    'pe_present_on_image',
]
NUM_CLASS = len(col_targets)
print('NUM_CLASS: {}'.format(NUM_CLASS))<jupyter_output>NUM_CLASS: 10<jupyter_text>Data loading<jupyter_code>DATA_DIR = "/mnt/disks/data4/rsna2020"
INPUT_DIR = DATA_DIR + "/orig"
PREPROCESSED_DIR = DATA_DIR + "/preprocessed"
df_train = pickle_load("{}/df_train2.pkl".format(PREPROCESSED_DIR))
print(df_train.shape)
df_train.head()
# make exam-level train data
df_train_exam = ri(df_train[df_train[col_groupby].duplicated()==False])
print(df_train_exam.shape)
df_train_exam.head()
df_train['num_series'] = df_train['num_images']
df_train['dicom_path'] = df_train['path']
# get npy path
df_train['npy_path'] = (
      train_npy_dir_path + '/'
    + df_train['SOPInstanceUID'] + ".npy"
)
print(df_train['npy_path'][0])
df_train.head()
# get exam index
df_train_exam['exam_index'] = np.arange(len(df_train_exam))
df_train = pd.merge(df_train, df_train_exam[[col_groupby, 'exam_index']], on=col_groupby, how='left')
df_train.head()
# sort data by series index
df_train = pd.merge(df_train_exam[[col_groupby]], df_train.sort_values('series_index'), how='left')
print(df_train.shape)
df_train.head()
df_tmp = df_train.groupby(col_groupby)['pe_present_on_image'].agg('sum').reset_index()
df_tmp.columns = [col_groupby, 'm_i']
df_train = pd.merge(df_train, df_tmp[[col_groupby, 'm_i']], on=col_groupby, how='left')
df_train['q_i'] = df_train['m_i'] / df_train['num_images'] 
df_train.head()
df_tmp = pd.read_csv("/mnt/disks/data4/rsna2020/preprocessed/df_train_meta.csv")
df_train = pd.merge(df_train, df_tmp[[col_groupby, 'RescaleIntercept', 'RescaleSlope', 'PatientPosition']], on=col_groupby, how='left')
df_tmp.head()
df_train = df_train[[
   
'StudyInstanceUID',
'SeriesInstanceUID',
'SOPInstanceUID',
'pe_present_on_image',
'negative_exam_for_pe',
'qa_motion',
'qa_contrast',
'flow_artifact',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'true_filling_defect_not_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate',
'dicom_path',
# 'z_pos',
'RescaleSlope',
'RescaleIntercept',
'PatientPosition',
'series_index',
'num_series',
'm_i',
'q_i',
'npy_path',
'exam_index', 
]]
# save
df_train.to_csv("{}/df_train.csv".format(preprocess_dir), index=None)
df_train.head()
df_train = df_train.iloc[:, :-4]
# postprocess config
# local
# config
# input_dir = "../input/orig"
# preprocess_dir = "../input/preprocess"
input_dir = "/mnt/disks/data5/rsna2020"
preprocess_dir = "/mnt/disks/data6/rsna2020/preprocessed"
df_train_path = "{}/df_train2.csv".format(preprocess_dir)
train_dir_path = "{}/train".format(input_dir)
train_npy_dir_path = "{}/train_law_npy".format(preprocess_dir)
load_dir_b0 = "/mnt/disks/data4/rsna2020/output/201025_3_2ndNNs_features_pool_flip"
pred_name_b0 = "cnn_preds"
load_dir_b2 = "/mnt/disks/data4/rsna2020/output/201026_3_2ndNNs_features_pool_flip_b0b2"
pred_name_b2 = "cnn_preds"
NUM_FOLD = 5



# github
input_dir = "../input/orig"
preprocess_dir = "../input/preprocess"
df_train_path = "{}/df_train.csv".format(preprocess_dir)
load_dir_b0 = "../output"
pred_name_b0 = "cnn_preds_b0"
load_dir_b2 = "../output"
pred_name_b2 = "cnn_preds_b2"
NUM_FOLD = 5
# preprocess config
# config
DEBUG = True # set False to do all preprocess
input_dir = "../input/orig"
preprocess_dir = "../input/preprocess"
# input_dir = "/mnt/disks/data5/rsna2020"
# preprocess_dir = "/mnt/disks/data6/rsna2020/preprocessed"
df_train_path = "{}/train.csv".format(input_dir)
train_dir_path = "{}/train".format(input_dir)
train_npy_dir_path = "{}/train_law_npy".format(preprocess_dir)
os.makedirs(preprocess_dir, exist_ok=True)
os.makedirs(train_npy_dir_path, exist_ok=True)
# train_stage1 config
# config
DEBUG = True # set False to do all process
preprocess_dir = "/mnt/disks/data6/rsna2020/preprocessed"
output_dir = "/mnt/disks/data6/rsna2020/output"
# preprocess_dir = "../input/preprocess"
# output_path = "../output"
df_train_path = "{}/df_train.csv".format(preprocess_dir)
# train_npy_dir_path = "{}/train_law_npy".format(preprocess_dir)
train_npy_dir_path = "/mnt/disks/data4/rsna2020/preprocessed/train_law_npy"
os.makedirs(output_dir, exist_ok=True)
# train_stage2　config
DEBUG = True # set False to do all process
preprocess_dir = "/mnt/disks/data6/rsna2020/preprocessed"
output_dir = "/mnt/disks/data6/rsna2020/output"
# output_dir = "../output"
# preprocess_dir = "../input/preprocess"
# output_path = "../output"
df_train_path = "{}/df_train.csv".format(preprocess_dir)
# train_npy_dir_path = "{}/train_law_npy".format(preprocess_dir)
train_npy_dir_path = "/mnt/disks/data4/rsna2020/preprocessed/train_law_npy"
feature_dir = "/mnt/disks/data4/rsna2020/output/201022_6_CNN_b0_1loss_512_fp16_bs80_lr1e3"
os.makedirs(output_dir, exist_ok=True)
# config predict
DEBUG = True # set False to do all process
col_index = 'SOPInstanceUID'
col_groupby = 'StudyInstanceUID'
# df_test_path = "../input/orig/test.csv"
# df_sub_path = "../input/orig/sample_submission.csv"
# test_image_path = "../input/orig/test/"

df_test_path = "/mnt/disks//data5/rsna2020/test.csv"
df_sub_path = "/mnt/disks/data5/rsna2020/sample_submission.csv"
test_image_path = "/mnt/disks/data5/rsna2020/test"

num_cpu = multiprocessing.cpu_count()
num_features_b0 = 1280
num_features_b2 = 1408
weight_dir_b0_1 = "../output/b0_stage1"
weight_dir_b0_2 = "../output/b0_stage2"
weight_dir_b2_1 = "../output/b2_stage1"
weight_dir_b2_2 = "../output/b2_stage2"
weight_dir_b0_1 = "../models/b0_stage1"
weight_dir_b0_2 = "../models/b0_stage2"
weight_dir_b2_1 = "../models/b2_stage1"
weight_dir_b2_2 = "../models/b2_stage2"
BATCH_SIZE = 64
NUM_FOLD = 5<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1558038485412787, 'fraction_numerical': 0.03181253879577902, 'mean_word_length': 6.58235294117647, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '333918', 'license': 'MIT', 'n_tokens_mistral': 2794, 'n_tokens_neox': 2663, 'n_words': 350, 'path': 'notebook/extra_processing.ipynb', 'repo_name': 'OsciiArt/Kaggle_RSNA2020_6th_Solution', 'size': '6444'}