<jupyter_start><jupyter_text>Transform 2020 Dataset<jupyter_code># Drop two columns fromm df_2020_train
df_2020_train.drop(['benign_malignant', 'target'], inplace=True, axis = 1)
# Replace Row values
df_2020_train = df_2020_train.replace({
    'diagnosis': {
        'unknown': 'UNK',
        'nevus': 'NV',
        'melanoma': 'MEL',
        'seborrheic keratosis': 'BKL',
        'lentigo NOS': 'BKL',
        'lichenoid keratosis': 'BKL',
        'solar lentigo': 'BKL',
        'cafe-au-lait macule': 'UNK',
        'atypical melanocytic proliferation': 'UNK'
    }
})
df_2020_train = df_2020_train.rename(columns= {
    'anatom_site_general_challenge': 'anatom_site_general',
    'image_name': 'image',
})

df_2020_test = df_2020_test.rename(columns= {
    'anatom_site_general_challenge': 'anatom_site_general',
    'image_name': 'image',
})
df_2020_train.diagnosis.unique()
df_2020_train.head()
df_2020_train[['anatom_site_general', 'sex']] = df_2020_train[['anatom_site_general', 'sex']].fillna(value = 'unknown')
df_2020_train['age_approx'] = df_2020_train['age_approx'].fillna(value = 50)

df_2020_test['anatom_site_general'] = df_2020_test['anatom_site_general'].fillna(value = 'unknown')
df_2020_test.isna().sum()
df_2020_train.isna().sum()<jupyter_output><empty_output><jupyter_text>Removed Duplicates<jupyter_code>df_2020_duplicates_train = df_2020_duplicates[df_2020_duplicates['partition'] == 'train']
df_2020_duplicates_test = df_2020_duplicates[df_2020_duplicates['partition'] == 'test']

df_2020_duplicates_train.reset_index(inplace=True, drop=True)
df_2020_duplicates_test.reset_index(inplace=True, drop=True)
df_2020_train.shape
df_2020_test.shape
remove_train = list(df_2020_duplicates_train['ISIC_id'].values)
remove_test = list(df_2020_duplicates_test['ISIC_id'].values)
df_2020_train = df_2020_train[~df_2020_train['image'].isin(remove_train)]
df_2020_test = df_2020_test[~df_2020_test['image'].isin(remove_test)]
print(df_2020_test.shape)
print(df_2020_train.shape)
df_2020_test.head()<jupyter_output><empty_output><jupyter_text>Transform 2019 Dataset
 
*Note*: 2019 Dataset is a strick Superset of 2018 dataset. Which means that all the 2018 data is included in the 2019 dataset. [Refer](https://forum.isic-archive.com/t/isic-2018-vs-isic-2019-dataset/1102)<jupyter_code>df_2019_train_GT.head()
# Convert Wide to Long Format
df_2019_train_GT_transformed = pd.melt(df_2019_train_GT, id_vars='image', var_name='diagnosis', value_name='value').\
    sort_values('image').\
    reset_index(drop=True)

# Select rows when diagnosis == 1
df_2019_train_GT_transformed = df_2019_train_GT_transformed[df_2019_train_GT_transformed['value'] == 1]
df_2019_train_GT_transformed.drop('value', inplace=True, axis = 1)
df_2019_train_GT_transformed.reset_index(inplace = True, drop = True)
df_2019_train_GT_transformed.head()
# Combine Train metadata and groundtruth variable.
df_2019_train_GT_combined = df_2019_train_GT_transformed.merge(df_2019_train_MD, on='image')
df_2019_train_GT_combined.drop('lesion_id', inplace=True, axis = 1)
df_2019_train_GT_combined.head()
df_2019_train_GT_combined[['anatom_site_general', 'sex']] = df_2019_train_GT_combined[['anatom_site_general', 'sex']].\
    fillna(value = 'unknown')
df_2019_train_GT_combined['age_approx'] = df_2019_train_GT_combined['age_approx'].fillna(value = 55)

df_2019_test_MD[['anatom_site_general', 'sex']] = df_2019_test_MD[['anatom_site_general', 'sex']].fillna(value = 'unknown')
df_2019_test_MD['age_approx'] = df_2019_test_MD['age_approx'].fillna(value = 60)
df_2019_train_GT_combined.isna().sum()
df_2019_test_MD.isna().sum()
df_2019_train_GT_combined.shape
df_2019_test_MD.shape<jupyter_output><empty_output><jupyter_text>Transform 2018 Dataset<jupyter_code>df_2018_valid_GT
# Convert Wide to Long Format
df_2018_valid_GT_transformed = pd.melt(df_2018_valid_GT, id_vars='image', var_name='diagnosis', value_name='value').\
    sort_values('image').\
    reset_index(drop=True)

# Select rows when diagnosis == 1
df_2018_valid_GT_transformed = df_2018_valid_GT_transformed[df_2018_valid_GT_transformed['value'] == 1]
df_2018_valid_GT_transformed.drop('value', inplace=True, axis = 1)
df_2018_valid_GT_transformed.reset_index(inplace = True, drop = True)
# Replace Row values
df_2018_valid_GT_transformed = df_2018_valid_GT_transformed.replace({
    'diagnosis': {
        'BCC': 'UNK',
        'AKIEC': 'UNK',
        'DF': 'UNK',
        'VASC': 'UNK'
    }
})
df_2018_valid_GT_transformed.head()<jupyter_output><empty_output><jupyter_text>Combining Dataframe<jupyter_code>df_2019_train_GT_combined.head()
df_2020_train.head()
df_2020_select = df_2020_train[['image', 'sex', 'age_approx', 'anatom_site_general', 'diagnosis']]
df_2019_select = df_2019_train_GT_combined[['image', 'sex', 'age_approx', 'anatom_site_general', 'diagnosis']]
# PD = Patient Demographic
df_2020_2019_PD = df_2020_select.append(df_2019_select, ignore_index=True)
df_2020_2019_PD.head()
df_2020_select_2 = df_2020_train[['image', 'diagnosis']]
df_2019_select_2 = df_2019_train_GT_combined[['image', 'diagnosis']]
df_2020_2019_9_label = df_2020_select_2.append(df_2019_select_2, ignore_index=True)
# Replace Row values
df_2020_2019_4_label = df_2020_2019_9_label.replace({
    'diagnosis': {
        'BCC': 'UNK',
        'AK': 'UNK',
        'SCC': 'UNK',
        'VASC': 'UNK',
        'DF': 'UNK'
    }
})<jupyter_output><empty_output><jupyter_text>*Note*: we are only selecting 2020 Test set as we do not have cropped images available for the 2019 dataset.<jupyter_code>df_2020_test_PD = df_2020_test[['image', 'sex', 'age_approx', 'anatom_site_general']]
df_2020_test_no_PD = df_2020_test[['image']]
#df_2019_test_select = df_2019_test_MD[['image', 'sex', 'age_approx', 'anatom_site_general']]
# PD = Patient Demographic
#df_2020_2019_PD_test = df_2020_test_select.append(df_2019_test_select, ignore_index=True)
df_2020_test_PD.head()
df_2020_test_no_PD.head()
print(df_2020_test_PD.shape)
print(df_2020_test_no_PD.shape)<jupyter_output>(10875, 4)
(10875, 1)<jupyter_text>Remove some of the rows from the CSV as the images are corrupted<jupyter_code>####### Test ########
# Test with Patient Demographic
#df_2020_test_PD.to_csv(save_path + 'test_2020_withPateintDetail.csv', header=True, index=False)

# Test no Patient Demographic
#df_2020_test_no_PD.to_csv(save_path + 'test_2020_no_PateintDetail.csv', header=True, index=False)

############ Train ##########
# Train save with pateint Demographic
df_2020_2019_PD = df_2020_2019_PD[df_2020_2019_PD['image'] != 'ISIC_0066580']
df_2020_2019_PD.reset_index(inplace=True, drop=True)

# Train save with no pateint Demographic and has 9 labels
#df_2020_2019_9_label.to_csv(save_path + 'train_2020_and_2019_with_9_Labels.csv', header=True, index=False)
df_2020_2019_9_label = df_2020_2019_9_label[df_2020_2019_9_label['image'] != 'ISIC_0066580']
df_2020_2019_9_label.reset_index(inplace=True, drop=True)

# Train save with no pateint Demographic and has 4 labels
#df_2020_2019_4_label.to_csv(save_path + 'train_2020_and_2019_with_4_Labels.csv', header=True, index=False)
df_2020_2019_4_label = df_2020_2019_4_label[df_2020_2019_4_label['image'] != 'ISIC_0066580']
df_2020_2019_4_label.reset_index(inplace=True, drop=True)

# Validation set
#df_2018_valid_GT_transformed.to_csv(save_path + 'valid_2018_noPateintDetail.csv', header=True, index=False)<jupyter_output><empty_output><jupyter_text>Save to CSV<jupyter_code>df_2020_2019_9_label.shape
save_path = "Data//Processed CSV's//"

####### Test ########
# Test save with pateint Demographic
#df_2020_2019_PD_test.to_csv(save_path + 'test_2020_and_2019_withPateintDetail.csv', header=True, index=False)

# Test with Patient Demographic
df_2020_test_PD.to_csv(save_path + 'test_2020_withPateintDetail.csv', header=True, index=False)

# Test no Patient Demographic
df_2020_test_no_PD.to_csv(save_path + 'test_2020_no_PateintDetail.csv', header=True, index=False)

############ Train ##########
# Train save with pateint Demographic
df_2020_2019_PD.to_csv(save_path + 'train_2020_and_2019_withPateintDetail_9_labels.csv', header=True, index=False)

# Train save with no pateint Demographic and has 9 labels
df_2020_2019_9_label.to_csv(save_path + 'train_2020_and_2019_with_9_Labels.csv', header=True, index=False)

# Train save with no pateint Demographic and has 4 labels
df_2020_2019_4_label.to_csv(save_path + 'train_2020_and_2019_with_4_Labels.csv', header=True, index=False)

# Validation set
df_2018_valid_GT_transformed.to_csv(save_path + 'valid_2018_noPateintDetail.csv', header=True, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1282288891462991, 'fraction_numerical': 0.09591103903625622, 'mean_word_length': 7.774390243902439, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '525482', 'license': 'MIT', 'n_tokens_mistral': 3938, 'n_tokens_neox': 3258, 'n_words': 478, 'path': 'Notebooks/Data Preprocessing.ipynb', 'repo_name': 'akshaybankapure/Skin-Cancer-Classification-using-Deep-Learning', 'size': '8633'}