<jupyter_start><jupyter_text>Visualizing Albumnetation Transformations Notebook Contents- Importing Packages and Dataframe  - Assigning Paths- Displaying Melanoma Images- Displaying Non-Melanoma Images   - Albumentations Transformations    - Transpose    - Vertical Flip    - Horizontal Flip    - Random Brightness    - Random Contrast    - CLAHE    - Hue Saturation Value    - Shift Scale Rotate    - Cutout    - Motion Blur    - Median Blur    - Gaussian Blur    - Gaussian Noise    - Optical Distortion    - Grid Distortion    - Elastic Transform- Creating Transformations- First Train-Validation Split- Second Train-Validation Split   Importing Packages and Dataframe<jupyter_code>import pandas as pd
import numpy as np
import cv2
from skimage import io
from glob import glob
import os
import pydicom
import matplotlib.pyplot as plt
import albumentations as A
from PIL import Image
from skimage.color import label2rgb

import warnings
warnings.simplefilter("ignore")
# importing dataframe
df = pd.read_csv("../data/tr.csv", index_col=0)
df.head()
df = df[['file', 'target']]
df.head()<jupyter_output><empty_output><jupyter_text>Assigning Paths<jupyter_code># === DICOM ===
# Create the paths
path_tr_dcm = '../split/train_dcm/' + df['file'] + '.dcm'
path_tr_jpg = '../split/train/mel/' + df['file'] + '.jpg'

# Append to the original dataframes
df['path_dcm'] = path_tr_dcm
df['path_jpg'] = path_tr_jpg
df['dcm'] = df.file.apply(lambda x: str(x) + '.dcm')
df['jpg'] = df.file.apply(lambda x: str(x) + '.jpg')
df.head()<jupyter_output><empty_output><jupyter_text>Displaying Melanoma Images<jupyter_code># display sample of melanoma images
plt.style.use('seaborn')
paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=3, ncols=5, figsize=(16,6))
plt.suptitle('Melanoma Images', fontsize=16)
for i in range(0, 15):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    x = i // 5
    y = i % 5
    axes[x, y].imshow(image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Displaying Non-Melanoma Images<jupyter_code># display sample of non-melanoma images
paths = glob('../split/train/not_mel/*.jpg')
fig, axes = plt.subplots(nrows=3, ncols=5, figsize=(16,6))
plt.suptitle('Non-Melanoma Images', fontsize=16)
for i in range(0, 15):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    x = i // 5
    y = i % 5
    axes[x, y].imshow(image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Albumentations Transformations  Transpose<jupyter_code># display Transpose augmentations (rotates image by )
transform = A.Compose([
    A.Transpose()
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16, 6))
plt.suptitle('Transpose', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256, 256))
    transformed = transform(image=image)
    aug_image = transformed['image']
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Vertical Flip<jupyter_code># display Vertical Flip augmentations
transform = A.Compose([
    A.VerticalFlip()
])
paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16, 6))
plt.suptitle('VerticalFlip', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256, 256))
    transformed = transform(image=image)
    aug_image = transformed['image']
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Horizontal Flip<jupyter_code># display Horizontal Flip augmentations
transform = A.Compose([
    A.HorizontalFlip()
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('HorizontalFlip', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Random Brightness<jupyter_code># display Random Brightness augmentations
transform = A.Compose([
    A.RandomBrightness(limit=0.3)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('RandomBrightness', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Random Contrast<jupyter_code># display Random Contrast augmentations
transform = A.Compose([
    A.RandomContrast(limit=0.3)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('RandomContrast', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>CLAHE<jupyter_code># display CLAHE augmentations
transform = A.Compose([
    A.CLAHE(clip_limit=6.0)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('CLAHE', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Hue Saturation Value<jupyter_code># display Hue Saturation Value augmentations
transform = A.Compose([
    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('HueSaturationValue', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Shift Scale Rotate<jupyter_code># display Shift Scale Rotate augmentations
transform = A.Compose([
    A.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=30, border_mode=0)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('ShiftScaleRotate', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Cutout<jupyter_code># display Cutout augmentations
image_size = 128

transform = A.Compose([
    A.Cutout(max_h_size=32, max_w_size=32, num_holes=8)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('Cutout', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Motion Blur<jupyter_code># display Motion Blur augmentations
transform = A.Compose([
    A.MotionBlur(blur_limit=5)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('MotionBlur', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Median Blur<jupyter_code># display Median Blur augmentations
transform = A.Compose([
    A.MedianBlur(blur_limit=7)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('MedianBlur', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Gaussian Blur<jupyter_code># display Gaussian Blur augmentations
transform = A.Compose([
    A.GaussianBlur(blur_limit=7)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('GaussianBlur', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Gaussian Noise<jupyter_code># display Gaussian Noise augmentations
transform = A.Compose([
    A.GaussNoise(var_limit=(5.0, 30.0))
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('GaussNoise', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Optical Distortion<jupyter_code># display Optical Distortion augmentations
transform = A.Compose([
    A.OpticalDistortion(distort_limit=1.0)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('OpticalDistortion', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Grid Distortion<jupyter_code># display Grid Distortion augmentations
transform = A.Compose([
    A.GridDistortion(num_steps=5, distort_limit=1.0)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('GridDistortion', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Elastic Transform<jupyter_code># display Elastic Transform augmentations
transform = A.Compose([
    A.ElasticTransform(alpha=3)
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16, 6))
plt.suptitle('ElasticTransform', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output><empty_output><jupyter_text>Creating Transformations<jupyter_code># defining augmentation pipeline
transform = A.Compose([
    A.Transpose(p=0.5),
    A.VerticalFlip(p=0.5),
    A.HorizontalFlip(p=0.5),
    A.RandomBrightness(limit=0.2, p=0.5),
    A.RandomContrast(limit=0.2, p=0.1),
    A.OneOf([
        A.MotionBlur(blur_limit=5),
        A.MedianBlur(blur_limit=5),
        A.GaussianBlur(blur_limit=5),
        A.GaussNoise(var_limit=(5.0, 30.0)),
    ], p=0.1),
    A.OneOf([
        A.OpticalDistortion(distort_limit=1.0),
        A.GridDistortion(num_steps=5, distort_limit=1.0),
        A.ElasticTransform(alpha=3),
    ], p=0.1),
    A.CLAHE(clip_limit=4.0, p=0.1),
    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.1),
    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.4),
    A.Resize(256, 256),
    A.Cutout(max_h_size=int(256 * 0.1), max_w_size=int(256 * 0.1), num_holes=1, p=0.2),
    A.Normalize()
])

paths = glob('../split/train/mel/*.jpg')
fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(16,6))
plt.suptitle('Albumentation Transformations', fontsize=16)
for i in range(0, 10):   
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = cv2.resize(image, (256,256))
    transformed = transform(image=image)
    aug_image = transformed['image']
   
    x = i // 5
    y = i % 5
    axes[x, y].imshow(aug_image, cmap=plt.cm.bone)
    axes[x, y].axis('off')<jupyter_output>Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping i[...]<jupyter_text>First Train-Validation Split<jupyter_code># paths = glob('../split/train/mel/2020/*.jpg')


# for i in range(0, 467):   
#     f, ax = plt.subplots()
#     image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
#     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
#     transformed = transform(image=image)
#     aug_image = transformed['image']
#     plt.imshow(aug_image)
#     plt.savefig("../split/train/mel/2020_T/aug_image_{}.png".format(i))
#     plt.close('all')
# paths = glob('../split/train/mel/2019/*.jpg')

# for i in tqdm(range(0, 4522)):   
#     f, ax = plt.subplots()
#     image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
#     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
#     transformed = transform(image=image)
#     aug_image = transformed['image']
#     plt.imshow(aug_image)
#     plt.savefig("../split/train/mel/2019_T/aug_image_{}.png".format(i))
#     plt.close('all')
# paths = glob('../split/train/mel/2018/*.jpg')

# for i in range(0, 1113):   
#     f, ax = plt.subplots()
#     image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
#     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
#     transformed = transform(image=image)
#     aug_image = transformed['image']
#     plt.imshow(aug_image)
#     plt.savefig("../split/train/mel/2018_T/aug_image_{}.png".format(i))
#     plt.close('all')<jupyter_output><empty_output><jupyter_text>Second Train-Validation Split<jupyter_code>paths = glob('../split/train/mel/*.jpg')


for i in range(0, 4084):   
    f, ax = plt.subplots()
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    transformed = transform(image=image)
    aug_image = transformed['image']
    plt.imshow(aug_image)
    plt.savefig("../split/train/mel/aug/aug_image_{}.jpg".format(i))
    plt.close('all')
paths = glob('../sample/train/mel/*.jpg')


for i in range(0, 817):   
    f, ax = plt.subplots()
    image = cv2.imread(paths[i], cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    transformed = transform(image=image)
    aug_image = transformed['image']
    plt.imshow(aug_image)
    plt.savefig("../sample/train/mel/aug/aug_image_{}.png".format(i))
    plt.close('all')<jupyter_output>to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the val[...]
{'fraction_non_alphanumeric': 0.1568759380984421, 'fraction_numerical': 0.038921380880906783, 'mean_word_length': 4.033081531648867, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '417525', 'license': 'MIT', 'n_tokens_mistral': 7888, 'n_tokens_neox': 7350, 'n_words': 1533, 'path': 'notebooks/albumentation.ipynb', 'repo_name': 'datascisteven/Melanoma-Image-Classification', 'size': '19321'}