<jupyter_start><jupyter_text>Data preprocessing<jupyter_code>import os
import shutil
from pathlib import Path
import imgaug as ia
import imgaug.augmenters as iaa
import imageio
from imgaug.augmenters.size import Resize<jupyter_output><empty_output><jupyter_text>initial directory structure - the folders 'images' and 'labes' contain the images and associated annotations used for training, validation and testing respectively- at the beginning, these correspond to the unaugmented training, validation or test data sets- if an augmented dataset is to be used for training/validation/tesing, the folder names must be adapted to 'images' and 'labels'<jupyter_code>containing files: 

|-- combined_dataset
|  |-- images
|  |-- labels
|  |-- txt_of_ECU+SFA+SCH+HGR

empty: 

|-- dataset
|  |-- train
|  |  |-- images
|  |  |-- labels
|  |  |-- images_orig
|  |  |-- labels_orig
|  |  |-- images_rot
|  |  |-- labels_rot
|  |  |-- images_rot_gauss
|  |  |-- labels_rot_gauss
|  |-- valid
|  |  |-- images
|  |  |-- labels
|  |  |-- images_orig
|  |  |-- labels_orig
|  |  |-- augmented 
|  |  |  |-- gauss
|  |  |  |  |-- images_gauss
|  |  |  |  |-- labels_gauss
|  |  |  |-- bright
|  |  |  |  |-- images_bright
|  |  |  |  |-- labels_bright
|  |  |  |-- dark
|  |  |  |  |-- images_dark
|  |  |  |  |-- labels_dark
|  |  |  |-- gauss_bright
|  |  |  |  |-- images_gauss_bright
|  |  |  |  |-- labels_gauss_bright
|  |  |  |-- gauss_dark
|  |  |  |  |-- images_gauss_dark
|  |  |  |  |-- labels_gauss_dark
|  |-- test
|  |  |-- images
|  |  |-- labels
|  |  |-- images_orig
|  |  |-- labels_orig
|  |  |-- augmented 
|  |  |  |-- gauss
|  |  |  |  |-- images_gauss
|  |  |  |  |-- labels_gauss
|  |  |  |-- bright
|  |  |  |  |-- images_bright
|  |  |  |  |-- labels_bright
|  |  |  |-- dark
|  |  |  |  |-- images_dark
|  |  |  |  |-- labels_dark
|  |  |  |-- gauss_bright
|  |  |  |  |-- images_gauss_bright
|  |  |  |  |-- labels_gauss_bright
|  |  |  |-- gauss_dark
|  |  |  |  |-- images_gauss_dark
|  |  |  |  |-- labels_gauss_dark
|  |-- class_dict.csv
|  |-- evaluated_classes<jupyter_output><empty_output><jupyter_text>data distribution<jupyter_code># Path of the combined dataset
SOURCE_PATH_IMG = Path("/Users/paulawi/Downloads/dataset/combined_dataset/images")
SOURCE_PATH_LBL = Path("/Users/paulawi/Downloads/dataset/combined_dataset/labels")

# Training dataset (70%)

FILENAMES_FILE_PATH = Path("/Users/paulawi/Downloads/combined_dataset/txt_of_ECU+SFA+SCH+HGR/train.txt")

DESTINATION_PATH_TRIMG = Path("/Users/paulawi/Downloads/dataset/train/images_orig")
DESTINATION_PATH_TRLBL = Path("/Users/paulawi/Downloads/dataset/train/labels_orig")

with FILENAMES_FILE_PATH.open(encoding="utf-8") as lines:
    for line in lines:
        image_file_path = SOURCE_PATH_IMG / (line.strip() + ".jpg")
        if image_file_path.is_file():
                shutil.copy(image_file_path, DESTINATION_PATH_TRIMG)

with FILENAMES_FILE_PATH.open(encoding="utf-8") as lines:
    for line in lines:
        image_file_path = SOURCE_PATH_LBL / (line.strip() + ".png")
        if image_file_path.is_file():
                shutil.copy(image_file_path, DESTINATION_PATH_TRLBL)

# Validation dataset (15%)

FILENAMES_FILE_PATH = Path("/Users/paulawi/Downloads/combined_dataset/txt_of_ECU+SFA+SCH+HGR/val.txt")

DESTINATION_PATH_VIMG = Path("/Users/paulawi/Downloads/dataset/valid/images_orig")
DESTINATION_PATH_VLBL = Path("/Users/paulawi/Downloads/dataset/valid/labels_orig")

with FILENAMES_FILE_PATH.open(encoding="utf-8") as lines:
    for line in lines:
        image_file_path = SOURCE_PATH_IMG / (line.strip() + ".jpg")
        if image_file_path.is_file():
                shutil.copy(image_file_path, DESTINATION_PATH_VIMG)

with FILENAMES_FILE_PATH.open(encoding="utf-8") as lines:
    for line in lines:
        image_file_path = SOURCE_PATH_LBL / (line.strip() + ".png")
        if image_file_path.is_file():
                shutil.copy(image_file_path, DESTINATION_PATH_VLBL)

# Test dataset (15%)

FILENAMES_FILE_PATH = Path("/Users/paulawi/Downloads/combined_dataset/txt_of_ECU+SFA+SCH+HGR/test.txt")

DESTINATION_PATH_TIMG = Path("/Users/paulawi/Downloads/dataset/test/images_orig")
DESTINATION_PATH_TLBL = Path("/Users/paulawi/Downloads/dataset/test/labels_orig")

with FILENAMES_FILE_PATH.open(encoding="utf-8") as lines:
    for line in lines:
        image_file_path = SOURCE_PATH_IMG / (line.strip() + ".jpg")
        if image_file_path.is_file():
                shutil.copy(image_file_path, DESTINATION_PATH_TIMG)

with FILENAMES_FILE_PATH.open(encoding="utf-8") as lines:
    for line in lines:
        image_file_path = SOURCE_PATH_LBL / (line.strip() + ".png")
        if image_file_path.is_file():
                shutil.copy(image_file_path, DESTINATION_PATH_TLBL)<jupyter_output><empty_output><jupyter_text>data augmentation<jupyter_code>ia.seed(0)
# resize and rotation 90°
seq1 = iaa.Sequential([
    iaa.Resize(512),
    iaa.Rotate((90,90), fit_output=True)
], random_order=False)

# resize and rotation 180°
seq2 = iaa.Sequential([
    iaa.Resize(512),
    iaa.Rotate((180,180), fit_output=True)
], random_order=False)

# resize and rotation 270°
seq3 = iaa.Sequential([
    iaa.Resize(512),
    iaa.Rotate((270,270), fit_output=True)
], random_order=False)

# additive gaussian noise
seq4 = iaa.Sequential([
    iaa.AdditiveGaussianNoise(loc=0, scale=(0.04*255, 0.04*255), per_channel=1)
], random_order=False)

seq5 = iaa.Sequential([
    iaa.Resize(512),
    iaa.AdditiveGaussianNoise(loc=0, scale=(0.04*255, 0.04*255), per_channel=1)
], random_order=False)

# darkening
seq6 = iaa.Sequential([
    iaa.Resize(512),
    iaa.MultiplyBrightness(mul=(0.9,0.9))
], random_order=False)

# brightening
seq7 = iaa.Sequential([
    iaa.Resize(512),
    iaa.MultiplyBrightness(mul=(1.1,1.1))
], random_order=False)

# only resize
seq8 = iaa.Sequential([
    iaa.Resize(512)
], random_order=False)   

seq9 = iaa.Sequential([], random_order=False)<jupyter_output><empty_output><jupyter_text>Training dataset<jupyter_code># Source
PATH_IMG = '/Users/paulawi/Downloads/dataset/train/images_orig'
PATH_LBL = '/Users/paulawi/Downloads/dataset/train/labels_orig'

# Destinations
PATH_T_IMG = '/Users/paulawi/Downloads/dataset/train/images'
PATH_T_LBL = '/Users/paulawi/Downloads/dataset/train/labels'

PATH_T_ROT_IMG = '/Users/paulawi/Downloads/dataset/train/images_rot'
PATH_T_ROT_LBL = '/Users/paulawi/Downloads/dataset/train/labels_rot'

PATH_T_ROT_GAUSS_IMG = '/Users/paulawi/Downloads/dataset/train/images_rot_gauss'
PATH_T_ROT_GAUSS_LBL = '/Users/paulawi/Downloads/dataset/train/labels_rot_gauss'

# sort data

images_orig = os.listdir(PATH_IMG)
images_orig.sort()

labels_orig = os.listdir(PATH_LBL)
labels_orig.sort()

#  in case remove .DS_Store 

#os.remove(PATH_IMG + '/.DS_Store')
#os.remove(PATH_LBL + '/.DS_Store')
#os.remove(PATH_T_ROT_GAUSS_IMG + '/.DS_Store')
#os.remove(PATH_T_ROT_GAUSS_LBL + '/.DS_Store')

# Rotation 90°
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_IMG + '/' + image)
    img_aug = seq1.augment_image(img)
    imageio.imwrite(os.path.join(PATH_IMG, PATH_T_ROT_IMG + '/' + 'rot90_%04d.jpg' % (i,)), img_aug)       
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_LBL + '/' + label)
    lbl_aug = seq1.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_LBL, PATH_T_ROT_LBL + '/' + 'rot90_%04d.png' % (j,)), lbl_aug)
    j += 1

# Rotation 180°
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_IMG + '/' + image)
    img_aug = seq2.augment_image(img)
    imageio.imwrite(os.path.join(PATH_IMG, PATH_T_ROT_IMG + '/' + 'rot180_%04d.jpg' % (i,)), img_aug)    
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_LBL + '/' + label)
    lbl_aug = seq2.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_LBL, PATH_T_ROT_LBL + '/' + 'rot180_%04d.png' % (j,)), lbl_aug)
    j += 1

# Rotation 270°
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_IMG + '/' + image)
    img_aug = seq3.augment_image(img)
    imageio.imwrite(os.path.join(PATH_IMG, PATH_T_ROT_IMG + '/' + 'rot270_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_LBL + '/' + label)
    lbl_aug = seq3.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_LBL, PATH_T_ROT_LBL + '/' + 'rot270_%04d.png' % (j,)), lbl_aug)
    j += 1

# Resize original images (512x512) and merging with rotated images
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_IMG + '/' + image)
    img_aug = seq8.augment_image(img)
    imageio.imwrite(os.path.join(PATH_IMG, PATH_T_IMG + '/' + 'resized_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_LBL, PATH_T_LBL + '/' + 'resized_%04d.png' % (j,)), lbl_aug)
    j += 1

i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_IMG + '/' + image)
    img_aug = seq8.augment_image(img)
    imageio.imwrite(os.path.join(PATH_IMG, PATH_T_ROT_IMG + '/' + 'resized_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_LBL, PATH_T_ROT_LBL + '/' + 'resized_%04d.png' % (j,)), lbl_aug)
    j += 1

# sort data 

images_rot = os.listdir(PATH_T_ROT_IMG)
images_rot.sort()

labels_rot = os.listdir(PATH_T_ROT_LBL)
labels_rot.sort()

# additive gaussian noise

#os.remove(PATH_T_ROT_IMG + '/.DS_Store')
#os.remove(PATH_T_ROT_LBL + '/.DS_Store')

i = 0
j = 0
for image in images_rot:
    img = imageio.imread(PATH_T_ROT_IMG + '/' + image)
    img_aug = seq4.augment_image(img)
    imageio.imwrite(os.path.join(PATH_T_ROT_IMG, PATH_T_ROT_GAUSS_IMG + '/' + 'gauss_%04d.jpg' % (i,)), img_aug)  
    i += 1

for label in labels_rot:
    lbl = imageio.imread(PATH_T_ROT_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_T_ROT_LBL, PATH_T_ROT_GAUSS_LBL + '/' + 'gauss_%04d.png' % (j,)), lbl_aug)
    j += 1

# Merging T_rot_gauss
rot_images = images_rot
for r_img in rot_images:
    shutil.copy(f"{PATH_T_ROT_IMG}/{r_img}", PATH_T_ROT_GAUSS_IMG)

rot_labels = labels_rot
for r_lbl in rot_labels:
    shutil.copy(f"{PATH_T_ROT_LBL}/{r_lbl}", PATH_T_ROT_GAUSS_LBL)<jupyter_output><empty_output><jupyter_text>Validation dataset<jupyter_code># Source
PATH_VAL_IMG = '/Users/paulawi/Downloads/dataset/valid/images_orig'
PATH_VAL_LBL = '/Users/paulawi/Downloads/dataset/valid/labels_orig'

# Destination
PATH_VAL_IMG_512 = '/Users/paulawi/Downloads/dataset/valid/images'
PATH_VAL_LBL_512 = '/Users/paulawi/Downloads/dataset/valid/labels'

PATH_VAL_IMG_GAUSS ='/Users/paulawi/Downloads/dataset/valid/augmented/gauss/images_gauss'
PATH_VAL_LBL_GAUSS = '/Users/paulawi/Downloads/dataset/valid/augmented/gauss/labels_gauss'

PATH_VAL_IMG_DUNKEL = '/Users/paulawi/Downloads/dataset/valid/augmented/dark/images_dark'
PATH_VAL_LBL_DUNKEL = '/Users/paulawi/Downloads/dataset/valid/augmented/dark/labels_dark'

PATH_VAL_IMG_HELL = '/Users/paulawi/Downloads/dataset/valid/augmented/bright/images_bright'
PATH_VAL_LBL_HELL = '/Users/paulawi/Downloads/dataset/valid/augmented/bright/labels_bright'

PATH_VAL_IMG_GH = '/Users/paulawi/Downloads/dataset/valid/augmented/gauss_bright/images_gauss_bright'
PATH_VAL_LBL_GH = '/Users/paulawi/Downloads/dataset/valid/augmented/gauss_bright/labels_gauss_bright'

PATH_VAL_IMG_GD = '/Users/paulawi/Downloads/dataset/valid/augmented/gauss_dark/images_gauss_dark'
PATH_VAL_LBL_GD = '/Users/paulawi/Downloads/dataset/valid/augmented/gauss_dark/labels_gauss_dark'

# sort data 
images_orig = os.listdir(PATH_VAL_IMG)
images_orig.sort()

labels_orig = os.listdir(PATH_VAL_LBL)
labels_orig.sort()

#   in case remove .DS_Store 

#os.remove(PATH_VAL_IMG + '/.DS_Store')
#os.remove(PATH_VAL_LBL + '/.DS_Store')

# Resize original images
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_VAL_IMG + '/' + image)
    img_aug = seq8.augment_image(img)
    imageio.imwrite(os.path.join(PATH_VAL_IMG, PATH_VAL_IMG_512 + '/' + 'resized_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_VAL_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_VAL_LBL, PATH_VAL_LBL_512 + '/' + 'resized_%04d.png' % (j,)), lbl_aug)
    j += 1

# additive gaussian noise
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_VAL_IMG + '/' + image)
    img_aug = seq5.augment_image(img)
    imageio.imwrite(os.path.join(PATH_VAL_IMG, PATH_VAL_IMG_GAUSS + '/' + 'gauss_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_VAL_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_VAL_LBL, PATH_VAL_LBL_GAUSS + '/' + 'gauss_%04d.png' % (j,)), lbl_aug)
    j += 1

# darkening 
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_VAL_IMG + '/' + image)
    img_aug = seq6.augment_image(img)
    imageio.imwrite(os.path.join(PATH_VAL_IMG, PATH_VAL_IMG_DUNKEL + '/' + 'dunkel_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_VAL_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_VAL_LBL, PATH_VAL_LBL_DUNKEL + '/' + 'dunkel_%04d.png' % (j,)), lbl_aug)
    j += 1

# brightening
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_VAL_IMG + '/' + image)
    img_aug = seq7.augment_image(img)
    imageio.imwrite(os.path.join(PATH_VAL_IMG, PATH_VAL_IMG_HELL + '/' + 'hell_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
        lbl = imageio.imread(PATH_VAL_LBL + '/' + label)
        lbl_aug = seq8.augment_image(lbl)
        imageio.imwrite(os.path.join(PATH_VAL_LBL, PATH_VAL_LBL_HELL + '/' + 'hell_%04d.png' % (j,)), lbl_aug)
        j += 1

# additive gaussian noise with darkening and brightening 

# sort data 

images_hell = os.listdir(PATH_VAL_IMG_HELL)
images_hell.sort() 

labels_hell = os.listdir(PATH_VAL_LBL_HELL)
labels_hell.sort() 

images_dunkel = os.listdir(PATH_VAL_IMG_DUNKEL)
images_dunkel.sort() 

labels_dunkel = os.listdir(PATH_VAL_LBL_DUNKEL)
labels_dunkel.sort() 

i = 0
j = 0
for image in images_hell:
    img = imageio.imread(PATH_VAL_IMG_HELL + '/' + image)
    img_aug = seq5.augment_image(img)
    imageio.imwrite(os.path.join(PATH_VAL_IMG_HELL, PATH_VAL_IMG_GH + '/' + 'gauss_hell_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_hell:
    lbl = imageio.imread(PATH_VAL_LBL_HELL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_VAL_LBL_HELL, PATH_VAL_LBL_GH + '/' + 'gauss_hell_%04d.png' % (j,)), lbl_aug)
    j += 1

i = 0
j = 0
for image in images_dunkel:
    img = imageio.imread(PATH_VAL_IMG_DUNKEL + '/' + image)
    img_aug = seq5.augment_image(img)
    imageio.imwrite(os.path.join(PATH_VAL_IMG_DUNKEL, PATH_VAL_IMG_GD + '/' + 'gauss_dunkel_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_dunkel:
    lbl = imageio.imread(PATH_VAL_LBL_DUNKEL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_VAL_LBL_DUNKEL, PATH_VAL_LBL_GD + '/' + 'gauss_dunkel_%04d.png' % (j,)), lbl_aug)
    j += 1<jupyter_output><empty_output><jupyter_text>Test dataset<jupyter_code># Source
PATH_TE_IMG = '/Users/paulawi/Downloads/dataset/test/images_orig'
PATH_TE_LBL = '/Users/paulawi/Downloads/dataset/test/labels_orig'

# Destination
PATH_TE_IMG_512 = '/Users/paulawi/Downloads/dataset/test/images'
PATH_TE_LBL_512 = '/Users/paulawi/Downloads/dataset/test/labels'

PATH_TE_IMG_GAUSS ='/Users/paulawi/Downloads/dataset/test/augmented/gauss/images_gauss'
PATH_TE_LBL_GAUSS = '/Users/paulawi/Downloads/dataset/test/augmented/gauss/labels_gauss'

PATH_TE_IMG_DUNKEL = '/Users/paulawi/Downloads/dataset/test/augmented/dark/images_dark'
PATH_TE_LBL_DUNKEL = '/Users/paulawi/Downloads/dataset/test/augmented/dark/labels_dark'

PATH_TE_IMG_HELL = '/Users/paulawi/Downloads/dataset/test/augmented/bright/images_bright'
PATH_TE_LBL_HELL = '/Users/paulawi/Downloads/dataset/test/augmented/bright/labels_bright'

PATH_TE_IMG_GH = '/Users/paulawi/Downloads/dataset/test/augmented/gauss_bright/images_gauss_bright'
PATH_TE_LBL_GH = '/Users/paulawi/Downloads/dataset/test/augmented/gauss_bright/labels_gauss_bright'

PATH_TE_IMG_GD = '/Users/paulawi/Downloads/dataset/test/augmented/gauss_dark/images_gauss_dark'
PATH_TE_LBL_GD = '/Users/paulawi/Downloads/dataset/test/augmented/gauss_dark/labels_gauss_dark'

# sort data 
images_orig = os.listdir(PATH_TE_IMG)
images_orig.sort()

labels_orig = os.listdir(PATH_TE_LBL)
labels_orig.sort()

#   in case remove .DS_Store 

#os.remove(PATH_TE_IMG + '/.DS_Store')
#os.remove(PATH_TE_LBL + '/.DS_Store')

# Resize original images
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_TE_IMG + '/' + image)
    img_aug = seq8.augment_image(img)
    imageio.imwrite(os.path.join(PATH_TE_IMG, PATH_TE_IMG_512 + '/' + 'resized_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_TE_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_TE_LBL, PATH_TE_LBL_512 + '/' + 'resized_%04d.png' % (j,)), lbl_aug)
    j += 1

# additive gaussian noise
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_TE_IMG + '/' + image)
    img_aug = seq5.augment_image(img)
    imageio.imwrite(os.path.join(PATH_TE_IMG, PATH_TE_IMG_GAUSS + '/' + 'gauss_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_TE_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_TE_LBL, PATH_TE_LBL_GAUSS + '/' + 'gauss_%04d.png' % (j,)), lbl_aug)
    j += 1

# darkening 
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_TE_IMG + '/' + image)
    img_aug = seq6.augment_image(img)
    imageio.imwrite(os.path.join(PATH_TE_IMG, PATH_TE_IMG_DUNKEL + '/' + 'dunkel_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(PATH_TE_LBL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_TE_LBL, PATH_TE_LBL_DUNKEL + '/' + 'dunkel_%04d.png' % (j,)), lbl_aug)
    j += 1

# brightening
i = 0
j = 0
for image in images_orig:
    img = imageio.imread(PATH_TE_IMG + '/' + image)
    img_aug = seq7.augment_image(img)
    imageio.imwrite(os.path.join(PATH_TE_IMG, PATH_TE_IMG_HELL + '/' + 'hell_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
        lbl = imageio.imread(PATH_TE_LBL + '/' + label)
        lbl_aug = seq8.augment_image(lbl)
        imageio.imwrite(os.path.join(PATH_TE_LBL, PATH_TE_LBL_HELL + '/' + 'hell_%04d.png' % (j,)), lbl_aug)
        j += 1

# additive gaussian noise with darkening and brightening 

# sort data 

images_hell = os.listdir(PATH_TE_IMG_HELL)
images_hell.sort() 

labels_hell = os.listdir(PATH_TE_LBL_HELL)
labels_hell.sort() 

images_dunkel = os.listdir(PATH_TE_IMG_DUNKEL)
images_dunkel.sort() 

labels_dunkel = os.listdir(PATH_TE_LBL_DUNKEL)
labels_dunkel.sort() 

i = 0
j = 0
for image in images_hell:
    img = imageio.imread(PATH_TE_IMG_HELL + '/' + image)
    img_aug = seq5.augment_image(img)
    imageio.imwrite(os.path.join(PATH_TE_IMG_HELL, PATH_TE_IMG_GH + '/' + 'gauss_hell_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_hell:
    lbl = imageio.imread(PATH_TE_LBL_HELL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_TE_LBL_HELL, PATH_TE_LBL_GH + '/' + 'gauss_hell_%04d.png' % (j,)), lbl_aug)
    j += 1

i = 0
j = 0
for image in images_dunkel:
    img = imageio.imread(PATH_TE_IMG_DUNKEL + '/' + image)
    img_aug = seq5.augment_image(img)
    imageio.imwrite(os.path.join(PATH_TE_IMG_DUNKEL, PATH_TE_IMG_GD + '/' + 'gauss_dunkel_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_dunkel:
    lbl = imageio.imread(PATH_TE_LBL_DUNKEL + '/' + label)
    lbl_aug = seq8.augment_image(lbl)
    imageio.imwrite(os.path.join(PATH_TE_LBL_DUNKEL, PATH_TE_LBL_GD + '/' + 'gauss_dunkel_%04d.png' % (j,)), lbl_aug)
    j += 1
SOURCE_IMG = '/Users/paulawi/Desktop/PreProTest/images_orig'
SOURCE_LBL = '/Users/paulawi/Desktop/PreProTest/labels_orig'

DEST_IMG = '/Users/paulawi/Desktop/PreProTest/images'
DEST_LBL = '/Users/paulawi/Desktop/PreProTest/labels'

os.remove(SOURCE_IMG + '/.DS_Store')
os.remove(SOURCE_LBL + '/.DS_Store')

ia.seed(0)

seq = iaa.Sequential([iaa.Resize(512)], random_order=False)

images_orig = os.listdir(SOURCE_IMG)
images_orig.sort()

labels_orig = os.listdir(SOURCE_LBL)
labels_orig.sort()

i = 0
j = 0
for image in images_orig:
    img = imageio.imread(SOURCE_IMG + '/' + image)
    img_aug = seq.augment_image(img)
    imageio.imwrite(os.path.join(SOURCE_IMG, DEST_IMG + '/' + 'gauss_%04d.jpg' % (i,)), img_aug)
    i += 1

for label in labels_orig:
    lbl = imageio.imread(SOURCE_LBL + '/' + label)
    lbl_aug = seq.augment_image(lbl)
    imageio.imwrite(os.path.join(SOURCE_LBL, DEST_LBL + '/' + 'gauss_%04d.png' % (j,)), lbl_aug)
    j += 1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1456776195176985, 'fraction_numerical': 0.01640577257556527, 'mean_word_length': 4.021005428369129, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 47, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '519835', 'license': 'Apache-2.0', 'n_tokens_mistral': 9292, 'n_tokens_neox': 9049, 'n_words': 1285, 'path': 'data_preprocessing.ipynb', 'repo_name': 'PAULAWLM/Amazing-Semantic-Segmentation', 'size': '21273'}