<jupyter_start><jupyter_text>IMAGE RANDOMIZATION (Training, Validation & Testing) Code to generate the basic train/validation/testing image database for Keras   by <NAME> Last Update: 01/08/2018 ----------------------------  ORIGINAL & CLAHE DATA IMAGE RANDOMIZATION (copies files)<jupyter_code>"""
 BASIC IMAGE DATABASE TRAIN/VALIDATION/TEST RANDOMIZATION CODE
 ---------------------------------
 by <NAME>
 Last Update: 2017/04/23
"""
import glob
import cv2
import os
import shutil
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import tensorflow as tf
tf.Session(config=tf.ConfigProto(log_device_placement=True)) #To ensure activation of GPUs in TF Backend
from keras.preprocessing.image import ImageDataGenerator


#IMAGE RANDOMIZATION AND AUGMENTATION HELPER FUNCTIONS
# Print iterations progress
def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█'):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end = '\r')
    # Print New Line on Complete
    if iteration == total: 
        print()

def data_randomization (inputpath, outputpath):
    # Print Message specifying database
    print('Randomized Spliting of database in:'+ inputpath)
    
    #Creation of required folders
    if not os.path.isdir(outputpath):
        os.mkdir(outputpath)
        
    for dirpath, dirnames, filenames in os.walk(inputpath):
        structure = os.path.join(outputpath + 'train/', dirpath[len(inputpath):])
        if not os.path.isdir(structure):
            os.mkdir(structure)
    
    for dirpath, dirnames, filenames in os.walk(inputpath):
        structure = os.path.join(outputpath + 'validation/', dirpath[len(inputpath):])
        if not os.path.isdir(structure):
            os.mkdir(structure)       
            
    for dirpath, dirnames, filenames in os.walk(inputpath):
        structure = os.path.join(outputpath + 'test/', dirpath[len(inputpath):])
        if not os.path.isdir(structure):
            os.mkdir(structure)
            
    class_num = 0
    all_file_dataframe = pd.DataFrame([])
    all_file_list = []
    all_train_set_df = pd.DataFrame([])
    all_validate_set_df = pd.DataFrame([])
    all_test_set_df = pd.DataFrame([])
    
    for directory in glob.iglob(inputpath + '*', recursive=True):
        class_file_list=[]
        df = pd.DataFrame([])
        for filename in glob.iglob(directory + '/' +'*.png', recursive=True):
            class_file_list.append(filename)
        
        class_num += 1
        all_file_list.append(class_file_list)
        df = pd.DataFrame({directory:class_file_list})
        all_file_dataframe = pd.concat([all_file_dataframe, df], axis=1)
        train, validate, test = np.split(df.sample(frac=1), [int(train_p*len(df)), int((train_p + validation_p)*len(df))])
        all_train_set_df = pd.concat([all_train_set_df, train], axis=1)
        all_validate_set_df = pd.concat([all_validate_set_df, validate], axis=1)
        all_test_set_df = pd.concat([all_test_set_df, test], axis=1)
    
    print('Total Number of classes: '+ str(len(all_file_list)))
    n = 0
    l = sum([len(files) for r, d, files in os.walk(inputpath)])-1  
    for dir_n in all_file_dataframe.columns:
        #CREATE FULLY RANDOMIZED TRAINING SET
        for train_file_n in list(all_train_set_df.loc[:, dir_n]):
            if isinstance(train_file_n,str):
                n += 1
                printProgressBar(n , l, prefix = 'Progress:', suffix = 'Complete', length = 50)
                shutil.copy2(train_file_n, train_file_n.replace(inputpath, outputpath + 'train/')) # Copy files to target filename is /data/test/img.png
        
        #CREATE FULLY RANDOMIZED VALIDATION SET
        for val_file_n in list(all_validate_set_df.loc[:, dir_n]):
            if isinstance(val_file_n,str):
                n += 1
                printProgressBar(n , l, prefix = 'Progress:', suffix = 'Complete', length = 50)
                shutil.copy2(val_file_n, val_file_n.replace(inputpath, outputpath + 'validation/')) # Copy files to target filename is /data/test/img.png
        #CREATE FULLY RANDOMIZED TESTING SET    
        for test_file_n in list(all_test_set_df.loc[:, dir_n]):
            if isinstance(test_file_n,str):
                n += 1
                printProgressBar(n , l, prefix = 'Progress:', suffix = 'Complete', length = 50)
                shutil.copy2(test_file_n, test_file_n.replace(inputpath, outputpath + 'test/')) # Copy files to target filename is /data/test/img.png
        
    print('Total number of randomized files: '+ str(n))
        
        
# SPLITING PERCENTAGES (TRAINING=60% / VALIDATION=20% / TESTING=20%)
train_p = 0.6
validation_p = 0.20
test_p = (1.0 - train_p - validation_p)
print('SPLITING PERCENTAGES ARE: (TRAINING=' + str(train_p*100) +'% / VALIDATION=' + str(validation_p*100) +'% / TESTING=' + str(test_p*100) + '%)')
#BASE DATA RANDOMIZATION
#Definition of folder tree structure for converted files for original database and Randomization
inputpath = './data/single_lesion_database/original_data/'
outputpath ='./data/single_lesion_database/original_data_randomized/'
data_randomization(inputpath, outputpath)     
#BASE DATA RANDOMIZATION
#Definition of folder tree structure for converted files for original database and Randomization
inputpath = './data/single_lesion_database/clahe_data/'
outputpath ='./data/single_lesion_database/clahe_data_randomized/'
data_randomization(inputpath, outputpath)<jupyter_output>Randomized Spliting of database in:./data/single_lesion_database/clahe_data/
Total Number of classes: 6
Progress: |██████████████████████████████████████████████████| 100.0% Complete
Total number of randomized files: 38277██████████████████████| 100.0% Complete
{'fraction_non_alphanumeric': 0.12954719877206447, 'fraction_numerical': 0.012586339217191097, 'mean_word_length': 3.075046904315197, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '370412', 'license': 'MIT', 'n_tokens_mistral': 2156, 'n_tokens_neox': 2020, 'n_words': 516, 'path': '00_C_DL_Image_Database_Randomization.ipynb', 'repo_name': 'myrinx/SPL_UD_DL', 'size': '6531'}