<jupyter_start><jupyter_text>Notebook DescriptionThis notebook contains the code for creating the directories to hold the train/test/validation data sorted by class**Date of Last Update**: November 23rd 2020 --- **INPUTS**: Data/train folder folder containing image files sorted by class**OUTPUTS**: Data/train, Data/val, Data/validationdirectory containing image files sorted by class and train test val ---  Previous Steps- download images and perform manual inspection Next Steps- selection of off the shelf model ---  Import Packages<jupyter_code>import shutil
import os
import numpy as np
import argparse<jupyter_output><empty_output><jupyter_text>--- Define Helper Functions<jupyter_code>def get_files_from_folder(data_path):
    """takes in path to files 
    returns a numpy array of files"""
    files = os.listdir(data_path)
    arrfiles = np.asarray(files)
    return arrfiles
def split(path_to_data, path_to_test_data, ratio=0.8):
    # get directories
    _, dirs, _ = next(os.walk(path_to_data))

    # calculates how many train data per class
    data_counter_per_class = np.zeros((len(dirs)))
    for i in range(len(dirs)):
        path = os.path.join(path_to_data, dirs[i])
        files = get_files_from_folder(path)
        data_counter_per_class[i] = len(files)
    test_counter = np.round(data_counter_per_class * (1 - ratio))

    # transfers files
    for i in range(len(dirs)):
        path_to_original = os.path.join(path_to_data, dirs[i])
        path_to_save = os.path.join(path_to_test_data, dirs[i])

        #creates directory
        if not os.path.exists(path_to_save):
            os.makedirs(path_to_save)
        
        #obtains array of files and shuffles the order
        files = get_files_from_folder(path_to_original)
        np.random.set_state = 42
        np.random.shuffle(files)
        
        # moves data
        for j in range(int(test_counter[i])):
            dst = os.path.join(path_to_save, files[j])
            src = os.path.join(path_to_original, files[j])
            shutil.move(src, dst)
    
    return print("file split complete")<jupyter_output><empty_output><jupyter_text>Define File Paths and Split Sizes<jupyter_code>path_to_data = 'Data/train'
path_to_test_data = 'Data/val'
path_to_val_data ='Data/validation' 
val_ratio=0.95 
train_ratio=0.8<jupyter_output><empty_output><jupyter_text>Perform Splits<jupyter_code># perform split for train/validation
# use ratio = 0.95
split(path_to_data, path_to_val_data, val_ratio)
# perform split for test/train
#use ratio = 0.8
split(path_to_data, path_to_test_data, train_ratio)<jupyter_output>file split complete
{'fraction_non_alphanumeric': 0.0945482272207396, 'fraction_numerical': 0.008006099885627144, 'mean_word_length': 3.904672897196262, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '127944', 'license': 'MIT', 'n_tokens_mistral': 880, 'n_tokens_neox': 850, 'n_words': 211, 'path': 'data_collection/02_data_splitting.ipynb', 'repo_name': 'maggiefi/CNNs-and-Transfer-Learning', 'size': '2623'}