<jupyter_start><jupyter_text>Combining CINIC-10 train and validation setsThis notebook serves as a tool to combine CINIC-10's train and validation sets into a new train set (larger). This configuration better matches CIFAR-10. ENSURE THAT CINIC-10 IS DOWNLOADED AND STORED IN ../data/cinic-10<jupyter_code>cinic_directory = "../data/cinic-10"
enlarge_directory = "../data/cinic-10-trainlarge"
classes = ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]
sets = ['train', 'valid', 'test']
if not os.path.exists(enlarge_directory):
    os.makedirs(enlarge_directory)
if not os.path.exists(enlarge_directory + '/train'):
    os.makedirs(enlarge_directory + '/train')
if not os.path.exists(enlarge_directory + '/test'):
    os.makedirs(enlarge_directory + '/test')
    
for c in classes:
    if not os.path.exists('{}/train/{}'.format(enlarge_directory, c)):
        os.makedirs('{}/train/{}'.format(enlarge_directory, c))
    if not os.path.exists('{}/test/{}'.format(enlarge_directory, c)):
        os.makedirs('{}/test/{}'.format(enlarge_directory, c))
for s in sets:
    for c in classes:
        source_directory = '{}/{}/{}'.format(cinic_directory, s, c)
        filenames = glob.glob('{}/*.png'.format(source_directory))
        for fn in filenames:
            dest_fn = fn.split('/')[-1]
            if s == 'train' or s == 'valid':
                dest_fn = '{}/train/{}/{}'.format(enlarge_directory, c, dest_fn)
                if symlink:
                    if not os.path.islink(dest_fn):
                        os.symlink(fn, dest_fn)
                else:
                    copyfile(fn, dest_fn)
                
                
            elif s == 'test':
                dest_fn = '{}/test/{}/{}'.format(enlarge_directory, c, dest_fn)
                if symlink:
                    if not os.path.islink(dest_fn):
                        os.symlink(fn, dest_fn)
                else:
                    copyfile(fn, dest_fn)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1527777777777778, 'fraction_numerical': 0.00744047619047619, 'mean_word_length': 2.356073211314476, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '281674', 'license': 'MIT', 'n_tokens_mistral': 666, 'n_tokens_neox': 642, 'n_words': 145, 'path': 'notebooks/enlarge-train-set.ipynb', 'repo_name': 'david-klindt/cinic-10', 'size': '2016'}