<jupyter_start><jupyter_text>1. Auxiliary functions<jupyter_code>EMOTIONS_PATH = "/Users/samuilstoychev/ckplus/CK+/Emotion"
IMAGES_PATH = "/Users/samuilstoychev/ckplus/CK+/cohn-kanade-images" 
def listdir(path):
    """List all items in a given directory (in ascending order)."""
    res = []
    for f in os.listdir(path):
        if not f.startswith('.'):
            res.append(f)
    return sorted(res)
def get_images(subject, sample): 
    folder = "/".join([IMAGES_PATH, subject, sample])
    images = listdir(folder)
    neutral = folder + "/" + images[0]
    emotional = [folder + "/" + img for img in [images[-3], images[-2], images[-1]]]
    return (neutral, emotional)<jupyter_output><empty_output><jupyter_text>2. Allocating images to emotions<jupyter_code>subjects = listdir(EMOTIONS_PATH) 
data = [[] for _ in range(8)] 

for subj in subjects: 
    samples = listdir(EMOTIONS_PATH + "/" + subj) 
    for samp in samples: 
        files = listdir(EMOTIONS_PATH + "/" + subj + "/" + samp)
        if len(files) is not 0: 
            fsubj, fsample, _, _ = files[0].split("_")
            emotion = open(EMOTIONS_PATH + "/" + subj + "/" + samp + "/" + files[0], "r").read().strip()
            emotion = int(float(emotion)) 
            neutral, emotional = get_images(fsubj, fsample)
            data[0].append(neutral)
            data[emotion] += emotional<jupyter_output><empty_output><jupyter_text>Distribution of emotions<jupyter_code>dist = [len(data[x]) for x in range(8)]
y_pos = np.arange(len(EMOTIONS))

plt.bar(y_pos, dist, align='center', alpha=0.8)
plt.xticks(y_pos, EMOTIONS)

plt.show()
print("Distribution of emotions:", dist)
print("Images in total:", sum(dist))<jupyter_output><empty_output><jupyter_text>3. Splitting into train and test<jupyter_code>def split_train_test(data, test_ratio=0.2): 
    train = [] 
    test = [] 
    
    for x in data: 
        random.shuffle(x)
        split_point = int(test_ratio * len(x))
        train.append(x[split_point:])
        test.append(x[:split_point])
    return (train, test)
train_data, test_data = split_train_test(data)<jupyter_output><empty_output><jupyter_text>4. Copy into new directory<jupyter_code>NEW_FOLDER_TRAIN = "/Users/samuilstoychev/ckplus_preprocessed/train/"
NEW_FOLDER_TEST = "/Users/samuilstoychev/ckplus_preprocessed/test/"
for emotion_index in range(8): 
    train_folder = NEW_FOLDER_TRAIN + "class_" + str(emotion_index) + "/" 
    test_folder = NEW_FOLDER_TEST + "class_" + str(emotion_index) + "/" 
    
    for img_address in train_data[emotion_index]: 
        filename = img_address.split("/")[-1]
        copyfile(img_address, train_folder + filename)
    for img_address in test_data[emotion_index]: 
        filename = img_address.split("/")[-1]
        copyfile(img_address, test_folder + filename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1426056338028169, 'fraction_numerical': 0.007394366197183098, 'mean_word_length': 3.735, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '116982', 'license': 'MIT', 'n_tokens_mistral': 1039, 'n_tokens_neox': 995, 'n_words': 186, 'path': 'CKPlusPreprocessing.ipynb', 'repo_name': 'samuilstoychev/research_project', 'size': '2840'}