<jupyter_start><jupyter_text>Aggregate the Data of each Participant againThis script aggregates the transformed data of each participant again, to get one .pkl file for each participant.<jupyter_code>import pandas as pd
import numpy as np
import os
import pickle
transformed_data_path = 'TransformedData'
aggregated_data_output_path = 'AggregatedData'
participants = []
for filename in os.listdir(transformed_data_path):
    if not filename.endswith(".pkl"):
        continue
    participants.append(filename.split("_")[0])

participants = set(participants)
print("Participants:", participants)
file_appendix = "rotated.pkl"

for participant in participants:
    print("Starting with Participant: " + participant)
    df = pd.DataFrame()
    file_index = 0
    filename = participant + "_" + str(file_index) + "_" + file_appendix
    while os.path.isfile(transformed_data_path + "/" + filename):
        df = df.append(pd.read_pickle(transformed_data_path + "/" + filename))
        file_index = file_index + 1
        filename = participant + "_" + str(file_index) + "_" + file_appendix
df.to_pickle(aggregated_data_output_path + "/" + participant + ".pkl")
print("Finished")<jupyter_output>Starting with Participant: 07
Finished
{'fraction_non_alphanumeric': 0.1048780487804878, 'fraction_numerical': 0.0040650406504065045, 'mean_word_length': 4.889952153110048, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '29738', 'license': 'MIT', 'n_tokens_mistral': 386, 'n_tokens_neox': 366, 'n_words': 92, 'path': 'Training/03 - Aggregate Participants.ipynb', 'repo_name': 'interactionlab/hand-marker-labeling', 'size': '1230'}