<jupyter_start><jupyter_text>This notebook contains the code needed to process the data which tracks the POIS and diversity, as generated by 'generate_POIS.ipynb'This code is part of the paper "The Importance of Being Restrained" DETAILS + CITATION INFO TO ADD HERE<jupyter_code>#Basic imports and setup
import numpy as np
import pickle as pkl

sdiss = ['saturate', 'mirror', 'COTN', 'unif_resample', 'toroidal']
#Function for processing. Note that the symbols chosen are arbitrary, but they should match the ones in common.py from pyade.
#This is a very ad-hoc solution, and could be done a lot better in future
def process_run(mylist, filename):
    idxs_div = [x.startswith('|') for x in mylist]
    div = [float(x[1:-1]) for x in np.array(mylist)[idxs_div]]
    idxs_fit = [x.startswith('!') for x in mylist]
    fit = [float(x[1:-1]) for x in np.array(mylist)[idxs_fit]] 
    idxs_popsize = [x.startswith('@') for x in mylist]
    popsize = [int(x[1:-1]) for x in np.array(mylist)[idxs_popsize]] 
    idxs_cos = [x.startswith('#') for x in mylist]
    cos = [float(x[1:]) if (float(x[1:])<2) else np.nan for x in np.array(mylist)[idxs_cos]] 
#     cos[[x == 2 for x in cos]] = np.nan
    corr = [int(x) for x in np.array(mylist)[[(not x) and (not y) and (not z) and (not a) for x,y,z,a in zip(idxs_div, idxs_fit, idxs_popsize, idxs_cos)]][:-1]]
    res = {'diversity' : div, 'fitness' : fit, 'corrections' : corr, 'popsize' : popsize, 'cosine' : cos}
    with open(filename, 'wb') as f:
        pkl.dump(res, f)
#Set parameters
de_version = 'shade' # We use both shade and lshade for the paper
dim = 5 # We use dims 5 and 30 for the paper
iid = 1 # we use only iid 1 in the paper to save some computation time
log_folder_location = "logs_dict" #where to put the resulting files
#Read the text file and split it into an array
with open(f'logs_full/log_all_fcts_sdis_5rep_iid{iid}_{de_version}_{dim}d.txt') as f:
    full_list = np.array(f.read().splitlines())
all_arrs = np.array_split(np.array(full_list), np.where(full_list == '/')[0])[1:-1]
idx = 0
for fid in range(1,25):
    for sdis in sdiss:
        for rep in range(5):
            temp = all_arrs[idx][1:]
            process_run(temp, f"{log_folder_location}/{de_version}_F{fid}_{dim}D_I{iid}_{sdis}_R{rep}.pkl")
            idx = idx+1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13605150214592274, 'fraction_numerical': 0.01201716738197425, 'mean_word_length': 4.100656455142232, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '582095', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 881, 'n_tokens_neox': 871, 'n_words': 279, 'path': 'process_POIS.ipynb', 'repo_name': 'Dvermetten/DE_TIOBR', 'size': '2330'}