<jupyter_start><jupyter_text>This notebook contains the code needed to process the data which tracks the POIS and diversity, as generated by the SOS frameworkBecause of the large size of these tables, not all in-between artifacts are providedThis code is part of the paper "The Importance of Being Restrained"<jupyter_code>import numpy as np
import pickle
import pandas as pd
from functools import partial
import glob 

import seaborn as sbs
import matplotlib.pyplot as plt

from scipy.stats import kendalltau, rankdata

font = {'size'   : 20}

plt.rc('font', **font)
base_folder = "/mnt/e/Research/DE/" #Update to required folder
output_location = "Datatables/"
def get_merged_dt(cross, sdis, F, CR, popsize):
    dt_large = pd.DataFrame()
    files = glob.glob(f"{base_folder}Runs_only/DEro{cross}{sdis}p{popsize}D30*F{F}Cr{CR}.txt")
    for f in files:
        dt_temp = pd.read_csv(f, sep=' ', header=None, skiprows=1)
        dt_large = dt_large.append(dt_temp)
    dt_large['cross'] = cross
    dt_large['sdis'] = sdis
    dt_large['F'] = F
    dt_large['CR'] = CR
    dt_large['popsize'] = popsize
    return dt_large
def get_full_dt():
    dt_full = pd.DataFrame()
    for cross in ['b','e']:
        for sdis in ['c', 'h', 'm', 's', 't', 'u']:
            for CR in ['005', '099']:
                for F in ['0916', '005']:
                    for popsize in [5,20,100]:
                        dt_temp = get_merged_dt(cross, sdis, F, CR, popsize)
                        dt_full = dt_full.append(dt_temp)
    return dt_full
def get_merged_dt_v2(cross, sdis, F, CR, popsize):
    dt_large = pd.DataFrame()
    files = glob.glob(f"{base_folder}CosineSimilarity-MoreData/CosineSimilarity-MoreData/7/DEro{cross}{sdis}p{popsize}D30*F{F}Cr{CR}.txt")
    if len(files) == 0:
        return dt_large
    for f in files:
        dt_temp = pd.read_csv(f, sep=' ', header=None, skiprows=1)
        dt_large = dt_large.append(dt_temp)
    dt_large['cross'] = cross
    dt_large['sdis'] = sdis
    dt_large['F'] = F
    dt_large['CR'] = CR
    dt_large['popsize'] = popsize
    dt_large.columns = ['cosine', 'applied', 'accept', 'cross', 'sdis', 'F', 'CR', 'popsize']
    return dt_large
for cross in ['b','e']:
    for sdis in ['c', 'h', 'm', 's', 't', 'u']:
        for CR in ['005','0285','052','0755','099']:
            for F in ['005','0285','052','0755','099']:
                for popsize in [5,20, 100]:
                    dt = get_merged_dt_v2(cross, sdis, F, CR, popsize)
                    dt.to_csv(f"{output_location}DEro{cross}_{sdis}_p{popsize}_F{F}CR{CR}_cosine.csv")
def get_merged_dt_v3(sdis, F, CR, popsize):
    dt_large = pd.DataFrame()
    files = glob.glob(f"{base_folder}CosineSimilarity-LookingCloser/7/DErob{sdis}p{popsize}D30*F{F}Cr{CR}.txt")
    if len(files) == 0:
        return dt_large
    for f in files:
        dt_temp = pd.read_csv(f, sep=' ', header=None, skiprows=1)
        dt_large = dt_large.append(dt_temp)
    dt_large['sdis'] = sdis
    dt_large['F'] = F
    dt_large['CR'] = CR
    dt_large['popsize'] = popsize
    dt_large.columns = ['cosine', 'nr_mut', 'nr_exceed', 'accept', 'sdis', 'F', 'CR', 'popsize']
    return dt_large
for sdis in ['c', 'h', 'm', 's', 't', 'u']:
    for popsize in [5, 20, 100]:
        for idx_0, F in enumerate(['099','0755','052','0285','005']):
            for idx_1, CR in enumerate(['0041','0081','0121','0161','0201']):
                dt = get_merged_dt_v3(sdis, F, CR, popsize)
                dt.to_csv(f"{output_location}DE_{sdis}_p{popsize}_F{F}CR{CR}_cosine_v3.csv")
def get_merged_dt_v4(sdis, F, CR, popsize):
    dt_large = pd.DataFrame()
    files = glob.glob(f"{base_folder}Div_cos_sim/CosineSimilarity/7/DErob{sdis}p{popsize}D30f0*_F{F}Cr{CR}.txt")
    if len(files) == 0:
        return dt_large
    for f in files:
        dt_temp = pd.read_csv(f, sep=' ', header=None, skiprows=1)
        dt_large = dt_large.append(dt_temp)
    dt_large['sdis'] = sdis
    dt_large['F'] = F
    dt_large['CR'] = CR
    dt_large['popsize'] = popsize
    dt_large.columns = ['cosine', 'nr_mut', 'nr_exceed', 'accept', 'sdis', 'F', 'CR', 'popsize']
    return dt_large
for F in ['0285', '099', '052', '005']: #'0755', 
    for CR in ['0755', '0285', '099', '052', '005', '00891', '01283', '01675', '02067', '02458']:
        for popsize in [5, 20, 100]:
            for sdis in  ['t', 'h', 'm', 's', 'c', 'u']:
                dt = get_merged_dt_v4(sdis, F, CR, popsize)
                dt.to_csv(f"{output_location}DE_{sdis}_p{popsize}_F{F}CR{CR}_cosine_v4.csv")
def get_diversity_dt(sdis, F, CR, popsize):
    dt_large = pd.DataFrame()
    files = glob.glob(f"/mnt/e/Research/DE/Div_cos_sim/CosineSimilarity/7/Diversity-DErob{sdis}p{popsize}D30f0*_F{F}Cr{CR}.txt")
    if len(files) == 0:
        return dt_large
    for f in files:
        dt_temp = pd.read_csv(f, sep=' ', header=None, skiprows=1)
        dt_large = dt_large.append(dt_temp)
    dt_large['sdis'] = sdis
    dt_large['F'] = F
    dt_large['CR'] = CR
    dt_large['popsize'] = popsize
    dt_large.columns = ['div0', 'div1', 'sdis', 'F', 'CR', 'popsize']
    return dt_large
for F in ['0755','0285', '099', '052', '005']: 
    for CR in ['0755', '0285', '099', '052', '005', '00891', '01283', '01675', '02067', '02458']:
        for popsize in [5, 20, 100]:
            for sdis in  ['t', 'h', 'm', 's', 'c', 'u']:
                dt = get_diversity_dt(sdis, F, CR, popsize)
                dt.to_csv(f"{output_location}DE_{sdis}_p{popsize}_F{F}CR{CR}_diversity.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17436823104693142, 'fraction_numerical': 0.04891696750902527, 'mean_word_length': 3.2362385321100917, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '4587', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2333, 'n_tokens_neox': 2151, 'n_words': 485, 'path': 'process_f0.ipynb', 'repo_name': 'Dvermetten/DE_TIOBR', 'size': '5540'}