<commit_before>import glob
from avidaspatial import *
import random
import scipy.stats

dir_list = glob.glob("env_*_1*")

data = ["env, ud, task, x, y, div, rank"]

for dir_name in dir_list:
    env_num = dir_name.split("_")[-2]
    task_loc_file = open(dir_name + "/task_locs.csv")
    task_locs = task_loc_file.readlines()
    task_loc_file.close()

    task_dict = {}
    for line in task_locs:
        sline = line.split(",")
        task_dict[int(sline[3])] = [int(i) for i in sline[:3]]

    grids = glob.glob(dir_name+"/data/grid_task.*.dat")
    envs = load_grid_data(grids)

    for j, env in enumerate(envs):
        ud = int(grids[j].split(".")[-2])
        env = agg_grid(env, mode)
        div_map = diversity_map(env, func=get_25_neighbors)
        task = task_dict[ud+1]
        print(task)
        div = div_map[task[2]][task[1]]
        rank = scipy.stats.percentileofscore(flatten_array(div_map), div)

        data.append(",".join([str(i) for i in [env_num, ud, task[0], taxk[1], task[2], div, rank]]))

outfile = open("diversity_ranks.csv")
outfile.write("\n".join(data))
outfile.close()<commit_msg>use correct neighbor func<commit_after>import glob
from avidaspatial import *
import random
import scipy.stats

dir_list = glob.glob("env_*_1*")

data = ["env, ud, task, x, y, div, rank"]

for dir_name in dir_list:
    env_num = dir_name.split("_")[-2]
    task_loc_file = open(dir_name + "/task_locs.csv")
    task_locs = task_loc_file.readlines()
    task_loc_file.close()

    task_dict = {}
    for line in task_locs:
        sline = line.split(",")
        task_dict[int(sline[3])] = [int(i) for i in sline[:3]]

    grids = glob.glob(dir_name+"/data/grid_task.*.dat")
    envs = load_grid_data(grids)

    for j, env in enumerate(envs):
        ud = int(grids[j].split(".")[-2])
        env = agg_grid(env, mode)
        div_map = diversity_map(env, neighbor_func=get_25_neighbors)
        task = task_dict[ud+1]
        print(task)
        div = div_map[task[2]][task[1]]
        rank = scipy.stats.percentileofscore(flatten_array(div_map), div)

        data.append(",".join([str(i) for i in [env_num, ud, task[0], taxk[1], task[2], div, rank]]))

outfile = open("diversity_ranks.csv")
outfile.write("\n".join(data))
outfile.close()
{'fraction_non_alphanumeric': 0.16290393979637008, 'fraction_numerical': 0.011509517485613104, 'mean_word_length': 3.4313725490196076, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2985692', 'n_tokens_mistral': 884, 'n_tokens_neox': 869, 'n_words': 170}