<jupyter_start><jupyter_text>Reformatting dataNotebook to reformat data, this is a manual one time thing so no need to look at it for the results.<jupyter_code>import numpy as np
import pandas as pd

gender_male_nl,gender_female_nl = [], []
with open('../Rodrigo-data/Clustering/male_word_file_nl.txt') as f:
    for l in f:
        gender_male_nl.append(l.strip())
with open('../Rodrigo-data/Clustering/female_word_file_nl.txt') as f:
    for l in f:
        gender_female_nl.append(l.strip())
gender_male_en,gender_female_en = [], []
with open('../Rodrigo-data/Clustering/male_word_file_en.txt') as f:
    for l in f:
        gender_male_en.append(l.strip())
with open('../Rodrigo-data/Clustering/female_word_file_en.txt') as f:
    for l in f:
        gender_female_en.append(l.strip())
female_dict = {'en':gender_female_en,'nl':gender_female_nl}
female = pd.DataFrame(female_dict)
female.to_csv('../Rodrigo-data/Clustering/female_words_en_nl.csv')
male_dict = {'en':gender_male_en,'nl':gender_male_nl}
male = pd.DataFrame(male_dict)
male.to_csv('../Rodrigo-data/Clustering/male_words_en_nl.csv')

def create_df(category, en_list, nl_list):
    size = len(en_list)
    category_list = [category]*size
    dict = {'category':category,'en':en_list,'nl':nl_list}
    return pd.DataFrame(dict)
import WEAT #import WEAT functions 
from lists.WEAT_lists_NL import * #Dutch dataset
from lists.WEAT_lists import * #Dutch dataset
WEATNL.W_1_Flowers
WEATLists.W_1_Flowers
a= create_df('flowers',WEATLists.W_1_Flowers, WEATNL.W_1_Flowers)
b = create_df('insects',WEATLists.W_1_Insects, WEATNL.W_1_Insects)
c = a.append(b)
c.loc[0]
triples = [
    ('flowers',WEATLists.W_1_Flowers, WEATNL.W_1_Flowers),
    ('insects',WEATLists.W_1_Insects, WEATNL.W_1_Insects),
    ('pleasant',WEATLists.W_1_Pleasant, WEATNL.W_1_Pleasant),
    ('unpleasant',WEATLists.W_1_Unpleasant, WEATNL.W_1_Unpleasant),
    ('instruments',WEATLists.W_2_Instruments, WEATNL.W_2_Instruments),
    ('weapons',WEATLists.W_2_Weapons, WEATNL.W_2_Weapons),
    ('Unused_full_list_European_American_names',WEATLists.W_3_Unused_full_list_European_American_names, WEATNL.W_3_Unused_full_list_European_American_names),
    ('european_american_names',WEATLists.W_3_European_American_names, WEATNL.W_3_European_American_names),
    ('Unused_full_list_African_American_names',WEATLists.W_3_Unused_full_list_African_American_names, WEATNL.W_3_Unused_full_list_African_American_names),
    ('african_american_names',WEATLists.W_3_African_American_names, WEATNL.W_3_African_American_names),
    ('male_names',WEATLists.W_6_Male_names, WEATNL.W_6_Male_names),
    ('female_names',WEATLists.W_6_Female_names, WEATNL.W_6_Female_names),
    ('career',WEATLists.W_6_Career, WEATNL.W_6_Career),
    ('family',WEATLists.W_6_Family, WEATNL.W_6_Family),
    ('math',WEATLists.W_7_Math, WEATNL.W_7_Math),
    ('arts',WEATLists.W_7_Arts, WEATNL.W_7_Arts),
    ('male_terms',WEATLists.W_7_Male_terms, WEATNL.W_7_Male_terms),
    ('female_terms',WEATLists.W_7_Female_terms, WEATNL.W_7_Female_terms),
    ('science',WEATLists.W_8_Science, WEATNL.W_8_Science),
    ('mental_disease',WEATLists.W_9_Mental_disease, WEATNL.W_9_Mental_disease),
    ('physical_disease',WEATLists.W_9_Physical_disease, WEATNL.W_9_Physical_disease),
    ('temporary',WEATLists.W_9_Temporary, WEATNL.W_9_Temporary),
    ('permanent',WEATLists.W_9_Permanent, WEATNL.W_9_Permanent),
    ('young_peoples_names',WEATLists.W_10_Young_peoples_names, WEATNL.W_10_Young_peoples_names),
    ('old_peoples_names',WEATLists.W_10_Old_peoples_names, WEATNL.W_10_Old_peoples_names)
    ]

weat_df = pd.DataFrame()
for t in triples:
    b = create_df(*t)
    weat_df = weat_df.append(b)
weat_df.to_csv('../Rodrigo-data/Clustering/weat_lists_en_nl.csv')<jupyter_output><empty_output><jupyter_text>SEAT data<jupyter_code># import json
# import numpy as np
# import pandas as pd
# #Read seat like the other embeddings and save them in csv
# test_path = '../Rodrigo-data/SEAT/tests'
# TEST_EXT = '.jsonl'
# # all_tests = sorted([ entry[:-len(TEST_EXT)] for entry in os.listdir(test_path) if not entry.startswith('.') and entry.endswith(TEST_EXT)], key=test_sort_key)
# all_tests = [entry[:-len(TEST_EXT)] for entry in os.listdir(test_path) if not entry.startswith('.') and entry.endswith(TEST_EXT)]
# # all_tests.sort()
# print(all_tests)
# abc = pd.read_json(test_path+'/'+all_tests[0]+TEST_EXT, orient ='columns')
# # abc.targ1.examples
# categories = abc.loc['category'].to_list()
# categories
# abc.columns
# at1 = abc.attr1.to_list()
# at2 = abc.attr2.to_list()
# targ1 = abc.attr1.to_list()
# targ2 = abc.targ2.to_list()
# at1_tuple = {'category':at1[0],'en':at1[1],'nl':[0]*len(at1[1])}
# at1_tuple = pd.DataFrame(at1_tuple)
# at2_tuple = pd.DataFrame({'category':at2[0],'en':at2[1],'nl':[0]*len(at2[1])})
# targ1_tuple = pd.DataFrame({'category':targ1[0],'en':targ1[1],'nl':[0]*len(targ1[1])})
# targ2_tuple = pd.DataFrame({'category':targ2[0],'en':targ2[1],'nl':[0]*len(targ2[1])})
# df_seat = pd.DataFrame()
# df_seat = df_seat.append(at1_tuple)
# df_seat = df_seat.append(at2_tuple)
# df_seat = df_seat.append(targ1_tuple)
# df_seat = df_seat.append(targ2_tuple)
# df_seat.to_csv(test_path+'/'+all_tests[0]+'_test0002.csv')<jupyter_output><empty_output><jupyter_text>Actual important methods<jupyter_code>import json
import numpy as np
import pandas as pd

def seat_to_csv(test_path, filename, extension='.jsonl'):
    seat = pd.read_json(test_path+filename+extension, orient ='columns')
    seat_df = pd.DataFrame()
    for col in seat:
        att_list = seat[col].to_list()
        att_dict = {'category':att_list[0],'en':att_list[1],'nl':[0]*len(att_list[1])}
        att_df = pd.DataFrame(att_dict)
        seat_df =seat_df.append(att_df)
    seat_df.to_csv('../Rodrigo-data/SEAT/csv/'+filename+'.csv')    

def update_SEAT_data():
    #Read seat like the other embeddings and save them in csv
    test_path = '../Rodrigo-data/SEAT/tests/'
    TEST_EXT = '.jsonl'
    # all_tests = sorted([ entry[:-len(TEST_EXT)] for entry in os.listdir(test_path) if not entry.startswith('.') and entry.endswith(TEST_EXT)], key=test_sort_key)
    all_tests = [entry[:-len(TEST_EXT)] for entry in os.listdir(test_path) if not entry.startswith('.') and entry.endswith(TEST_EXT)]
    for test_name in all_tests:
        seat_to_csv(test_path, filename=test_name)
update_SEAT_data()<jupyter_output><empty_output><jupyter_text>Pairs Gender from Wiki dump<jupyter_code>words = [["vrouw", "man"], ["meisje", "jongen"], ["zij", "hij"], ["moeder", "vader"], ["dochter", "zoon"], ["zus","broer"], ["vrouwelijk","mannelijk"], ["haar","zijn"], ["haarzelf","hemzelf"], ["Emma","Daan"]]
def match(a,L):
	for b in L:
		if a == b:
			return True
	return False

def replace(a,new,L):
	Lnew = []
	for b in L:
		if a == b:
			Lnew.append(new)
		else:
			Lnew.append(b)
	return ' '.join(Lnew)

def template2(words, sent, sent_words_list, all_pairs):
	for (female, male) in words:
		if match(female, sent_words_list):
			sent_f = sent
			sent_m = replace(female,male,sent_words_list)
			pair = (sent_f,sent_m)
			all_pairs.append(pair)
		if match(male, sent_words_list):
			sent_f = replace(male,female,sent_words_list)
			sent_m = sent
			pair = (sent_f,sent_m)
			all_pairs.append(pair)
	return all_pairs<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15005476451259583, 'fraction_numerical': 0.016292442497261774, 'mean_word_length': 5.943916349809886, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '186428', 'license': 'MIT', 'n_tokens_mistral': 3161, 'n_tokens_neox': 2957, 'n_words': 428, 'path': 'Code/parallelize_data.ipynb', 'repo_name': 'Noixas/Evaluating-Bias-In-Dutch-Word-Embeddings', 'size': '7304'}