<jupyter_start><jupyter_text>Evaluation Imports<jupyter_code>import ast
import os
os.environ['MODIN_ENGINE'] = 'dask'
import modin.pandas as pd
from collections import Counter

from nltk.corpus import stopwords
stopwords = stopwords.words('german')
import string
from HanTa import HanoverTagger as ht
tagger = ht.HanoverTagger('./morphmodel_ger.pgz')
import re

import numpy as np

import matplotlib.pyplot as plt
import seaborn as sn
from sklearn import preprocessing
from scipy.stats import ttest_rel
from  scipy  import  stats<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>chunksize= 10 ** 6
chunks = pd.read_csv('./data/btw17_rawdata_gender.csv', chunksize=chunksize)
btw17_rawdata_df = pd.concat(chunks)
btw17_rawdata_df.head(3)<jupyter_output><empty_output><jupyter_text>Preprocessing*Preprocessing Pipeline from preprocessing.ipynb is used except for duplicate removal*<jupyter_code>with open('./data/all_names_to_delete.txt', 'r', encoding='utf-8') as f:
    names = [term[:-1] for term in f.readlines()]
word_pattern = re.compile('\w+')
with open('./data/wordlist-german.txt', 'r', encoding='utf-8') as f:
    german_terms = [term[:-1].lower() for term in f.readlines()]
def delete_names(matchobj)->str:
    word = matchobj.group(0)
    if word in names:
        return ""
    else:
        return word
def preprocess_keep_duplicates(suggestions:list)->list:
    suggestions = tuple(map(lambda x: x.translate(str.maketrans(string.punctuation, ' ' * len(string.punctuation))), suggestions))
    suggestions = [word_pattern.sub(delete_names, term.lower()) for term in suggestions]
    suggestions = [suggestion for suggestion in suggestions if len(suggestion.split()) == 1]
    suggestions = tuple(map(lambda x: re.sub('[0-9]*', '', x), suggestions))
    suggestions = [x.lower() for x in suggestions]
    suggestions = [x for x in suggestions if not 'http' in x]
    suggestions = [x for x in suggestions if not x in stopwords]
    suggestions = [tagger.analyze(tagger.analyze(suggestion)[0])[0].lower()
                   for suggestion in suggestions
                   if tagger.analyze(tagger.analyze(suggestion)[0])[1] in ['NN']]
    suggestions = [re.sub('(^..?\s|\s..?\s|\s..?$|^..?$)', '', string) for string in suggestions]
    suggestions = [x.strip() for x in suggestions]
    suggestions = list(filter(None, suggestions))
    suggestions = [term for term in suggestions if term in german_terms]
    return (suggestions)<jupyter_output><empty_output><jupyter_text>Group by gender*Create one DataFrame for terms suggested for male politicians and one for terms suggested for female politicians* *female*<jupyter_code>female_suggestions = btw17_rawdata_df.groupby('gender').get_group('female')<jupyter_output><empty_output><jupyter_text>*male*<jupyter_code>male_suggestions = btw17_rawdata_df.groupby('gender').get_group('male')<jupyter_output><empty_output><jupyter_text>Turn DataFrames into flat lists, preprocess the terms of each list and keep the duplicates\**Preprocessing is performed on slices in a loop and results are saved to file as a workaround for memory errors* *female*<jupyter_code>for i in range(0,len(female_suggestions)+1,int(len(female_suggestions)/10)):
    j = i+int(len(female_suggestions)/10)
    female_suggestions_prep = [term for row in female_suggestions.suggestions[i:j].to_list() for term in ast.literal_eval(row)]
    with open('./data/female_suggestions_prep.txt','a',encoding='utf-8') as file:
        for item in female_suggestions_prep:
            file.write(item+'\n')<jupyter_output><empty_output><jupyter_text>*male*<jupyter_code>for i in range(0,len(male_suggestions)+1,int(len(male_suggestions)/10)):
    j = i+int(len(male_suggestions)/10)
    male_suggestions_prep = [term for row in male_suggestions.suggestions.iloc[i:j].to_list() for term in ast.literal_eval(row)]
    with open('./data/male_suggestions_prep.txt','a',encoding='utf-8') as file:
        for item in male_suggestions_prep:
            file.write(item+'\n')<jupyter_output><empty_output><jupyter_text>Count occurrences of words per cluster and gender  Load Clusters<jupyter_code>cluster_df = pd.read_csv('./data/manual_clustering.csv')<jupyter_output><empty_output><jupyter_text>*Remove the "geographical" and "other" clusters since they're not comparable*<jupyter_code>cluster_df = cluster_df[1:-1]
clusters = [ast.literal_eval(cluster) for cluster in cluster_df.cluster_terms.to_list()]<jupyter_output><empty_output><jupyter_text>Load suggestion terms and count occurences per cluster *female*<jupyter_code>with open('./data/female_suggestions_prep.txt', 'r',encoding = 'utf-8') as file:
    female_suggestions = [term[:-1] for term in file.readlines() if term != '']
female_counts = Counter(female_suggestions)
for index,cluster in enumerate(clusters):
    count = 0
    for term in cluster:
        count += int(female_counts[term])
    cluster_df.at[index,'female_count'] = count
cluster_df[['cluster_label','cluster_terms','female_count']].to_csv('./data/cluster_count_female.csv')<jupyter_output><empty_output><jupyter_text>*male*<jupyter_code>with open('./data/male_suggestions_prep.txt', 'r',encoding = 'utf-8') as file:
    content = file.read()
    male_suggestions = content.split('\n')#[term[:-1] for term in file.readlines() if term != '']
male_counts = Counter(male_suggestions)
for cluster in clusters:
    count=0
    for term in cluster:
        print(term)
        count+=int(male_counts[term])
    with open('../male_cluster_counts.csv','a') as file:
        file.write(str(count)+'\n')<jupyter_output><empty_output><jupyter_text>Genderize API Evaluation Load Gender data<jupyter_code>btw17_name_gender_df = pd.read_csv('./data/btw17_name_gender_df.csv')
btw17_name_gender_df.head(5)<jupyter_output><empty_output><jupyter_text>Load and prepare testset<jupyter_code>politician_df = pd.read_csv('./data/politiker.csv')
politician_df.head(5)

politician_df['f_name'] = politician_df['Name'].apply(lambda x: x.split(' ')[0])
politician_df = politician_df[['Gender','f_name']]
politician_df = politician_df.rename(columns={'Gender': 'gender_test'})
politician_df = politician_df[['f_name','gender_test']]
politician_df = politician_df.drop_duplicates()
politician_df.head(5)
btw17_name_gender_df = btw17_name_gender_df[['f_name','gender']]
btw17_name_gender_df.head(5)
politician_df = politician_df.dropna()
btw17_name_gender_df = btw17_name_gender_df.dropna()
print(politician_df.shape)
btw17_name_gender_df.shape
politician_df = politician_df.reset_index(drop=True)
btw17_name_gender_df = btw17_name_gender_df.reset_index(drop=True)
merged_gender_df = btw17_name_gender_df.merge(politician_df, on='f_name').astype(str)
merged_gender_df.columns = ['f_name','gender_pred','gender_true']
merged_gender_df.head(5)<jupyter_output><empty_output><jupyter_text>Create a confusion matrix<jupyter_code>gender_pred = merged_gender_df['gender_pred']
gender_true = merged_gender_df['gender_true']

confusion_df = pd.crosstab(gender_true, gender_pred, margins=True,rownames=['gender_true'], colnames=['gender_pred'])
print(confusion_df)
plt.clf()
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_aspect(1)
res = sn.heatmap(confusion_df.T, annot=True, fmt='.2f', cmap="YlGnBu", cbar=False)
plt.title('Confusion Matrix')
plt.xlabel('True Gender')
plt.ylabel('Predicted Gender')
plt.savefig("./data/confusion_matrix.png", dpi=300)
plt.show()<jupyter_output><empty_output><jupyter_text>Gender Cluster Evaluation<jupyter_code>cluster_count_df = pd.read_csv('./data/cluster_count_female.csv')
cluster_count_df['male_count']  = pd.read_csv('./data/male_cluster_counts.csv')
cluster_count_df.head(3)
cluster_count_df = cluster_count_df[['cluster_label','female_count', 'male_count']]
cluster_count_df.head(10)
x = cluster_count_df[['female_count', 'male_count']].values #returns a numpy array
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)
cluster_count_df = pd.concat([cluster_count_df,pd.DataFrame(x_scaled)], axis=1)
cluster_count_df.columns = ['cluster_label','female_count','male_count','female_norm','male_norm']
cluster_count_df.head(10)

total_female_norm = pd.Series(cluster_count_df['female_norm'], dtype='float').sum()
total_male_norm = pd.Series(cluster_count_df['male_norm'], dtype='float').sum()

cluster_count_df['female_perc'] = cluster_count_df.female_norm.apply(lambda x: float(x) / total_female_norm )
cluster_count_df['male_perc'] = cluster_count_df.male_norm.apply(lambda x: float(x) / total_male_norm )
cluster_count_df.head(5)<jupyter_output><empty_output><jupyter_text>Plot distribution<jupyter_code># data to plot
n_cluster = 18
male_perc = cluster_count_df['male_perc']*100
female_perc = cluster_count_df['female_perc']*100

# create plot
fig, ax = plt.subplots()
index = np.arange(n_cluster)
bar_width = 0.40
opacity = 0.9

rects1 = plt.bar(index, male_perc, bar_width,
alpha=opacity,
color='c',
label='Männlich')

rects2 = plt.bar(index + bar_width, female_perc, bar_width,
alpha=opacity,
color='y',
label='Weiblich')

plt.xlabel('Cluster Label')
plt.ylabel('Anteil [%]')
plt.title('Prozentuale Verteilung nach Geschlecht und Cluster')
plt.xticks(index + bar_width ,['Unternehmen',
                               'Hobbies',
                               'Nachrichten',
                               'Militär/Polizei',
                               'Technologie',
                               'Kultur',
                               'Bildung',
                               'Wissenschaft',
                               'Mode',
                               'Privatleben',
                               'Transport',
                               'Politik',
                               'Wirtschaft',
                               'Speisen',
                               'Gesundheit',
                               'Recht',
                               'Terror',
                               'Religion'], rotation = 80)
plt.legend()

plt.tight_layout()
plt.savefig('./data/distribution_gender_cluster.png', dpi=300)
plt.show()<jupyter_output><empty_output><jupyter_text>Independent T-Test<jupyter_code>female_cluster_norm = cluster_count_df['female_norm'].apply(lambda x: float(x))
male_cluster_norm = cluster_count_df['male_norm'].apply(lambda x: float(x))
stats.ttest_ind(male_cluster_norm,female_cluster_norm)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13076997205358004, 'fraction_numerical': 0.010889467090681315, 'mean_word_length': 4.92351598173516, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '362152', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3699, 'n_tokens_neox': 3514, 'n_words': 612, 'path': 'src/evaluation.ipynb', 'repo_name': 'melaniefumfack/DIS25_NLP', 'size': '10377'}