<jupyter_start><jupyter_text>|    -de-   | Occup. | Wiki pages | Validated pages | Ambigious pages | Redirects | Redir. to oposit gender ||:-:|:-:|:-:|:-:|:-:|:-:|:-:|| Masculine |  4274  |    934     |       804       |       115       |    503    |            5            ||  Feminine |  4274  |     20     |        13       |        5        |    310    |           214           ||  Neutral  |  183   |     37     |        36       |        0        |     7     |      to M:3, to F:0     ||           | All | Wiki pages | Redirects | No page ||:-:|:-:|:-:|:-:|:-:|| Masculine |  4274  |    934     |    503    |   2837  ||  Feminine |  4274  |     20     |    310    |   3944  ||  Neutral  |  183   |     37     |     7     |   139   ||           | All wiki pages | Validated pages | Ambigious pages | Not validated pages ||:-:|:-:|:-:|:-:|:-:|| Masculine |    934     |       804       |       115       |          15         ||  Feminine |     20     |        13        |        5        |          2          ||  Neutral  |     37     |        36       |        0        |          1          ||           | All wiki pages | Validated pages about profession | Other pages ||:-:|:-:|:-:|:-:|:-:|| Masculine |    934     |       815       |       119        ||  Feminine |     20     |        18       |        2         ||  Neutral  |     37     |        36       |        1         ||           | All redirects | Redirects to opposite gender | Other redirects ||:-:|:-:|:-:|:-:|| Masculine |    503    |             5              |       498       ||  Feminine |    310    |            214             |        96       ||  Neutral  |     7     |       to M:3, to F:0       |        4        |<jupyter_code>set(words_revert.keys())-set(no_page_f)-set(page_f)-set(f_links_to_mascuilne)-set(f_links_to_smth.keys())\
-set(f_links_to_mascuilne_with_link.keys())-set(f_links_to_mascuilne_lev.keys())<jupyter_output><empty_output><jupyter_text>do not start next one<jupyter_code>from prettytable import PrettyTable
from tabulate import tabulate

import json
def load_simple_json(filename):
    with open(filename, 'r') as f:
        return json.load(f)


headers=["Occupation", 'Wiki pages', "Redirects","No wiki page or redir","All"]
t = PrettyTable(headers)


table=[]
for l in ["de","fr","ru"]:
    m_no_page=load_simple_json(l+'/wiki/m_no_page.json')
    m_redirection=load_simple_json(l+'/wiki/m_redirection.json')
    m_page=load_simple_json(l+'/wiki/m_page.json')
    words=len(m_no_page)+len(m_page)+ len(m_redirection)
    if l=="de":
        words_data=load_simple_json('de/occupation_all.json')
        words=len(words_data)
        m_page_levenshtein=load_simple_json(l+'/wiki/m_page_levenshtein.json')
        n_page=load_simple_json(l+'/wiki/n_page.json')
        n_no_page=load_simple_json(l+'/wiki/n_no_page.json')
        f_links_levenshtein=load_simple_json(l+'/wiki/f_links_to_mascuilne_lev.json')
        t.add_row(['{0}: Masculine'.format(l), len(m_page)+len(m_page_levenshtein),
                   len(m_redirection),len(m_no_page),words])
        f_no_page=load_simple_json(l+'/wiki/f_no_page.json')
        f_redirection=load_simple_json(l+'/wiki/f_redirection.json')
        f_page=load_simple_json(l+'/wiki/f_page.json')
        t.add_row(['{0}:  Feminine'.format(l),len(f_page),len(f_redirection),len(f_no_page), words])
        t.add_row(['{0}:  Neutral'.format(l),len(n_page),len(redirection_n),len(n_no_page), len(neutral)])
        table.append(['{0}: Masculine'.format(l), len(m_page)+len(m_page_levenshtein),
                   len(m_redirection),len(m_no_page),words])
        table.append(['{0}:  Feminine'.format(l), len(f_page),len(f_redirection),len(f_no_page),words])
    else:
        t.add_row(['{0}: Masculine'.format(l),len(m_page),len(m_redirection),len(m_no_page),words])
        table.append(['{0}: Masculine'.format(l),len(m_page),len(m_redirection),len(m_no_page),words])
print t

print tabulate(table, headers=headers, tablefmt="latex")#, numalign="left")<jupyter_output>+---------------+------------+-----------+-----------------------+------+
|   Occupation  | Wiki pages | Redirects | No wiki page or redir | All  |
+---------------+------------+-----------+-----------------------+------+
| de: Masculine |    934     |    503    |          2837         | 4274 |
| de:  Feminine |     21     |    310    |          3943         | 4274 |
|  de:  Neutral |     37     |     7     |          139          | 183  |
| fr: Masculine |     92     |     47    |          113          | 252  |
| ru: Masculine |    227     |    147    |          7296         | 7670 |
+---------------+------------+-----------+-----------------------+------+
\begin{tabular}{lrrrr}
\hline
 Occupation    &   Wiki pages &   Redirects &   No wiki page or redir &   All \\
\hline
 de: Masculine &          934 &         503 &                    2837 &  4274 \\
 de:  Feminine &           21 &         310 &                    3943 &  4274 \\
 fr: Masculine &           92 &          47 &         [...]<jupyter_text>|Occupation  | Wiki pages | Redirects | No wiki page or redir | All  || :-|-|-|-|:-:|| de: Masculine |    934     |    503    |          2837         | 4274 || de:  Feminine |     20     |    310    |          3944         | 4274 || de:   Neutral |     37     |      7    |          139          |  183 || fr: Masculine |     92     |     47    |          113          | 252  || ru: Masculine |    227     |    147    |          7296         | 7670 |  Print all biases examples<jupyter_code>f_b=[]
for i in words:
    if i in m_links_to_feminine:
        print "Female bias:", i ,"=redirects to=>", words[i][1]
        f_b.append(i)
    elif words[i][1] in page_f_ambigious:
        if (i not in page_m)&(i not in m_links_to_smth):
            print "Female bias:", i ,"=has only page=", words[i][1]
            f_b.append(i)
    elif words[i][1] in page_f_validated:
        if (i not in page_m)&(i not in m_links_to_smth):
            print "Female bias:", i ,"=has only page=", words[i][1]
            f_b.append(i)
print "\n\n"

n_b=[]
for i in words:
    if (i in page_m_validated)&(words[i][1] in page_f_validated):
        print "Neutral:", i ,"<= both pages exist=>", words[i][1]
        n_b.append(i)
    elif (i in page_m_ambigious)&(words[i][1] in page_f_validated):
        print "Neutral:", i ,"<= both pages exist=>", words[i][1]
        n_b.append(i)
    elif (i in m_links_to_smth)&(words[i][1] in page_f_validated):
        print "Neutral:", i ,"<=this redirects to smth ,both pages exist=>", words[i][1]
        n_b.append(i)
    elif words[i][1] in page_f_ambigious:
        if i in page_m:
            print "Neutral:", i ,"<= both pages exist & this is ambigious page=>", words[i][1]
            n_b.append(i)
        elif i in m_links_to_smth:
            print "Neutral:", i ,"<= this redirects to smth | this is ambigious page=>", words[i][1]
            n_b.append(i)
n_b2=[]
for i in page_n_validated:
    print "Neutral:", i 
    n_b2.append(i)
    
m_neutral_lev=load_simple_json('de/wiki/m_neutral_lev.json')    
    
for i in m_neutral_lev:
    if (i in no_page_m) and (i not in f_b):
        print "Neutral: field name of ",i, " =only exist= ", m_neutral_lev[i]
        n_b.append(i)
print len(f_b), len(n_b),len(n_b2)    


    
with open("de/wiki/f_bias.json","w")as f:
    json.dump(f_b, f, indent=4)
with open("de/wiki/n_bias_male_labels.json","w")as f:
    json.dump(n_b, f, indent=4)
with open("de/wiki/n_bias_n_labels.json","w")as f:
    json.dump(n_b2, f, indent=4)  
m_b=[]
for i in words:
    if words[i][1] in f_links_to_mascuilne:
        print "Male bias:", words[i][1] ,"=redirects to=>", i
        m_b.append(i)
    elif f_links_to_mascuilne_lev.has_key(words[i][1]):
        print "==Male bias:",words[i][1] ,"=redirects to=>", f_links_to_mascuilne_lev[words[i][1]].values()
        m_b.append(i)
    elif f_links_to_mascuilne_with_link.has_key(words[i][1]):
        print "=Male bias:",words[i][1] ,"=redirects to=>", f_links_to_mascuilne_with_link[words[i][1]][1]
        m_b.append(i)
    elif (i in page_m_validated) & (i not in n_b) & (i not in f_b):
        print "++Male bias: Only page ", i , " exist"
        m_b.append(i)
    elif i in n_links_to_masculine.values():
        #is already in male bias
        print "!!!!!!!!!!!!!!",i
    elif m_page_levenshtein.has_key(i):
        print "--Male bias: Only page ", i,m_page_levenshtein[i] , " exist"
        m_b.append(i)#27

print len(m_b)
with open("de/wiki/m_bias.json","w")as f:
    json.dump(m_b, f, indent=4)<jupyter_output>++Male bias: Only page  Mikrotechnologe  exist
--Male bias: Only page  Metallformer Metallverformer  exist
++Male bias: Only page  Honorarprofessor  exist
Male bias: Komparsin =redirects to=> Komparse
Male bias: Missionarin =redirects to=> Missionar
Male bias: Historikerin =redirects to=> Historiker
Male bias: Fremdenführerin =redirects to=> Fremdenführer
Male bias: Richterin =redirects to=> Richter
Male bias: Bestatterin =redirects to=> Bestatter
Male bias: Philosophin =redirects to=> Philosoph
++Male bias: Only page  Kartograf  exist
++Male bias: Only page  Physikalisch-technischer Assistent  exist
Male bias: Galeristin =redirects to=> Galerist
++Male bias: Only page  Weinküfer  exist
Male bias: Sport- und Fitnesskauffrau =redirects to=> Sport- und Fitnesskaufmann
++Male bias: Only page  Diätkoch  exist
Male bias: Pädagogin =redirects to=> Pädagoge
++Male bias: Only page  Schriftgießer  exist
++Male bias: Only page  Systemelektroniker  exist
++Male bias: Only page  Automatenfachmann [...]<jupyter_text>Store all links in one file<jupyter_code>import pandas as pd
import json
def load_simple_json(filename):
    with open(filename, 'r') as f:
        return json.load(f)
    
words=load_simple_json('de/occupation_all.json')#{masculine:[[],[feminine]]}
words_revert={words[i][1]:i for i in words}#feminine:masculine
neutral=load_simple_json('de/neutral_cleaned.json')

no_page_m=load_simple_json('de/wiki/m_no_page.json')
no_page_f=load_simple_json('de/wiki/f_no_page.json')
no_page_n=load_simple_json('de/wiki/n_no_page.json')
page_m=load_simple_json('de/wiki/m_page.json')
page_f=load_simple_json('de/wiki/f_page.json')
page_n=load_simple_json('de/wiki/n_page.json')
m_page_levenshtein=load_simple_json('de/wiki/m_page_levenshtein.json')
#n_page_levenshtein=load_simple_json('de/wiki/n_page_levenshtein.json')


page_f_validated=load_simple_json('de/wiki/f_page_validated.json')
page_f_ambigious=load_simple_json('de/wiki/f_page_ambigious.json')
page_f_other=load_simple_json('de/wiki/f_page_other.json')  
    
page_m_validated=load_simple_json('de/wiki/m_page_validated.json') 
page_m_ambigious=load_simple_json('de/wiki/m_page_ambigious.json')
page_m_other=load_simple_json('de/wiki/m_page_other.json')

page_n_validated=load_simple_json('de/wiki/n_page_validated.json')
page_n_ambigious=load_simple_json('de/wiki/n_page_ambigious.json')
page_n_other=load_simple_json('de/wiki/n_page_other.json') 

m_links_to_feminine=load_simple_json('de/wiki/m_links_to_feminine.json')
m_links_to_feminine_lev=load_simple_json('de/wiki/m_links_to_feminine_lev.json')
m_links_to_smth=load_simple_json('de/wiki/m_links_to_smth.json')
f_links_to_mascuilne=load_simple_json('de/wiki/f_links_to_mascuilne.json')
f_links_to_smth=load_simple_json('de/wiki/f_links_to_smth.json')
redirection_n=load_simple_json('de/wiki/n_redirects.json')
n_links_to_masculine=load_simple_json('de/wiki/n_links_to_masculine.json')
n_links_to_smth=load_simple_json('de/wiki/n_links_to_smth.json')

f_links_to_mascuilne_lev=load_simple_json('de/wiki/f_links_to_mascuilne_lev.json')
f_links_to_mascuilne_with_link=load_simple_json('de/wiki/f_links_to_mascuilne_with_link.json')
f_redirection=load_simple_json('de/wiki/f_redirection.json')
m_redirection=load_simple_json('de/wiki/m_redirection.json')


f_bias=load_simple_json('de/wiki/f_bias.json')
m_bias=load_simple_json('de/wiki/m_bias.json')
n_bias_male_labels=load_simple_json('de/wiki/n_bias_male_labels.json')
n_bias_n_labels=load_simple_json('de/wiki/n_bias_n_labels.json')
all_bias={words[i][1]: "female" for i in f_bias}
#all_bias2={i: "neutral" for i in n_bias_n_labels}
{all_bias.update({i: "neutral"}) for i in n_bias_n_labels}
{all_bias.update({i: "male"}) for i in m_bias}
{all_bias.update({i: "neutral"}) for i in n_bias_male_labels}
    
dict_m=[]
#df1=pd.DataFrame(columns=["Name","WikiName","Oppositgender","Bias"])
for m in words:
    if m in page_m_validated:
        dict_m.append({"Name":m,"WikiName":m,"Oppositgender":words[m][1],"Bias":all_bias[m],"Ambigious":0})
    elif words[m][1] in f_links_to_mascuilne:
        #print m
        dict_m.append({"Name":m,"WikiName":m,"Oppositgender":words[m][1],"Bias":all_bias[m],"Ambigious":1})
    elif m_page_levenshtein.has_key(m):
        #print m
        dict_m.append({"Name":m,"WikiName":m_page_levenshtein[m],"Oppositgender":words[m][1],
                       "Bias":all_bias[m],"Ambigious":0})
    elif f_links_to_mascuilne_with_link.has_key(words[m][1]) :
        #print m,f_links_to_mascuilne_with_link[words[m][1]][1]
        dict_m.append({"Name":m,"WikiName":f_links_to_mascuilne_with_link[words[m][1]][1],"Oppositgender":words[m][1],
                       "Bias":all_bias[m],"Ambigious":0})
    elif (m in page_m_ambigious)& (all_bias.has_key(m)):
        dict_m.append({"Name":m,"WikiName":m,"Oppositgender":words[m][1],"Bias":all_bias[m],"Ambigious":1})
    elif (m_links_to_smth.has_key(m))& (all_bias.has_key(m)):
        #print m
        dict_m.append({"Name":m,"WikiName":m_links_to_smth[m],"Oppositgender":words[m][1],"Bias":all_bias[m],"Ambigious":0})
    elif m_neutral_lev.has_key(m):
        print m,m_neutral_lev[m]
        dict_m.append({"Name":m,"WikiName":m_neutral_lev[m],"Oppositgender":words[m][1],"Bias":all_bias[m],"Ambigious":0})
        #pass
    if words[m][1] in page_f_validated:
        #print m
        dict_m.append({"Name":words[m][1],"WikiName":words[m][1],"Oppositgender":m,
                       "Bias":all_bias[m] if m in all_bias else all_bias[words[m][1]],"Ambigious":0})
    elif words[m][1] in page_f_ambigious:
        #print m
        dict_m.append({"Name":words[m][1],"WikiName":words[m][1],"Oppositgender":m,
                       "Bias":all_bias[m] if m in all_bias else all_bias[words[m][1]],"Ambigious":1})
for i in n_bias_n_labels:
    dict_m.append({"Name":i,"WikiName":i,"Bias":all_bias[i],"Ambigious":0})
df1=pd.DataFrame(dict_m)

#df1=df1.append(dict_m,ignore_index=True)   
df1[df1.Bias=="neutral"]#67=36neutral+16f_and_m+11n_redirects+4f
print len(df1)
df1.to_csv("de/wiki/all_wiki_pages.csv", sep='\t', encoding='utf-8')
import pandas as pd
import json
def load_simple_json(filename):
    with open(filename, 'r') as f:
        return json.load(f)
    
df=pd.read_csv('de/wiki/all_wiki_pages.csv',sep='\t', encoding='utf-8',index_col=0)
print len(df[(df.Bias=="male")] )
print len(df[(df.Bias=="male")& (df.Name!=df.WikiName)] )
print len(df[(df.Ambigious==1)] )
df[(df.Ambigious==1)]<jupyter_output>812
37
16
{'fraction_non_alphanumeric': 0.16456284879085836, 'fraction_numerical': 0.026175923465320224, 'mean_word_length': 3.01306318315116, 'pattern_counts': {'":': 50, '<': 19, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '72950', 'license': 'MIT', 'n_tokens_mistral': 6069, 'n_tokens_neox': 5731, 'n_words': 941, 'path': '2.4.Plot tables with number of matched articles and redirections. Save redirection bias groups.ipynb', 'repo_name': 'gesiscss/Wikipedia-Language-Olga-master', 'size': '15052'}