<jupyter_start><jupyter_text>Download DataThis notebook downloads the necessary data to replicate the results of our paper on Gender Inequalities on Wikipedia.Note that we use a file named `dbpedia_config.py` where we set which language editions we will we study, as well as where to save and load data files.By [<NAME>](http://carnby.github.io).<jupyter_code>!cat dbpedia_config.py
import subprocess
import os
import dbpedia_config
target = dbpedia_config.DATA_FOLDER
languages = dbpedia_config.LANGUAGES
# Ontology
# note that previously (2014 version and earlier) this was in bzip format.
if not os.path.exists('{0}/dbpedia.owl'.format(target)):
    subprocess.call(['/usr/bin/wget', 
        'http://downloads.dbpedia.org/2015-10/dbpedia_2015-10.owl',
        '-O', '{0}/dbpedia.owl'.format(target)], 
        stdout=None, stderr=None)
# current version: http://wiki.dbpedia.org/Downloads2015-04
db_uri = 'http://downloads.dbpedia.org/2015-10/core-i18n'
for lang in languages:
    if not os.path.exists('{0}/instance_types_{1}.ttl.bz2'.format(target, lang)):
        subprocess.call(['/usr/bin/wget', 
            '{1}/{0}/instance_types_{0}.ttl.bz2'.format(lang, db_uri),
            '-O', '{0}/instance_types_{1}.ttl.bz2'.format(target, lang)], 
            stdout=None, stderr=None)
    
    if not os.path.exists('{0}/interlanguage_links_{1}.ttl.bz2'.format(target, lang)):
        subprocess.call(['/usr/bin/wget', 
            '{1}/{0}/interlanguage_links_{0}.ttl.bz2'.format(lang, db_uri),
            '-O', '{0}/interlanguage_links_{1}.ttl.bz2'.format(target, lang)], 
            stdout=None, stderr=None)
    
    if not os.path.exists('{0}/labels_{1}.ttl.bz2'.format(target, lang)):
        subprocess.call(['/usr/bin/wget', 
            '{1}/{0}/labels_{0}.ttl.bz2'.format(lang, db_uri),
            '-O', '{0}/labels_{1}.ttl.bz2'.format(target, lang)], 
            stdout=None, stderr=None)
        
    if not os.path.exists('{0}/mappingbased_literals_{1}.ttl.bz2'.format(target, lang)):
        subprocess.call(['/usr/bin/wget', 
            '{1}/{0}/mappingbased_literals_{0}.ttl.bz2'.format(lang, db_uri),
            '-O', '{0}/mappingbased_literals_{1}.ttl.bz2'.format(target, lang)], 
            stdout=None, stderr=None)
        
    if not os.path.exists('{0}/mappingbased_objects_{1}.ttl.bz2'.format(target, lang)):
        subprocess.call(['/usr/bin/wget', 
            '{1}/{0}/mappingbased_objects_{0}.ttl.bz2'.format(lang, db_uri),
            '-O', '{0}/mappingbased_objects_{1}.ttl.bz2'.format(target, lang)], 
            stdout=None, stderr=None)
# http://oldwiki.dbpedia.org/Datasets/NLP#h172-7
dbpedia_gender = 'http://wifo5-04.informatik.uni-mannheim.de/downloads/datasets/genders_en.nt.bz2'

if not os.path.exists('{0}/genders_en.nt.bz2'.format(target)):
    subprocess.call(['/usr/bin/wget', 
        dbpedia_gender,
        '-O', '{0}/genders_en.nt.bz2'.format(target)], 
        stdout=None, stderr=None)
# http://www.davidbamman.com/?p=12
# note that, in previous versions, this was a text file. now it's a bzipped file with n-triplets.
wikipedia_gender = 'http://www.ark.cs.cmu.edu/bio/data/wiki.genders.txt'

if not os.path.exists('{0}/wiki.genders.txt'.format(target)):
    subprocess.call(['/usr/bin/wget', 
        dbpedia_gender,
        '-O', '{0}/wiki.genders.txt'.format(target)], 
        stdout=None, stderr=None)
if not os.path.exists('{0}/long_abstracts_{1}.nt.bz2'.format(target, dbpedia_config.MAIN_LANGUAGE)):
    subprocess.call(['/usr/bin/wget', 
        '{1}/{0}/long_abstracts_{0}.ttl.bz2'.format(dbpedia_config.MAIN_LANGUAGE, db_uri),
        '-O', '{0}/long_abstracts_{1}.ttl.bz2'.format(target, dbpedia_config.MAIN_LANGUAGE)], 
        stdout=None, stderr=None)
# network data for english only
if not os.path.exists('{0}/page_links_{1}.ttl.bz2'.format(target, dbpedia_config.MAIN_LANGUAGE)):
    subprocess.call(['/usr/bin/wget', 
        '{1}/{0}/page_links_{0}.nt.bz2'.format(dbpedia_config.MAIN_LANGUAGE, db_uri),
        '-O', '{0}/page_links_{1}.ttl.bz2'.format(target, dbpedia_config.MAIN_LANGUAGE)], 
        stdout=None, stderr=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17854554240154627, 'fraction_numerical': 0.02850930176371104, 'mean_word_length': 3.485373781148429, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '8836', 'license': 'MIT', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1497, 'n_words': 219, 'path': 'notebooks/Notability - 01 - Download DBPedia and Gender Data.ipynb', 'repo_name': 'clauwag/WikipediaGenderInequality', 'size': '4156'}