<commit_before>__author__ = 'marc'

from lxml import html
from SPARQLWrapper import SPARQLWrapper, JSON
import requests, json, urlparse, urllib2


def main():
    # scrape wikipedia page with a list of all museums in the Netherlands
    entities_nl = get_entities('http://nl.wikipedia.org/wiki/Lijst_van_musea_in_Nederland',
                               '//div[@id="bodyContent"]/div[@id="mw-content-text"]'
                               '/h3/following-sibling::ul/li/a[not(@rel="nofollow")]/@href')
    entities_adam = get_entities('http://nl.wikipedia.org/wiki/Lijst_van_musea_in_Amsterdam',
                                 '//div[@id="bodyContent"]/div[@id="mw-content-text"]'
                                 '/ul[1]/li/a[not(@rel="nofollow")]/@href')
    museums_urls = entities_adam + entities_nl
    preprocess_entities(museums_urls)


def get_wiki_title(wiki_url):
    museum_page = requests.get('http://nl.wikipedia.org' + wiki_url)
    museum_tree = html.fromstring(museum_page.text)
    museum_entity = museum_tree.xpath('//h1[@id="firstHeading"]/text()')

    if museum_entity[0].startswith('Bezig'):
        parsed = urlparse.urlparse(wiki_url)
        return urlparse.parse_qs(parsed.query)['title'][0]

    return museum_entity[0].replace(' ', '_')


def check_blacklist(wiki_title):
    blacklist = [line.strip() for line in open('config/blacklist.cfg')]
    return wiki_title in blacklist


def get_entities(wiki_page, xpath):
    # scrape wikipedia page with a list of all museums in the Netherlands
    page = requests.get(wiki_page)
    tree = html.fromstring(page.text)
    return tree.xpath(xpath)


def get_geo_data(museum_title):
    # get geo information for museum
    geo_url = "https://maps.googleapis.com/maps/api/geocode/json?address=%s" % museum_title
    geo_response = urllib2.urlopen(geo_url)
    geo_jsongeocode = geo_response.read()
    return json.loads(geo_jsongeocode)


def get_dbpedia_data(museum_title):
    dbpedia_page = 'http://nl.dbpedia.org/resource/' + museum_title

    # Get dbpedia properties for museum
    sparql = SPARQLWrapper("http://nl.dbpedia.org/sparql")
    query = "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> " \
            "SELECT * WHERE { <%s> ?property ?entity }" % dbpedia_page
    sparql.setQuery(query)
    sparql.setReturnFormat(JSON)
    results = sparql.query().convert()
    return results["results"]["bindings"]


def get_hashtag(museum_title):
    return '#%s' % museum_title.replace('_', '')


def create_collections(collection_name, collection):
    with open('data/%s.json' % collection_name, 'w') as outfile:
        json.dump(collection, outfile)


def preprocess_entities(museums_urls):
    museum_titles = []
    museums = []
    hashtags = []
    locations = []
    succeeded = 0
    failed = 0
    duplicated = 0
    blacklisted = 0
    nogeodata = 0

    for url in museums_urls:
        try:
            museum_title = get_wiki_title(url)
            if museum_title in museum_titles:
                duplicated += 1
                print "[duplicated] processed entity: %s" % museum_title
                continue
            if check_blacklist(museum_title):
                blacklisted += 1
                print "[blacklisted] processed entity: %s" % museum_title
                continue
            geo_data = get_geo_data(museum_title)
            if geo_data['status'] == "ZERO_RESULTS":
                nogeodata += 1
                print "[no geodata] processed entity: %s" % museum_title
                continue

            # Add the results to the right collection<commit_msg>changed geodata check<commit_after>__author__ = 'marc'

from lxml import html
from SPARQLWrapper import SPARQLWrapper, JSON
import requests, json, urlparse, urllib2


def main():
    # scrape wikipedia page with a list of all museums in the Netherlands
    entities_nl = get_entities('http://nl.wikipedia.org/wiki/Lijst_van_musea_in_Nederland',
                               '//div[@id="bodyContent"]/div[@id="mw-content-text"]'
                               '/h3/following-sibling::ul/li/a[not(@rel="nofollow")]/@href')
    entities_adam = get_entities('http://nl.wikipedia.org/wiki/Lijst_van_musea_in_Amsterdam',
                                 '//div[@id="bodyContent"]/div[@id="mw-content-text"]'
                                 '/ul[1]/li/a[not(@rel="nofollow")]/@href')
    museums_urls = entities_adam + entities_nl
    preprocess_entities(museums_urls)


def get_wiki_title(wiki_url):
    museum_page = requests.get('http://nl.wikipedia.org' + wiki_url)
    museum_tree = html.fromstring(museum_page.text)
    museum_entity = museum_tree.xpath('//h1[@id="firstHeading"]/text()')

    if museum_entity[0].startswith('Bezig'):
        parsed = urlparse.urlparse(wiki_url)
        return urlparse.parse_qs(parsed.query)['title'][0]

    return museum_entity[0].replace(' ', '_')


def check_blacklist(wiki_title):
    blacklist = [line.strip() for line in open('config/blacklist.cfg')]
    return wiki_title in blacklist


def get_entities(wiki_page, xpath):
    # scrape wikipedia page with a list of all museums in the Netherlands
    page = requests.get(wiki_page)
    tree = html.fromstring(page.text)
    return tree.xpath(xpath)


def get_geo_data(museum_title):
    # get geo information for museum
    geo_url = "https://maps.googleapis.com/maps/api/geocode/json?address=%s" % museum_title
    geo_response = urllib2.urlopen(geo_url)
    geo_jsongeocode = geo_response.read()
    return json.loads(geo_jsongeocode)


def get_dbpedia_data(museum_title):
    dbpedia_page = 'http://nl.dbpedia.org/resource/' + museum_title

    # Get dbpedia properties for museum
    sparql = SPARQLWrapper("http://nl.dbpedia.org/sparql")
    query = "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> " \
            "SELECT * WHERE { <%s> ?property ?entity }" % dbpedia_page
    sparql.setQuery(query)
    sparql.setReturnFormat(JSON)
    results = sparql.query().convert()
    return results["results"]["bindings"]


def get_hashtag(museum_title):
    return '#%s' % museum_title.replace('_', '')


def create_collections(collection_name, collection):
    with open('data/%s.json' % collection_name, 'w') as outfile:
        json.dump(collection, outfile)


def preprocess_entities(museums_urls):
    museum_titles = []
    museums = []
    hashtags = []
    locations = []
    succeeded = 0
    failed = 0
    duplicated = 0
    blacklisted = 0
    nogeodata = 0

    for url in museums_urls:
        try:
            museum_title = get_wiki_title(url)
            if museum_title in museum_titles:
                duplicated += 1
                print "[duplicated] processed entity: %s" % museum_title
                continue
            if check_blacklist(museum_title):
                blacklisted += 1
                print "[blacklisted] processed entity: %s" % museum_title
                continue
            geo_data = get_geo_data(museum_title)
            if geo_data['status'] != "OK":
                nogeodata += 1
                print "[no geodata] processed entity: %s" % museum_title
                continue

            # Add the results to the right collection
{'fraction_non_alphanumeric': 0.11977715877437325, 'fraction_numerical': 0.006406685236768802, 'mean_word_length': 2.7735155018392015, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5472225', 'n_tokens_mistral': 2372, 'n_tokens_neox': 2303, 'n_words': 443}