<commit_before>import json
from requests import get
from urllib import urlencode

resultCount=1
urlSearch = 'https://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
urlDetail = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&props=labels%7Cinfo%7Csitelinks%7Csitelinks%2Furls%7Cdescriptions%7Cclaims&{query}'
urlMap = 'https://www.openstreetmap.org/?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'

def request(query, params):
    params['url'] = urlSearch.format(query=urlencode({'srsearch': query, 'srlimit': resultCount}))
    return params


def response(resp):
    results = []
    search_res = json.loads(resp.text)

    wikidata_ids = set()
    for r in search_res.get('query', {}).get('search', {}):
        wikidata_ids.add(r.get('title', ''))

    language = resp.search_params['language'].split('_')[0]
    if language == 'all':
        language = 'en'
    url = urlDetail.format(query=urlencode({'ids': '|'.join(wikidata_ids), 'languages': language + '|en'}))

    htmlresponse = get(url)
    jsonresponse = json.loads(htmlresponse.content)
    for wikidata_id in wikidata_ids:
        results = results + getDetail(jsonresponse, wikidata_id, language)

    return results

def getDetail(jsonresponse, wikidata_id, language):
    results = []
    urls = []
    attributes = []

    result = jsonresponse.get('entities', {}).get(wikidata_id, {})

    title = result.get('labels', {}).get(language, {}).get('value', None)
    if title == None:
        title = result.get('labels', {}).get('en', {}).get('value', None)
    if title == None:
        return results

    description = result.get('descriptions', {}).get(language, {}).get('value', None)
    if description == None:
        description = result.get('descriptions', {}).get('en', {}).get('value', '')

    claims = result.get('claims', {})
    official_website = get_string(claims, 'P856', None)
    if official_website != None:
        urls.append({ 'title' : 'Official site', 'url': official_website })
        results.append({ 'title': title, 'url' : official_website })

    wikipedia_link_count = 0
    if language != 'en':
        wikipedia_link_count += add_url(urls, 'Wikipedia (' + language + ')', get_wikilink(result, language + 'wiki'))
    wikipedia_en_link = get_wikilink(result, 'enwiki')
    wikipedia_link_count += add_url(urls, 'Wikipedia (en)', wikipedia_en_link)
    if wikipedia_link_count == 0:
        misc_language = get_wiki_firstlanguage(result, 'wiki')
        if misc_language != None:
            add_url(urls, 'Wikipedia (' + misc_language + ')', get_wikilink(result, misc_language + 'wiki'))
        
    if language != 'en':
        add_url(urls, 'Wiki voyage (' + language + ')', get_wikilink(result, language + 'wikivoyage'))
    add_url(urls, 'Wiki voyage (en)', get_wikilink(result, 'enwikivoyage'))

    if language != 'en':
        add_url(urls, 'Wikiquote (' + language + ')', get_wikilink(result, language + 'wikiquote'))
    add_url(urls, 'Wikiquote (en)', get_wikilink(result, 'enwikiquote'))

    add_url(urls, 'Commons wiki', get_wikilink(result, 'commonswiki'))

    add_url(urls, 'Location', get_geolink(claims, 'P625', None))

    add_url(urls, 'Wikidata', 'https://www.wikidata.org/wiki/' + wikidata_id + '?uselang='+ language)

    musicbrainz_work_id = get_string(claims, 'P435')
    if musicbrainz_work_id != None:
        add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/work/' + musicbrainz_work_id)

    musicbrainz_artist_id = get_string(claims, 'P434')
    if musicbrainz_artist_id != None:
        add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/artist/' + musicbrainz_artist_id)

    musicbrainz_release_group_id = get_string(claims, 'P436')
    if musicbrainz_release_group_id != None:
        add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/release-group/' + musicbrainz_release_group_id)

    musicbrainz_label_id = get_string(claims, 'P966')
    if musicbrainz_label_id != None:
        add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/label/' + musicbrainz_label_id)

    # musicbrainz_area_id = get_string(claims, 'P982')
    # P1407 MusicBrainz series ID
    # P1004 MusicBrainz place ID
    # P1330 MusicBrainz instrument ID
    # P1407 MusicBrainz series ID

    postal_code = get_string(claims, 'P281', None)
    if postal_code != None:
        attributes.append({'label' : 'Postal code(s)', 'value' : postal_code})

    date_of_birth = get_time(claims, 'P569', None)
    if date_of_birth != None:
        attributes.append({'label' : 'Date of birth', 'value' : date_of_birth})

    date_of_death = get_time(claims, 'P570', None)
    if date_of_death != None:
        attributes.append({'label' : 'Date of death', 'value' : date_of_death})

    if len(attributes)==0 and len(urls)==2 and len(description)==0:
        results.append({
                'url': urls[0]['url'],
                'title': title,
                'content': description
                })
    else:
        results.append({
                'infobox' : title,
                'id' : wikipedia_en_link,
                'content' : description,
                'attributes' : attributes,
                'urls' : urls
                })

    return results


def add_url(urls, title, url):
    if url != None:
        urls.append({'title' : title, 'url' : url})
        return 1
    else:
        return 0

def get_mainsnak(claims, propertyName):
    propValue = claims.get(propertyName, {})
    if len(propValue) == 0:
        return None

    propValue = propValue[0].get('mainsnak', None)
    return propValue


def get_string(claims, propertyName, defaultValue=None):
    propValue = claims.get(propertyName, {})
    if len(propValue) == 0:
        return defaultValue

    result = []
    for e in propValue:
        mainsnak = e.get('mainsnak', {})

        datavalue = mainsnak.get('datavalue', {})
        if datavalue != None:
            result.append(datavalue.get('value', ''))

    if len(result) == 0:
        return defaultValue
    else:
        return ', '.join(result)
<commit_msg>[fix] wikidata: using only the first url<commit_after>import json
from requests import get
from urllib import urlencode

resultCount=1
urlSearch = 'https://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
urlDetail = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&props=labels%7Cinfo%7Csitelinks%7Csitelinks%2Furls%7Cdescriptions%7Cclaims&{query}'
urlMap = 'https://www.openstreetmap.org/?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'

def request(query, params):
    params['url'] = urlSearch.format(query=urlencode({'srsearch': query, 'srlimit': resultCount}))
    return params


def response(resp):
    results = []
    search_res = json.loads(resp.text)

    wikidata_ids = set()
    for r in search_res.get('query', {}).get('search', {}):
        wikidata_ids.add(r.get('title', ''))

    language = resp.search_params['language'].split('_')[0]
    if language == 'all':
        language = 'en'
    url = urlDetail.format(query=urlencode({'ids': '|'.join(wikidata_ids), 'languages': language + '|en'}))

    htmlresponse = get(url)
    jsonresponse = json.loads(htmlresponse.content)
    for wikidata_id in wikidata_ids:
        results = results + getDetail(jsonresponse, wikidata_id, language)

    return results

def getDetail(jsonresponse, wikidata_id, language):
    results = []
    urls = []
    attributes = []

    result = jsonresponse.get('entities', {}).get(wikidata_id, {})

    title = result.get('labels', {}).get(language, {}).get('value', None)
    if title == None:
        title = result.get('labels', {}).get('en', {}).get('value', None)
    if title == None:
        return results

    description = result.get('descriptions', {}).get(language, {}).get('value', None)
    if description == None:
        description = result.get('descriptions', {}).get('en', {}).get('value', '')

    claims = result.get('claims', {})
    official_website = get_string(claims, 'P856', None)
    if official_website != None:
        urls.append({ 'title' : 'Official site', 'url': official_website })
        results.append({ 'title': title, 'url' : official_website })

    wikipedia_link_count = 0
    if language != 'en':
        wikipedia_link_count += add_url(urls, 'Wikipedia (' + language + ')', get_wikilink(result, language + 'wiki'))
    wikipedia_en_link = get_wikilink(result, 'enwiki')
    wikipedia_link_count += add_url(urls, 'Wikipedia (en)', wikipedia_en_link)
    if wikipedia_link_count == 0:
        misc_language = get_wiki_firstlanguage(result, 'wiki')
        if misc_language != None:
            add_url(urls, 'Wikipedia (' + misc_language + ')', get_wikilink(result, misc_language + 'wiki'))

    if language != 'en':
        add_url(urls, 'Wiki voyage (' + language + ')', get_wikilink(result, language + 'wikivoyage'))
    add_url(urls, 'Wiki voyage (en)', get_wikilink(result, 'enwikivoyage'))

    if language != 'en':
        add_url(urls, 'Wikiquote (' + language + ')', get_wikilink(result, language + 'wikiquote'))
    add_url(urls, 'Wikiquote (en)', get_wikilink(result, 'enwikiquote'))

    add_url(urls, 'Commons wiki', get_wikilink(result, 'commonswiki'))

    add_url(urls, 'Location', get_geolink(claims, 'P625', None))

    add_url(urls, 'Wikidata', 'https://www.wikidata.org/wiki/' + wikidata_id + '?uselang='+ language)

    musicbrainz_work_id = get_string(claims, 'P435')
    if musicbrainz_work_id != None:
        add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/work/' + musicbrainz_work_id)

    musicbrainz_artist_id = get_string(claims, 'P434')
    if musicbrainz_artist_id != None:
        add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/artist/' + musicbrainz_artist_id)

    musicbrainz_release_group_id = get_string(claims, 'P436')
    if musicbrainz_release_group_id != None:
        add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/release-group/' + musicbrainz_release_group_id)

    musicbrainz_label_id = get_string(claims, 'P966')
    if musicbrainz_label_id != None:
        add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/label/' + musicbrainz_label_id)

    # musicbrainz_area_id = get_string(claims, 'P982')
    # P1407 MusicBrainz series ID
    # P1004 MusicBrainz place ID
    # P1330 MusicBrainz instrument ID
    # P1407 MusicBrainz series ID

    postal_code = get_string(claims, 'P281', None)
    if postal_code != None:
        attributes.append({'label' : 'Postal code(s)', 'value' : postal_code})

    date_of_birth = get_time(claims, 'P569', None)
    if date_of_birth != None:
        attributes.append({'label' : 'Date of birth', 'value' : date_of_birth})

    date_of_death = get_time(claims, 'P570', None)
    if date_of_death != None:
        attributes.append({'label' : 'Date of death', 'value' : date_of_death})

    if len(attributes)==0 and len(urls)==2 and len(description)==0:
        results.append({
                'url': urls[0]['url'],
                'title': title,
                'content': description
                })
    else:
        results.append({
                'infobox' : title,
                'id' : wikipedia_en_link,
                'content' : description,
                'attributes' : attributes,
                'urls' : urls
                })

    return results


def add_url(urls, title, url):
    if url != None:
        urls.append({'title' : title, 'url' : url})
        return 1
    else:
        return 0

def get_mainsnak(claims, propertyName):
    propValue = claims.get(propertyName, {})
    if len(propValue) == 0:
        return None

    propValue = propValue[0].get('mainsnak', None)
    return propValue


def get_string(claims, propertyName, defaultValue=None):
    propValue = claims.get(propertyName, {})
    if len(propValue) == 0:
        return defaultValue

    result = []
    for e in propValue:
        mainsnak = e.get('mainsnak', {})

        datavalue = mainsnak.get('datavalue', {})
        if datavalue != None:
            result.append(datavalue.get('value', ''))

    if len(result) == 0:
        return defaultValue
    else:
        #TODO handle multiple urls
        return result[0]

{'fraction_non_alphanumeric': 0.1501467231822628, 'fraction_numerical': 0.011004238669709815, 'mean_word_length': 3.405385996409336, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6507550', 'n_tokens_mistral': 4309, 'n_tokens_neox': 4066, 'n_words': 802}