<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
import json
import csv              # CSV file manipulations


def api_call(titles):
    titles_list = ('|').join(titles)
    print(titles_list)

    api_base = "https://wikidata.org/w/api.php"

    params = [('format', 'json'),
              ('action', 'wbgetentities'),
              ('props', 'info|claims|sitelinks|labels'),
              ('sites', 'frwiki'),
              ('sitefilter', 'frwiki'),
              ('languages', 'fr'),
              ('titles', titles_list)]

    try:
        response = requests.get(api_base, params=params)
        results = json.loads(response.text)
        return results

    except 'ConnectionError':
        print('Server unreachable')

communes = []
with open('resources/extract-osm-fixed.csv', 'r') as csv_osm_extract:
    reader = csv.DictReader(csv_osm_extract)
    for row in reader:
        communes.append((row['lien-wikipedia'].strip(), {
            'insee': row['insee'].strip(),
            'titre': row['titre'].strip(),
            'qid': ''
            }))
csv_osm_extract.closed

communes_dict = dict(communes)

# print(communes)

num_communes = len(communes)
chunk_size = 50
# num_chunks = math.ceil(num_communes/chunk_size)

last = 0
missing_insee = []
mismatched_insee = []
missing_labels = []
mismatched_labels = []
wrong_titles = []

while last < num_communes:
    chunk = communes[last:last + chunk_size]

    titles = []
    for commune in chunk:
        title = commune[0].split(':')[1]
        titles.append(title)

    api_results = api_call(titles)

    entities = api_results['entities']
    for qid, data in entities.items():
        print('Fetching data for item {}'.format(qid))
        if qid[0] != 'Q':
            wrong_titles.append(data['title'])
            print('Wrong title: {}'.format(data['title']))
        else:
            wp_title = 'fr:' + data['sitelinks']['frwiki']['title']
            wd_label = data['labels']['fr']['value']
            claims = data['claims']
            try:
                wd_insee = claims['P374'][0]['mainsnak']['datavalue']['value']
            except:
                print('No insee claim for item {}'.format(qid))

            if wd_insee:
                dict_insee = communes_dict[wp_title]['insee']
                if dict_insee != wd_insee:
                    mismatched_insee.append((qid,
                                             wp_title,<commit_msg>fixed a bug setting the wrong insee code<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
import json
import csv              # CSV file manipulations


def api_call(titles):
    titles_list = ('|').join(titles)
    print(titles_list)

    api_base = "https://wikidata.org/w/api.php"

    params = [('format', 'json'),
              ('action', 'wbgetentities'),
              ('props', 'info|claims|sitelinks|labels'),
              ('sites', 'frwiki'),
              ('sitefilter', 'frwiki'),
              ('languages', 'fr'),
              ('titles', titles_list)]

    try:
        response = requests.get(api_base, params=params)
        results = json.loads(response.text)
        return results

    except 'ConnectionError':
        print('Server unreachable')

communes = []
with open('resources/extract-osm-fixed.csv', 'r') as csv_osm_extract:
    reader = csv.DictReader(csv_osm_extract)
    for row in reader:
        communes.append((row['lien-wikipedia'].strip(), {
            'insee': row['insee'].strip(),
            'titre': row['titre'].strip(),
            'qid': ''
            }))
csv_osm_extract.closed

communes_dict = dict(communes)

# print(communes)

num_communes = len(communes)
chunk_size = 50
# num_chunks = math.ceil(num_communes/chunk_size)

last = 0
missing_insee = []
mismatched_insee = []
missing_labels = []
mismatched_labels = []
wrong_titles = []

while last < num_communes:
    chunk = communes[last:last + chunk_size]

    titles = []
    for commune in chunk:
        title = commune[0].split(':')[1]
        titles.append(title)

    api_results = api_call(titles)

    entities = api_results['entities']
    for qid, data in entities.items():
        print('Fetching data for item {}'.format(qid))
        if qid[0] != 'Q':
            wrong_titles.append(data['title'])
            print('Wrong title: {}'.format(data['title']))
        else:
            wp_title = 'fr:' + data['sitelinks']['frwiki']['title']
            wd_label = data['labels']['fr']['value']
            claims = data['claims']
            try:
                wd_insee = claims['P374'][0]['mainsnak']['datavalue']['value']
            except:
                wd_insee = ''
                print('No insee claim for item {}'.format(qid))

            if wd_insee:
                dict_insee = communes_dict[wp_title]['insee']
                if dict_insee != wd_insee:
                    mismatched_insee.append((qid,
                                             wp_title,
{'fraction_non_alphanumeric': 0.1504318136171922, 'fraction_numerical': 0.004820245029122314, 'mean_word_length': 2.24853228962818, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5259929', 'n_tokens_mistral': 1571, 'n_tokens_neox': 1530, 'n_words': 295}