<jupyter_start><jupyter_text>Extract data from Wikipedia Infobox Templates into csv file Overview* list_template_usage: takes Site object and Template name and returns a Generator* extract_template_data: takes a Generator, a Template name, a Field name and returns a List<jupyter_code>import pywikibot
import re

site = pywikibot.Site("en", 'wikipedia')
import pprint

def prettyPrint(variable):
    pp = pprint.PrettyPrinter(indent=0)
    pp.pprint(variable)
def template_name_replace_underscores(template_name):
    return template_name.replace('_', ' ')
from pywikibot import pagegenerators as pg

def list_template_usage(site_obj, template_name):
    """
    Takes Site object and template name and returns a generator.

    The function expects a Site object (pywikibot.Site()) and
    a template name (String). It creates a list of all
    pages using that template and returns them as a generator.
    The generator will load 50 pages at a time for iteration.
    """
    template_name = template_name_replace_underscores(template_name)
    name = "{}:{}".format(site_obj.namespace(10), template_name)
    tmpl_page = pywikibot.Page(site_obj, name)
    ref_gen = tmpl_page.getReferences(follow_redirects=False)
    filter_gen = pg.NamespaceFilterPageGenerator(ref_gen, namespaces=[0])
    generator = site_obj.preloadpages(filter_gen, pageprops=True)
    return generator
def convert_field(field):
    #convert template in feet
    match = re.search(r"{{(c|C)onvert\|\d+\.?\d*\|(ft|feet)(\||})", field)
    if match:
        ft = re.search(r"\d+\.?\d*", match[0])[0]
        return(ft,'ft')
    #convert template in metre
    match = re.search(r"{{(c|C)onvert\|\d+\.?\d*\|m(\||})", field)
    if match:
        metre = re.search(r"\d+\.?\d*", match[0])[0]
        return(metre,'m')
    #convert template in mm
    match = re.search(r"{{(c|C)onvert\|\d+\.?\d*\|mm(\||})", field)
    if match:
        mm = re.search(r"\d+\.?\d*", match[0])[0]
        return(mm,'mm')
    #convert template in kg
    match = re.search(r"{{(c|C)onvert\|\d+\.?\d*\|kg(\||})", field)
    if match:
        kg = re.search(r"\d+\.?\d*", match[0])[0]
        return(kg,'kg')
    #convert template in lb
    match = re.search(r"{{(c|C)onvert\|\d+\.?\d*\|lb(\||})", field)
    if match:
        lb = re.search(r"\d+\.?\d*", match[0])[0]
        return(lb,'lb')
    #convert template in sqft
    match = re.search(r"{{(c|C)onvert\|\d+\.?\d*\|sqft(\||})", field)
    if match:
        sqft = re.search(r"\d+\.?\d*", match[0])[0]
        return(sqft,'sqft')    
    #convert template in m2
    match = re.search(r"{{(c|C)onvert\|\d+\.?\d*\|m2(\||})", field)
    if match:
        m2 = re.search(r"\d+\.?\d*", match[0])[0]
        return(m2,'m2')  
    #field in metre
    match = re.search(r"\d+\.?\d*m|\d+\.?\d*\sm", field)
    if match:
        metre = re.search(r"\d+\.?\d*", match[0])[0]
        return(metre,'m')
    #field in ft
    match = re.search(r"\d+\.?\d*ft|\d+\.?\d*\sft", field)
    if match:
        ft = re.search(r"\d+\.?\d*", match[0])[0]
        return(ft,'ft')
    #field wikipedia entry -> wikidata qid
    match = re.search(r"(\[\[.*\||\[\[.*\]\])", field)
    if match:
        article = match[0].replace("[[","").replace("]]","")
        page = pywikibot.Page(site, article)
        qid = pywikibot.ItemPage.fromPage(page).id
        return("",qid)
    else:
        return(field,"")


print(convert_field("{{convert|9.15|m|abbr=on}}"))
print(convert_field("{{Convert|36|ft|m|abbr=on}}"))
print(convert_field("5.79m"))
print(convert_field("{{convert|4915|mm|abbr=on}}"))
print(convert_field("60ft"))
print(convert_field("LOA is several meters"))
print(convert_field("{{convert|58|ft}}"))
print(convert_field("{{convert|50|feet|abbr=on}}"))
print(convert_field("[[Bruce Farr]]"))
print(convert_field("[[Bruce Farr|BF]]"))
print(convert_field("{{convert|225|lb|kg|0|abbr=on}}"))
print(convert_field("{{convert|180|kg|lb|abbr=on}}"))

def extract_template_data(generator, template_name, field_name):
    """
    Takes a Generator, a Template name, a Field name and returns a List

    The function can also take a header (list of strings) that will be
    the headers of the table (Needs to be the same dimension as the table).
    The first column needs to be a link to property (It will be made into
    a link. In this example the second column is a list of links to Q-items.
    """
    template_name = template_name_replace_underscores(template_name)
    template_data = []
    header = ["qid", "url", field_name, "unit"]
    template_data.append(header)
 
    for page in generator:
        try:
            qid = pywikibot.ItemPage.fromPage(page).id
        except:
            print('no qid')
            qid = 'no qid'
        page_url = page.full_url()
        tmpl_list = page.raw_extracted_templates
        for tmpl in tmpl_list:
            if template_name in tmpl:
                try:
                    dico=tmpl[1]
                    (field,unit)=convert_field(dico[field_name])
                    print("qid: %s, url: %s | %s: %s %s" % (qid, page_url, field_name, field, unit))
                    template_data.append([qid, page_url, field, unit])
                except:
                    print('no such field')
                    
    return template_data
import csv

def list2csv(filename, list_name):
    """
    Takes a List and write a csv file
    """
    with open(filename, "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerows(list_name)
template_name = "Infobox_sailboat_specifications"
field_name = "ballast"

#template_name = "Infobox sailing yacht"
#field_name = "length"

filename = template_name + "." + field_name + ".csv"
tmpl_gen = list_template_usage(site, template_name)
template_data = extract_template_data(tmpl_gen, template_name, field_name)
list2csv(filename, template_data)<jupyter_output><empty_output><jupyter_text>Push to Wikidata<jupyter_code>def push2wikidata(statement_property, statement_value, statement_unit):
    if exist(statement_property, statement_value, statement_unit):
        print('statement already exist')
    else:
        create(statement_property, statement_value, statement_unit)

def exist(statement_property, statement_value, statement_unit):
    return('true')

def create(statement_property, statement_value, statement_unit):
    return('true')


#item_id = 'Q97179551' #J/92s
item_id = 'Q930962' #Etchells
statement_property = 'P2067'
statement_value = 2822
statement_unit = 'lb'
qualifier_id = 'P642'




push2wikidata(statement_property, statement_value, statement_unit)
from wikibaseintegrator import wbi_core

my_first_wikidata_item = wbi_core.ItemEngine(item_id)

# to check successful installation and retrieval of the data, you can print the json representation of the item
prettyPrint(my_first_wikidata_item.get_json_representation()['claims'][statement_property][0]['qualifiers'][qualifier_id][0])
#https://www.wikidata.org/wiki/Wikidata:Pywikibot_-_Python_3_Tutorial/Setting_statements
#Setting statements
# -*- coding: utf-8  -*-
import pywikibot
"""
First check if P2373 already exists for Q15935. If not add claim P2373 with the string 'Kanye_west'.
This works for claims that only allow a single value.
"""
site_data = pywikibot.Site("wikidata", "wikidata")
repo_data = site_data.data_repository()
item_data = pywikibot.ItemPage(repo_data, "Q97179551")
claims = item_data.get('claims') #Get all the existing claims
item_data.get() #Fetch all page data, and cache it.
claims['claims']
statement_property = 'P2067'
statement_value = 2822
statement_unit = 'lb'


if statement_property in claims[u'claims']: #if a value is present already, print error.
    pywikibot.output(u'Error: statement already exists!')
else:
    stringclaim = pywikibot.Claim(repo_data, statement_property) #Else, add the value
    stringclaim.setTarget(statement_value)
    item_data.addClaim(stringclaim, summary=u'adding statement via bot')
claims['P2067']
# -*- coding: utf-8  -*-
import pywikibot
"""
Adding a qualifier to existing claims/statements
"""
site_data = pywikibot.Site("wikidata", "wikidata")
repo_data = site_data.data_repository()
item_data = pywikibot.ItemPage(repo_data, "Q97179551")
statement_property = 'P2067' # 'P2067' mass
statement_qualifier = 'P642' # 'P642' of
statement_qualifier_target = 'Q5636358' # 'Q5636358' displacement

statement_value = 2822
statement_unit = 'lb'


item_data.get() #Fetch all page data, and cache it.



for claim in item_data.claims[statement_property]: #Finds all statements with property (P)
    if statement_qualifier not in claim.qualifiers: #If not already exist
        print('statement_qualifier not found')
    else:
        for values in claim.qualifiers[statement_qualifier]:
            if statement_qualifier_target == values.target.id:
                print('statement_qualifier_target already defined')
            else:
                print('statement_qualifier_target not found')

        # Generate QUALIFIER FOR EXISTENCE STATEMENTS/CLAIMS
        #qualifier = pywikibot.Claim(repo, u'P642')
        #target = pywikibot.ItemPage(repo, "Q35409")
        #qualifier.setTarget(target)

        #claim.addQualifier(qualifier, summary=u'Adding a qualifier.') #Adding qualifier to all statements (P131)
item_data.claims['P2067'][0].qualifiers['P642'][0].target
prettyPrint(item_data.claims['P2067'])<jupyter_output>[   Claim.fromJSON(DataSite("wikidata", "wikidata"), {'mainsnak': {'snaktype': 'value', 'property': 'P2067', 'datatype': 'quantity', 'datavalue': {'value': {'amount': '+2550', 'upperBound': None, 'lowerBound': None, 'unit': 'http://www.wikidata.org/entity/Q11570'}, 'type': 'quantity'}}, 'type': 'statement', 'id': 'Q97179551$2cb0f681-4bcd-ede2-a147-7aa352b7ca64', 'rank': 'normal', 'qualifiers': {'P642': [{'snaktype': 'value', 'property': 'P642', 'datatype': 'wikibase-item', 'datavalue': {'value': {'entity-type': 'item', 'numeric-id': 5636358}, 'type': 'wikibase-entityid'}, 'hash': '9b35e18a0b409cbb243dcafd61faca309a7677bc'}]}, 'qualifiers-order': ['P642'], 'references': [{'snaks': {'P854': [{'snaktype': 'value', 'property': 'P854', 'datatype': 'url', 'datavalue': {'value': 'https://jboats.com/j92s-tech-specs', 'type': 'string'}}], 'P813': [{'snaktype': 'value', 'property': 'P813', 'datatype': 'time', 'datavalue': {'value': {'time': '+00000002021-03-30T00:00:00Z', 'precision': 11, 'after[...]<jupyter_text>Alternative<jupyter_code>def strip_dict_entries(dict_in):
    """Strip whitespace from all keys and (string) values in a dictionary."""
    dict_out = dict()
    if not isinstance(dict_in, dict):
        raise MyError('strip_dict_entries() expects a dictionary object'
                      'as input but found "%s"' % type(dict_in).__name__)
    for k, v in dict_in.items():
        if isinstance(v,str):
            v = v.strip()
        dict_out[k.strip()] = v
    return dict_out
def extract_template_data2(generator, template_name, field_name, header=None):
    """
    Takes a Generator, a Template name, a Field name and returns a List

    The function can also take a header (list of strings) that will be
    the headers of the table (Needs to be the same dimension as the table).
    The first column needs to be a link to property (It will be made into
    a link. In this example the second column is a list of links to Q-items.
    """
    template_data = []
    result = []

    if header != None:
        template_data.append(header)

    for page in generator:                    
        templates = pywikibot.textlib.extract_templates_and_params(page.text)
        for tp in templates:
            if tp[0] == template_name:
                #print(tp[1])
                result.append(strip_dict_entries(tp[1]))
            
    return result
result
def get_all_template_entries(wikitext, template_name):
    """Return a list of all arguments for instances of a given template."""
    templates = pywikibot.textlib.extract_templates_and_params(wikitext)
    result = []
    for tp in templates:
        if tp[0] == template_name:
            print(tp[1])
            result.append(strip_dict_entries(tp[1]))
    return result
toto = get_all_template_entries(page.text, template_name)
toto
import mwparserfromhell
import pywikibot

def parse(title):
    site = pywikibot.Site()
    page = pywikibot.Page(site, title)
    text = page.get()
    return mwparserfromhell.parse(text)


for page in tmpl_gen:
    item = pywikibot.ItemPage.fromPage(page)
    print(page.title(), item.getID())
    print(page)
type(page)
templates = page.templatesWithParams()
(template_page,item)=templates[0]
print(type(item))
item
templates_raw = page.raw_extracted_templates
(template_0, item_0)=templates_raw[0]
print(type(item_0))
item_0
item_0["loa"]
for (template, fielddict) in templates:
            # Clean up template
            try:
                template = pywikibot.Page(page.site, template,
                                          ns=10).title(with_ns=False)
            except pywikibot.exceptions.InvalidTitle:
                pywikibot.error(
                    "Failed parsing template; '{}' should be "
                    'the template name.'.format(template))
                continue
            # We found the template we were looking for
            for field, value in fielddict.items():
                field = field.strip()
for (template, fielddict) in templates:
    print(template)
    print(fielddict)
fielddict
(template, fielddict)=templates[0]
fielddict
(field, value) = fielddict.items(2)
import pywikibot
from pywikibot import pagegenerators as pg

def list_template_usage(site_obj, template_name):
    """
    Takes Site object and template name and returns a generator.

    The function expects a Site object (pywikibot.Site()) and
    a template name (String). It creates a list of all
    pages using that template and returns them as a generator.
    It only returns pages in the 121-namespace (property-talk-pages).
    The generator will load 50 pages at a time for iteration.
    """
    name = "{}:{}".format(site.namespace(10), template_name)
    tmpl_page = pywikibot.Page(site, name)
    ref_gen = pg.ReferringPageGenerator(tmpl_page, onlyTemplateInclusion=True)
    filter_gen = pg.NamespaceFilterPageGenerator(ref_gen, namespaces=[10])
    generator = site.preloadpages(filter_gen, pageprops=True)
    return generator
site = pywikibot.Site("en", 'wikipedia')
tmpl_gen = list_template_usage(site, "Infobox_sailboat_specifications")<jupyter_output><empty_output><jupyter_text>code for single page<jupyter_code>import pywikibot

#template_name = "Infobox_sailboat_specifications"
#field_name = "loa"
template_name = "Infobox sailing yacht"
field_name = "designer"
filename = template_name + "." + field_name + ".csv"

site = pywikibot.Site("en", "wikipedia")
#page = pywikibot.Page(site, "ETAP_32s")
#page = pywikibot.Page(site, "Finn (dinghy)")
#page = pywikibot.Page(site, "49er (dinghy)")
#page = pywikibot.Page(site, "Squib (keelboat)")
#page = pywikibot.Page(site, "Hobie 16")
page = pywikibot.Page(site, "Maiden_%28yacht%29")
page_url = page.full_url()
qid = pywikibot.ItemPage.fromPage(page).id
print(qid)
tmpl_list = page.raw_extracted_templates
template_name = template_name_replace_underscores(template_name)
for tmpl in tmpl_list:
    if template_name in tmpl:
        try:
            dico=tmpl[1]
            field=dico[field_name]
            print("qid: %s, page: %s | %s: %s" % (qid, page_url, field_name, field))
            template_data.append([qid, page_url, field])
        except:
            print('no such field')
print(convert_field("{{convert|58|ft}}"))
field
type(field)
page = pywikibot.Page(site, field)
item = pywikibot.ItemPage.fromPage(page)

print(item)<jupyter_output><empty_output><jupyter_text>Using WikidataIntegrator* https://github.com/SuLab/WikidataIntegrator* https://github.com/SuLab/scheduled-bots/blob/SARS-CoV-2/scheduled_bots/geneprotein/VirusGeneBot.py* https://github.com/SuLab/scheduled-bots/blob/master/scheduled_bots/genedisease/GeneDiseaseBot.py* https://github.com/SuLab/scheduled-bots/blob/master/scheduled_bots/civic/bot.py* https://github.com/SuLab/scheduled-bots/blob/master/scheduled_bots/iedb/bot.py<jupyter_code>!python3.6 -m pip install --upgrade pip
!pip install tqdm
!pip install wikidataintegrator
!pip install shexer
import os
import sys
from tqdm import tqdm
import pandas as pd
from datetime import datetime
sys.path.insert(0, "/home/gstupp/projects/WikidataIntegrator")
from wikidataintegrator import wdi_core, wdi_login, wdi_helpers, wdi_property_store
# create a file called local.py with your credentials
# from local import WDUSER, WDPASS<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13724902607132156, 'fraction_numerical': 0.020677255019478574, 'mean_word_length': 3.8421358096343585, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 7, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '591888', 'license': 'CC0-1.0', 'n_tokens_mistral': 5840, 'n_tokens_neox': 5529, 'n_words': 1305, 'path': 'infobox2csv.ipynb', 'repo_name': 'simonletort/Sailbot', 'size': '16685'}