<jupyter_start><jupyter_text>Template code for interacting with WikidataThis notebook is a collection of functions and code blocks to use when interacting with Wikidata or other instances of Wikibase.<jupyter_code># (c) 2021 Vanderbilt University. 
# This program is released under a GNU General Public License v3.0 http://www.gnu.org/licenses/gpl-3.0
# Author: <NAME>
# 2021-01-17

import requests
import json
import csv
import sys # Read CLI arguments
import datetime
from pathlib import Path
#import os
#from time import sleep

# Use the following code for a stand-alone script if you want to pass in a value (e.g. file path) when running
# the script from the command line. If no arguments are passed, the "else" value will be used.

if len(sys.argv) == 2: # if exactly one argument passed (i.e. the configuration file path)
    file_path = sys.argv[1] # sys.argv[0] is the script name
else:
    file_path = 'file.csv'


# ----------------
# File IO
# ----------------

# Many functions operate on a list of dictionaries, where each item in the list represents a spreadsheet row
# and each column is identified by a dictionary item whose key is the column header in the spreadsheet.
# The first two functions read and write from files into this data structure.

# Read from a CSV file into a list of dictionaries
def read_dicts_from_csv(filename):
    with open(filename, 'r', newline='', encoding='utf-8') as file_object:
        dict_object = csv.DictReader(file_object)
        array = []
        for row in dict_object:
            array.append(row)
    return array

# Write a list of dictionaries to a CSV file
# The fieldnames object is a list of strings whose items are the keys in the row dictionaries that are chosen
# to be the columns in the output spreadsheet. The order in the list determines the order of the columns.
# To determine the field names from the first dict in the list, use this code:
'''
fieldnames = table[0].keys()
'''
# The column headers of the output will be in the order in which they occur in the dict (usually the order they were added)
def write_dicts_to_csv(table, filename, fieldnames):
    with open(filename, 'w', newline='', encoding='utf-8') as csv_file_object:
        writer = csv.DictWriter(csv_file_object, fieldnames=fieldnames)
        writer.writeheader()
        for row in table:
            writer.writerow(row)


# If configuration or other data are stored in a file as JSON, this function loads them into a Python data structure

# Load JSON file data from local drive into a Python data structure
def load_json_into_data_struct(path):
    with open(path, 'rt', encoding='utf-8') as file_object:
        file_text = file_object.read()
    structure = json.loads(file_text)
    # uncomment the following line to view the data
    # print(json.loads(structure, indent = 2))
    return(structure)

# Load JSON file data from GitHub into a Python data structure
# NOTE: requires the requests module to be installed!
# raw_file_url must be the URL of the raw file, not the web page about the file
def load_github_json_into_data_struct(raw_file_url):
    response_object = requests.get(raw_file_url)
    file_text = response_object.text
    structure = json.loads(file_text)
    # uncomment the following line to view the data
    # print(json.loads(structure, indent = 2))
    return(structure)

# This function will load some credential from a text file, either in the home directory or current working directory
# The value of the directory variable should be either 'home' or 'working'
# Keeping the credential in the home directory prevents accidentally uploading it with the notebook.
# The function returns a single string, so if there is more than one credential (e.g. key plus secret), additional
# parsing of the return value may be required. 
def load_credential(filename, directory):
    cred = ''
    # to change the script to look for the credential in the working directory, change the value of home to empty string
    if directory == 'home':
        home = str(Path.home()) #gets path to home directory; works for both Win and Mac
        credential_path = home + '/' + filename
    else:
        directory = 'working'
        credential_path = filename
    try:
        with open(credential_path, 'rt', encoding='utf-8') as file_object:
            cred = file_object.read()
    except:
        print(filename + ' file not found - is it in your ' + directory + ' directory?')
        exit()
    return(cred)

# ----------------
# Code for interacting with a Wikibase query interface (SPARQL endpoint). Typically, it's the Wikidata Query Service
# ----------------

endpoint = 'https://query.wikidata.org/sparql'
accept_media_type = 'application/json'
# Replace this value with your own user agent header string
user_agent_header = 'VanderBot/1.7 (https://github.com/HeardLibrary/linked-data/tree/master/vanderbot; mailto:<EMAIL>)'

# The following code generates a request header dictionary suitable for sending to a SPARQL endpoint.
# If the query is SELECT, use the JSON media type above. For CONSTRUCT queryies use text/turtle to get RDF/Turtle
# Best to send a user-agent header because some Wikimedia servers don't like unidentified clients

# 2021-03-31 note: to get UTF-8 text in the SPARQL queries to work properly, send URL-encoded text rather than raw text
# in the POST operation
def generate_header_dictionary(accept_media_type,user_agent_header):
    request_header_dictionary = {
        'Accept' : accept_media_type,
#        'Content-Type': 'application/sparql-query',
        'Content-Type': 'application/x-www-form-urlencoded',
        'User-Agent': user_agent_header
    }
    return request_header_dictionary

# The following function requires the request header generated above
sparql_request_header = generate_header_dictionary(accept_media_type,user_agent_header)
# The query is a valid SPARQL query string

# Sends a query to the query service endpoint. 
# NOTE: request_header and endpoint are global variables defined earlier in the script
def send_sparql_query(query_string):
    # You can delete the two print statements if the queries are short. However, for large/long queries,
    # it's good to let the user know what's going on.
    print('querying SPARQL endpoint to acquire item metadata')
    #response = requests.post(endpoint, data=query_string.encode('utf-8'), headers=sparql_request_header)
    response = requests.post(endpoint, data=dict(query=query_string), headers=sparql_request_header) # use URL-encoded method
    #print(response.text) # uncomment to view the raw response, e.g. if you are getting an error
    data = response.json()

    # Extract the values from the response JSON
    results = data['results']['bindings']
    
    print('done retrieving data')
    # print(json.dumps(results, indent=2))
    return results

# ----------------
# Utility code
# ----------------

# Generate the current UTC xsd:date
def generate_utc_date():
    whole_time_string_z = datetime.datetime.utcnow().isoformat() # form: 2019-12-05T15:35:04.959311
    date_z = whole_time_string_z.split('T')[0] # form 2019-12-05
    return date_z

# Extracts the local name part of an IRI, e.g. a qNumber from a Wikidata IRI
def extract_local_name(iri):
    # pattern is http://www.wikidata.org/entity/Q6386232
    pieces = iri.split('/')
    last_piece = len(pieces)
    return pieces[last_piece - 1]

# Extracts the UUID and qId from a statement IRI and returns them as a tuple
def extract_statement_uuid(iri):
    # pattern is http://www.wikidata.org/entity/statement/Q7552806-8B88E0CA-BCC8-49D5-9AC2-F1755464F1A2
    pieces = iri.split('/')
    statement_id = pieces[5]
    pieces = statement_id.split('-')
    # UUID is the first item of the tuple, Q ID is the second item
    return pieces[1] + '-' + pieces[2] + '-' + pieces[3] + '-' + pieces[4] + '-' + pieces[5], pieces[0]

# To sort a list of dictionaries by a particular dictionary key's values, define the following function
# then invoke the sort using the code that follows

# function to use in sort
def sort_funct(row):
    return row['filename'] # sort by the filename key

'''
output_list.sort(key = sort_funct) # sort by the filename field
'''<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09140568099053169, 'fraction_numerical': 0.013231366836610827, 'mean_word_length': 3.689243027888446, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '476775', 'license': 'CC0-1.0', 'n_tokens_mistral': 2416, 'n_tokens_neox': 2314, 'n_words': 949, 'path': 'code/wikidata/template.ipynb', 'repo_name': 'HeardLibrary/digital-scholarship', 'size': '8266'}