<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# This script retrieves data about the use of classes and properties
# on Wikidata from SPARQL. The results are processed and stored in the
# files properties.json and classes.json. The script must be run in a
# directory that already contains these files (possibly with no content
# other than an empty map {}). Normally, the files are first generated
# by the Java export tool, which provides some statistics that cannot
# be obtained in an acceptable time from SPARQL.

import requests
import json
import os
import time
#import pprint

SPARQL_SERVICE_URL = 'https://query.wikidata.org/sparql'

def doSparqlQuery(query):
	r = requests.get(SPARQL_SERVICE_URL, params={'query': query, 'format': 'json'});
	return json.loads(r.text)

def sparqlQuery(query):
	try:
		return doSparqlQuery(query)
	except ValueError:
		print "SPARQL query failed, possibly because of a time out. Waiting for 60 sec ...\n"
		time.sleep(60)
		print "Retrying query ...\n"
		try:
			return doSparqlQuery(query)
		except ValueError:
			print "Failed to get SPARQL results. Giving up."
			return json.loads('{ results: { bindings: {} }')


def setPropertyStatistics(propertyStatistics, propertyId, statisticType, numberString):
	try:
		if propertyId in propertyStatistics:
			propertyStatistics[propertyId][statisticType] = int(numberString)
	except ValueError:
		print "Error reading count value " + numberString + ": not an integer number"


def storeStatistics(key, value):
	with open('statistics.json', 'r') as outfile:
		data = json.load(outfile)
		data[key] = value
		with open('statistics.json', 'w') as outfile:
			json.dump(data, outfile)

def updateClassRecords() :
	print "Fetching class ids and labels for classes with direct instances ..."
	startTime = time.strftime("%Y-%m-%dT%H:%M:%S")

	resultsClasses = sparqlQuery("""SELECT ?cl ?clLabel ?c
	WHERE {
	{ SELECT ?cl (count(*) as ?c) WHERE { ?i wdt:P31 ?cl } GROUP BY ?cl }
	SERVICE wikibase:label {
			bd:serviceParam wikibase:language "en" .
	}
	}""")

	print "Augmenting current class data ..."
	classesWithInstances = {}
	with open('classes.json') as classFile:    
		data = json.load(classFile)
		for binding in resultsClasses['results']['bindings']:
			classUri = binding['cl']['value']
			if classUri[0:31] == 'http://www.wikidata.org/entity/':
				classId = classUri[32:]
				classesWithInstances[classId] = True
				classLabel = binding['clLabel']['value']
				classInstances = int(binding['c']['value'])
				if classId in data:
					data[classId]['l'] = classLabel
					data[classId]['i'] = classInstances
				else:
					data[classId] = {'l':classLabel, 'i':classInstances}
		# zero instance counts for classes not found in the query:
		for classId in data :
			if classId not in classesWithInstances :
				data[classId]['i'] = 0

		print "Writing json class data ..."
		with open('classes-new.json', 'w') as outfile:
			json.dump(data, outfile,separators=(',', ':'))

		print "Replacing classes json file ..."
		os.rename("classes-new.json","classes.json")

	storeStatistics('classUpdate',startTime)
	print "Class update complete."


def updatePropertyRecords() :
	print "Fetching property ids, labels, and types ..."
	startTime = time.strftime("%Y-%m-%dT%H:%M:%S")

	resultsProperties = sparqlQuery("""PREFIX wikibase: <http://wikiba.se/ontology#>
	SELECT ?id ?idLabel ?type
	WHERE {
		?id a wikibase:Property .
		?id wikibase:propertyType ?type
		SERVICE wikibase:label {
			bd:serviceParam wikibase:language "en" .
		}
	}""")

	propertyStatistics = {}
	for binding in resultsProperties['results']['bindings']:
		propertyUri = binding['id']['value']
		if propertyUri[0:31] == 'http://www.wikidata.org/entity/':
			propertyLabel = binding['idLabel']['value']
			typeUri = binding['type']['value']
			if typeUri[0:26] == 'http://wikiba.se/ontology#':
				typeName = typeUri[26:]
			else:
				typeName = 'UNKNOWN'<commit_msg>mention start times in log<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# This script retrieves data about the use of classes and properties
# on Wikidata from SPARQL. The results are processed and stored in the
# files properties.json and classes.json. The script must be run in a
# directory that already contains these files (possibly with no content
# other than an empty map {}). Normally, the files are first generated
# by the Java export tool, which provides some statistics that cannot
# be obtained in an acceptable time from SPARQL.

import requests
import json
import os
import time
#import pprint

SPARQL_SERVICE_URL = 'https://query.wikidata.org/sparql'

def doSparqlQuery(query):
	r = requests.get(SPARQL_SERVICE_URL, params={'query': query, 'format': 'json'});
	return json.loads(r.text)

def sparqlQuery(query):
	try:
		return doSparqlQuery(query)
	except ValueError:
		print "SPARQL query failed, possibly because of a time out. Waiting for 60 sec ...\n"
		time.sleep(60)
		print "Retrying query ...\n"
		try:
			return doSparqlQuery(query)
		except ValueError:
			print "Failed to get SPARQL results. Giving up."
			return json.loads('{ results: { bindings: {} }')


def setPropertyStatistics(propertyStatistics, propertyId, statisticType, numberString):
	try:
		if propertyId in propertyStatistics:
			propertyStatistics[propertyId][statisticType] = int(numberString)
	except ValueError:
		print "Error reading count value " + numberString + ": not an integer number"


def storeStatistics(key, value):
	with open('statistics.json', 'r') as outfile:
		data = json.load(outfile)
		data[key] = value
		with open('statistics.json', 'w') as outfile:
			json.dump(data, outfile)

def updateClassRecords() :
	startTime = time.strftime("%Y-%m-%dT%H:%M:%S")
	print "[" + startTime + "] Fetching class ids and labels for classes with direct instances ..."

	resultsClasses = sparqlQuery("""SELECT ?cl ?clLabel ?c
	WHERE {
	{ SELECT ?cl (count(*) as ?c) WHERE { ?i wdt:P31 ?cl } GROUP BY ?cl }
	SERVICE wikibase:label {
			bd:serviceParam wikibase:language "en" .
	}
	}""")

	print "Augmenting current class data ..."
	classesWithInstances = {}
	with open('classes.json') as classFile:    
		data = json.load(classFile)
		for binding in resultsClasses['results']['bindings']:
			classUri = binding['cl']['value']
			if classUri[0:31] == 'http://www.wikidata.org/entity/':
				classId = classUri[32:]
				classesWithInstances[classId] = True
				classLabel = binding['clLabel']['value']
				classInstances = int(binding['c']['value'])
				if classId in data:
					data[classId]['l'] = classLabel
					data[classId]['i'] = classInstances
				else:
					data[classId] = {'l':classLabel, 'i':classInstances}
		# zero instance counts for classes not found in the query:
		for classId in data :
			if classId not in classesWithInstances :
				data[classId]['i'] = 0

		print "Writing json class data ..."
		with open('classes-new.json', 'w') as outfile:
			json.dump(data, outfile,separators=(',', ':'))

		print "Replacing classes json file ..."
		os.rename("classes-new.json","classes.json")

	storeStatistics('classUpdate',startTime)
	print "Class update complete."


def updatePropertyRecords() :
	startTime = time.strftime("%Y-%m-%dT%H:%M:%S")
	print "[" + startTime + "] Fetching property ids, labels, and types ..."

	resultsProperties = sparqlQuery("""PREFIX wikibase: <http://wikiba.se/ontology#>
	SELECT ?id ?idLabel ?type
	WHERE {
		?id a wikibase:Property .
		?id wikibase:propertyType ?type
		SERVICE wikibase:label {
			bd:serviceParam wikibase:language "en" .
		}
	}""")

	propertyStatistics = {}
	for binding in resultsProperties['results']['bindings']:
		propertyUri = binding['id']['value']
		if propertyUri[0:31] == 'http://www.wikidata.org/entity/':
			propertyLabel = binding['idLabel']['value']
			typeUri = binding['type']['value']
			if typeUri[0:26] == 'http://wikiba.se/ontology#':
				typeName = typeUri[26:]
			else:
				typeName = 'UNKNOWN'
{'fraction_non_alphanumeric': 0.1384712571067593, 'fraction_numerical': 0.005306380290587492, 'mean_word_length': 4.396046353101568, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4525921', 'n_tokens_mistral': 2778, 'n_tokens_neox': 2519, 'n_words': 731}