<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/python

import re, urllib, codecs, json

DEFAULT_API="https://commons.wikimedia.org/w/api.php"
MAX_URL_SIZE=1024


def urlEncodeNonAscii(b):
    return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)

class MwQueryError(Exception):
	def __init__(self, value):
	    self.value = value
	def __str__(self): 
		return repr(self.value)

class MwWiki:
	""" 
	Wiki API 
	"""
	def __init__(self, url_api=DEFAULT_API):
		self.url = url_api

	def send_to_api(self, request, debug=False):
		# add action to url
		url_req = "%s?action=%s" % (self.url, request.action)
		# add each property
		for k in request.prop.keys():
			url_req += "&%s=%s" % (k, urlEncodeNonAscii(request.prop[k]))
		# add the format
		url_req += "&format=%s" % (request.format)
		#print url_req
		if not debug:
			return urllib.urlopen(url_req).read()
		else:
			return url_req

	def process_prop_query(self, request, titles):
		"""
		Quick and dirty prop query support
		"""
		if request.action != 'query' or not ('prop' in request.prop.keys()):
			raise MwQueryError('Not a prop query')
		if 'titles' in request.prop.keys():
			raise MwQueryError('process_prop_query should not have titles in the request object')
		url_base = self.send_to_api(request, debug=True)
		url_req = url_base
		results = dict()
		for title in titles:
			# testing length
			if len(url_req) + len(title) + 8 >= MAX_URL_SIZE:
				self.process_prop_query_results(url_req, results)
				# init a new query
				url_req = url_base
			
			if '&titles' not in url_req:
				url_req += "&titles=%s" % (urlEncodeNonAscii(title))
			else:
				url_req += "|%s" % (urlEncodeNonAscii(title))
		if len(url_req) > len(url_base):
			self.process_prop_query_results(url_req, results)
		return results

	def process_prop_query_results(self, url_req, results):
		try:
			req_result = json.loads(urllib.urlopen(url_req).read())
			if 'query-continue' in req_result.keys():
				raise MwQueryError("continue not supported for prop query") 
			r = req_result['query']['pages']
			for p in r:
				results[p] = r[p]
		except KeyError:
			print "Empty result for --> %s" % (url_req)			

	
	def process_query(self, request, result=[]):
		"""
		Quick and dirty continue support for list query
		"""
		if request.action != 'query':
			raise MwQueryError("action is not a query")
		result_name = None
		if 'list' in request.prop.keys():
			result_name = request.prop['list']
		else:
			raise MwQueryError("query has no list")
		query_result = json.loads(self.send_to_api(request))

		result.extend(query_result[request.action][result_name])<commit_msg>Using httplib2 to adress unicode encoding in filenames<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/python

import re, urllib, codecs, json, httplib2

DEFAULT_API="https://commons.wikimedia.org/w/api.php"
MAX_URL_SIZE=1024


def urlEncodeNonAscii(b):
    return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)

class MwQueryError(Exception):
	def __init__(self, value):
	    self.value = value
	def __str__(self): 
		return repr(self.value)

class MwWiki:
	""" 
	Wiki API 
	"""
	def __init__(self, url_api=DEFAULT_API):
		self.url = url_api

	def __encode_param(self, param):
		if not isinstance(param, unicode):
			return unicode(param)
		else:
			return param

	def send_to_api(self, request, debug=False):
		# add action to url
		url_req = "%s?action=%s" % (self.url, request.action)
		# add each property
		for k in request.prop.keys():
			#url_req += "&%s=%s" % (k, urlEncodeNonAscii(request.prop[k]))
			url_req += "&%s=%s" % (k, self.__encode_param(request.prop[k]))
		# add the format
		url_req += "&format=%s" % (request.format)
		#print url_req
		if not debug:
			uri = httplib2.iri2uri(unicode(url_req))
			return urllib.urlopen(uri).read()
		else:
			return url_req

	def process_prop_query(self, request, titles):
		"""
		Quick and dirty prop query support
		"""
		if request.action != 'query' or not ('prop' in request.prop.keys()):
			raise MwQueryError('Not a prop query')
		if 'titles' in request.prop.keys():
			raise MwQueryError('process_prop_query should not have titles in the request object')
		url_base = self.send_to_api(request, debug=True)
		url_req = url_base
		results = dict()
		for title in titles:
			# testing length
			if len(url_req) + len(title) + 8 >= MAX_URL_SIZE:
				self.process_prop_query_results(url_req, results)
				# init a new query
				url_req = url_base
			
			if '&titles' not in url_req:
				url_req += "&titles=%s" % (self.__encode_param(title))
			else:
				url_req += "|%s" % (self.__encode_param(title))
		if len(url_req) > len(url_base):
			self.process_prop_query_results(url_req, results)
		return results

	def process_prop_query_results(self, url_req, results):
		try:
			uri = httplib2.iri2uri(unicode(url_req))
			req_result  = json.loads(urllib.urlopen(uri).read())
			# req_result = json.loads(self.__get_url(url_req).read())
			if 'query-continue' in req_result.keys():
				raise MwQueryError("continue not supported for prop query") 
			r = req_result['query']['pages']
			for p in r:
				results[p] = r[p]
		except KeyError:
			print "Empty result for --> %s" % (url_req)			

	
	def process_query(self, request, result=[]):
		"""
		Quick and dirty continue support for list query
		"""
		if request.action != 'query':
			raise MwQueryError("action is not a query")
		result_name = None
		if 'list' in request.prop.keys():
			result_name = request.prop['list']
		else:
			raise MwQueryError("query has no list")
		query_result = json.loads(self.send_to_api(request))

		result.extend(query_result[request.action][result_name])
{'fraction_non_alphanumeric': 0.1402200922967696, 'fraction_numerical': 0.004969826056088037, 'mean_word_length': 4.062893081761007, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4453914', 'n_tokens_mistral': 2268, 'n_tokens_neox': 2018, 'n_words': 495}