<commit_before>__ver__ = '0.6.0'

import urllib, urlparse
import time, random
import sys, weakref
import socket

import simplejson
import http

import errors
import listing, page
import compatibility

try:
	import gzip
except ImportError:
	gzip = None
try:
	from cStringIO import StringIO
except ImportError:
	from StringIO import StringIO

def parse_timestamp(t):
	if t == '0000-00-00T00:00:00Z':
		return (0, 0, 0, 0, 0, 0, 0, 0)
	return time.strptime(t, '%Y-%m-%dT%H:%M:%SZ')
	
class WaitToken(object):
	def __init__(self):
		self.id = '%x' % random.randint(0, sys.maxint)
	def __hash__(self):
		return hash(self.id)

class Site(object):
	api_limit = 500
	def __init__(self, host, path = '/w/', pool = None, retry_timeout = 30, 
			max_retries = 25, wait_callback = lambda *x: None, 
			max_lag = 3, compress = True, force_login = True, do_init = True):
		self.host = host
		self.path = path
		self.credentials = None
		self.compress = compress
		
		self.retry_timeout = retry_timeout
		self.max_retries = max_retries
		self.wait_callback = wait_callback
		self.max_lag = str(max_lag)
		
		self.wait_tokens = weakref.WeakKeyDictionary()
			
		self.blocked = False
		self.hasmsg = False
		self.groups = []
		self.rights = []
		self.tokens = {}
		self.force_login = force_login
		
		if pool is None:
			self.connection = http.HTTPPool()
		else:
			self.connection = pool
			
		self.version = None
			
		self.Pages = listing.PageList(self)
		self.Categories = listing.PageList(self, namespace = 14)
		self.Images = listing.PageList(self, namespace = 6)
		
		self.namespaces = self.default_namespaces
		
		self.initialized = False
		
		if do_init:
			try:
				self.site_init()
			except errors.APIError, e:
				# Private wiki, do init after login
				if e[0] != u'unknown_action': raise
				
			
	def site_init(self):
		meta = self.api('query', meta = 'siteinfo|userinfo', 
			siprop = 'general|namespaces', uiprop = 'groups|rights')
		self.site = meta['query']['general']
		self.namespaces = dict(((i['id'], i.get('*', '')) for i in meta['query']['namespaces'].itervalues()))
			
		if self.site['generator'].startswith('MediaWiki '):
			version = self.site['generator'][10:].split('.')
			if len(version) == 2 and version[1].endswith('alpha'):
				self.version = (int(version[0]), int(version[1][:-5]), 'alpha')
			elif len(version) == 3:
				self.version = (int(version[0]), int(version[1]), int(version[2]))
			else:
				raise errors.MediaWikiVersionError('Unknown MediaWiki %s' % '.'.join(version))
		else:
			raise errors.MediaWikiVersionError('Unknown generator %s' % self.site['generator'])
		# Require 1.11 until some compatibility issues are fixed
		self.require(1, 11)
			
		userinfo = compatibility.userinfo(meta, self.require(1, 12, raise_error = False))
		self.username = userinfo['name']
		self.groups = userinfo.get('groups', [])
		self.rights = userinfo.get('rights', [])
		self.initialized = True
		
		
	default_namespaces = {0: u'', 1: u'Talk', 2: u'User', 3: u'User talk', 4: u'Project', 5: u'Project talk', 
		6: u'Image', 7: u'Image talk', 8: u'MediaWiki', 9: u'MediaWiki talk', 10: u'Template', 11: u'Template talk', 
		12: u'Help', 13: u'Help talk', 14: u'Category', 15: u'Category talk', -1: u'Special', -2: u'Media'}
		
	def __repr__(self):
		return "<Site object '%s%s'>" % (self.host, self.path)
		
	
	def api(self, action, *args, **kwargs):
		kwargs.update(args)
		if action == 'query':
			if 'meta' in kwargs:
				kwargs['meta'] += '|userinfo'
			else:
				kwargs['meta'] = 'userinfo'
			if 'uiprop' in kwargs:
				kwargs['uiprop'] += '|blockinfo|hasmsg'
			else:
				kwargs['uiprop'] = 'blockinfo|hasmsg'
			
		token = self.wait_token()
		while True:
			info = self.raw_api(action, **kwargs)
			if not info: info = {}
				
			try:
				userinfo = compatibility.userinfo(info, self.require(1, 12, raise_error = None))
			except KeyError:
				userinfo = ()
			if 'blockedby' in userinfo:
				self.blocked = (userinfo['blockedby'], userinfo.get('blockreason', u''))
			else:
				self.blocked = False
			self.hasmsg = 'message' in userinfo
			self.logged_in = 'anon' not in userinfo
			if 'error' in info:
				if info['error']['code'] in (u'internal_api_error_DBConnectionError', ):
					self.wait(token)
					continue
				raise errors.APIError(info['error']['code'],
					info['error']['info'])
			return info
		
	@staticmethod
	def _to_str(data):
		if type(data) is unicode:
			return data.encode('utf-8')
		return str(data)
	@staticmethod
	def _query_string(*args, **kwargs):
		kwargs.update(args)
		qs = urllib.urlencode([(k, Site._to_str(v)) for k, v in kwargs.iteritems()
			if k != 'wpEditToken'])
		if 'wpEditToken' in kwargs: 
			qs += '&wpEditToken=' + Site._to_str(kwargs['wpEditToken'])
		return qs
		
	def raw_call(self, script, data):
		url = self.path + script + '.php'
		headers = {'Content-Type': 'application/x-www-form-urlencoded'}
		if self.compress and gzip:
			headers['Accept-Encoding'] = 'gzip'
		
		token = self.wait_token((script, data))
		while True:<commit_msg>Revert to r25<commit_after>__ver__ = '0.6.0'

import urllib, urlparse
import time, random
import sys, weakref
import socket

import simplejson
import http

import errors
import listing, page
import compatibility

try:
	import gzip
except ImportError:
	gzip = None
try:
	from cStringIO import StringIO
except ImportError:
	from StringIO import StringIO

def parse_timestamp(t):
	if t == '0000-00-00T00:00:00Z':
		return (0, 0, 0, 0, 0, 0, 0, 0)
	return time.strptime(t, '%Y-%m-%dT%H:%M:%SZ')
	
class WaitToken(object):
	def __init__(self):
		self.id = '%x' % random.randint(0, sys.maxint)
	def __hash__(self):
		return hash(self.id)

class Site(object):
	api_limit = 500
	def __init__(self, host, path = '/w/', pool = None, retry_timeout = 30, 
			max_retries = 25, wait_callback = lambda *x: None, 
			max_lag = 3, compress = True, force_login = True, do_init = True):
		self.host = host
		self.path = path
		self.credentials = None
		self.compress = compress
		
		self.retry_timeout = retry_timeout
		self.max_retries = max_retries
		self.wait_callback = wait_callback
		self.max_lag = str(max_lag)
		
		self.wait_tokens = weakref.WeakKeyDictionary()
			
		self.blocked = False
		self.hasmsg = False
		self.groups = []
		self.rights = []
		self.tokens = {}
		self.force_login = force_login
		
		if pool is None:
			self.connection = http.HTTPPool()
		else:
			self.connection = pool
			
		self.version = None
			
		self.Pages = listing.PageList(self)
		self.Categories = listing.PageList(self, namespace = 14)
		self.Images = listing.PageList(self, namespace = 6)
		
		self.namespaces = self.default_namespaces
		
		self.initialized = False
		
		if do_init:
			try:
				self.site_init()
			except errors.APIError, e:
				# Private wiki, do init after login
				if e[0] != u'unknown_action': raise
				
			
	def site_init(self):
		meta = self.api('query', meta = 'siteinfo|userinfo', 
			siprop = 'general|namespaces', uiprop = 'groups|rights')
		self.site = meta['query']['general']
		self.namespaces = dict(((i['id'], i.get('*', '')) for i in meta['query']['namespaces'].itervalues()))
			
		if self.site['generator'].startswith('MediaWiki '):
			version = self.site['generator'][10:].split('.')
			if len(version) == 2 and version[1].endswith('alpha'):
				self.version = (int(version[0]), int(version[1][:-5]), 'alpha')
			elif len(version) == 3:
				self.version = (int(version[0]), int(version[1]), int(version[2]))
			else:
				raise errors.MediaWikiVersionError('Unknown MediaWiki %s' % '.'.join(version))
		else:
			raise errors.MediaWikiVersionError('Unknown generator %s' % self.site['generator'])
		# Require 1.11 until some compatibility issues are fixed
		self.require(1, 11)
			
		userinfo = compatibility.userinfo(meta, self.require(1, 12, raise_error = False))
		self.username = userinfo['name']
		self.groups = userinfo.get('groups', [])
		self.rights = userinfo.get('rights', [])
		self.initialized = True
		
		
	default_namespaces = {0: u'', 1: u'Talk', 2: u'User', 3: u'User talk', 4: u'Project', 5: u'Project talk', 
		6: u'Image', 7: u'Image talk', 8: u'MediaWiki', 9: u'MediaWiki talk', 10: u'Template', 11: u'Template talk', 
		12: u'Help', 13: u'Help talk', 14: u'Category', 15: u'Category talk', -1: u'Special', -2: u'Media'}
		
	def __repr__(self):
		return "<Site object '%s%s'>" % (self.host, self.path)
		
	
	def api(self, action, *args, **kwargs):
		kwargs.update(args)
		if action == 'query':
			if 'meta' in kwargs:
				kwargs['meta'] += '|userinfo'
			else:
				kwargs['meta'] = 'userinfo'
			if 'uiprop' in kwargs:
				kwargs['uiprop'] += '|blockinfo|hasmsg'
			else:
				kwargs['uiprop'] = 'blockinfo|hasmsg'
			
		token = self.wait_token()
		while True:
			info = self.raw_api(action, **kwargs)
			if not info: info = {}
				
			try:
				userinfo = compatibility.userinfo(info, self.require(1, 12, raise_error = None))
			except KeyError:
				userinfo = ()
			if 'blockedby' in userinfo:
				self.blocked = (userinfo['blockedby'], userinfo.get('blockreason', u''))
			else:
				self.blocked = False
			self.hasmsg = 'message' in userinfo
			self.logged_in = 'anon' not in userinfo
			if 'error' in info:
				if info['error']['code'] in (u'internal_api_error_DBConnectionError', ):
					self.wait(token)
					continue
				raise errors.APIError(info['error']['code'],
					info['error']['info'])
			return info
		
	@staticmethod
	def _to_str(data):
		if type(data) is unicode:
			return data.encode('utf-8')
		return str(data)
	@staticmethod
	def _query_string(*args, **kwargs):
		kwargs.update(args)
		return urllib.urlencode([(k, Site._to_str(v)) for k, v in kwargs.iteritems()])
		
	def raw_call(self, script, data):
		url = self.path + script + '.php'
		headers = {'Content-Type': 'application/x-www-form-urlencoded'}
		if self.compress and gzip:
			headers['Accept-Encoding'] = 'gzip'
		
		token = self.wait_token((script, data))
		while True:
{'fraction_non_alphanumeric': 0.15600687076891986, 'fraction_numerical': 0.017581085177326462, 'mean_word_length': 3.8448360254527656, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1830410', 'n_tokens_mistral': 4067, 'n_tokens_neox': 3468, 'n_words': 859}