<commit_before>
__all__ = ['XunleiClient']

import urllib
import urllib2
import cookielib
import re
import time
import os.path
import json
from ast import literal_eval

def retry(f):
	#retry_sleeps = [1, 1, 1]
	retry_sleeps = [1, 2, 3, 5, 10, 20, 30, 60] + [60] * 60
	def withretry(*args, **kwargs):
		for second in retry_sleeps:
			try:
				return f(*args, **kwargs)
			except:
				import traceback
				import sys
				print "Exception in user code:"
				traceback.print_exc(file=sys.stdout)
				time.sleep(second)
		raise
	return withretry

class Logger:
	def stdout(self, message):
		print message
	def info(self, message):
		print message
	def debug(self, message):
		pass
	def trace(self, message):
		pass

logger = Logger()

class XunleiClient:
	page_size = 9999
	bt_page_size = 9999
	def __init__(self, username=None, password=<PASSWORD>, cookie_path=None, login=True):
		self.username = username
		self.password = password
		self.cookie_path = cookie_path
		if cookie_path:
			self.cookiejar = cookielib.LWPCookieJar()
			if os.path.exists(cookie_path):
				self.load_cookies()
		else:
			self.cookiejar = cookielib.CookieJar()
		self.set_page_size(self.page_size)
		self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookiejar))
		if login:
			if not self.has_logged_in():
				self.login()
			else:
				self.id = self.get_userid()

	@retry
	def urlopen(self, url, **args):
		logger.debug(url)
#		import traceback
#		for line in traceback.format_stack():
#			print line.strip()
		if 'data' in args and type(args['data']) == dict:<commit_msg>fix paging when task number > 100<commit_after>
__all__ = ['XunleiClient']

import urllib
import urllib2
import cookielib
import re
import time
import os.path
import json
from ast import literal_eval

def retry(f):
	#retry_sleeps = [1, 1, 1]
	retry_sleeps = [1, 2, 3, 5, 10, 20, 30, 60] + [60] * 60
	def withretry(*args, **kwargs):
		for second in retry_sleeps:
			try:
				return f(*args, **kwargs)
			except:
				import traceback
				import sys
				print "Exception in user code:"
				traceback.print_exc(file=sys.stdout)
				time.sleep(second)
		raise
	return withretry

class Logger:
	def stdout(self, message):
		print message
	def info(self, message):
		print message
	def debug(self, message):
		pass
	def trace(self, message):
		pass

logger = Logger()

class XunleiClient:
	page_size = 100
	bt_page_size = 9999
	def __init__(self, username=None, password=<PASSWORD>, cookie_path=None, login=True):
		self.username = username
		self.password = password
		self.cookie_path = cookie_path
		if cookie_path:
			self.cookiejar = cookielib.LWPCookieJar()
			if os.path.exists(cookie_path):
				self.load_cookies()
		else:
			self.cookiejar = cookielib.CookieJar()
		self.set_page_size(self.page_size)
		self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookiejar))
		if login:
			if not self.has_logged_in():
				self.login()
			else:
				self.id = self.get_userid()

	@retry
	def urlopen(self, url, **args):
		logger.debug(url)
#		import traceback
#		for line in traceback.format_stack():
#			print line.strip()
		if 'data' in args and type(args['data']) == dict:
{'fraction_non_alphanumeric': 0.11463956811686249, 'fraction_numerical': 0.019688790092092727, 'mean_word_length': 4.121951219512195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1288583', 'n_tokens_mistral': 1322, 'n_tokens_neox': 1154, 'n_words': 298}