<commit_before>
__all__ = ['XunleiClient']

import urllib
import urllib2
import cookielib
import re
import time
import os.path
import json
from ast import literal_eval


def retry(f_or_arg, *args):
	#retry_sleeps = [1, 1, 1]
	retry_sleeps = [1, 2, 3, 5, 10, 20, 30, 60] + [60] * 60
	def decorator(f):
		def withretry(*args, **kwargs):
			for second in retry_sleeps:
				try:
					return f(*args, **kwargs)
				except:
					import traceback
					logger.debug("Exception happened. Retrying...")
					logger.debug(traceback.format_exc())
					time.sleep(second)
			raise
		return withretry
	if callable(f_or_arg) and not args:
		return decorator(f_or_arg)
	else:
		a = f_or_arg
		assert type(a) == int
		assert not args
		retry_sleeps = [1] * a
		return decorator

class Logger:
	def stdout(self, message):
		print message
	def info(self, message):
		print message
	def debug(self, message):
		pass
	def trace(self, message):
		pass

logger = Logger()

class WithAttrSnapshot:
	def __init__(self, object, **attrs):
		self.object = object
		self.attrs = attrs
	def __enter__(self):
		self.old_attrs = []
		for k in self.attrs:
			if hasattr(self.object, k):
				self.old_attrs.append((k, True, getattr(self.object, k)))
			else:
				self.old_attrs.append((k, False, None))
		for k in self.attrs:
			setattr(self.object, k, self.attrs[k])
	def __exit__(self, exc_type, exc_val, exc_tb):
		for k, has_old_attr, v in self.old_attrs:
			if has_old_attr:
				setattr(self.object, k, v)
			else:
				delattr(self.object, k)

class WithAttr:
	def __init__(self, object):
		self.object = object
	def __call__(self, **kwargs):
		return WithAttrSnapshot(self.object, **kwargs)
	def __getattr__(self, k):
		return lambda (v): WithAttrSnapshot(self.object, **{k:v})

class OnDemandTaskList:
	def __init__(self, fetch_page, page_size, limit):
		self.fetch_page = fetch_page
		if limit and page_size > limit:
			page_size = limit
		self.page_size = page_size
		self.limit = limit
		self.pages = {}
		self.chunk = []
		self.completed = False
		self.total_task_number = None
		self.real_total_task_number = None

	def get_nth_task(self, n):
		if self.completed:
			return self.chunk[n]
		if n < len(self.chunk):
			return self.chunk[n]
		page = n / self.page_size
		n_in_page = n - page * self.page_size
		if page not in self.pages:
			self.hit_page(page)
		return self.pages[page][n_in_page]

	def hit_page(self, page):
		if page in self.pages:
			return self.pages[page]
		info = self.fetch_page(page, self.page_size)
		if self.total_task_number is None:
			self.total_task_number = info['total_task_number']
		tasks = info['tasks']
		for i, t in enumerate(tasks):
			t['#'] = self.page_size * page + i
		if len(self.chunk) == self.page_size * page:
			self.chunk.extend(tasks)
		# if 0 < len(tasks) < self.page_size or len(self.chunk) == self.page_size * page:
		if 0 < len(tasks) < self.page_size or (len(tasks) == 0 and (page == 0 or len(self.pages.get(page - 1, [])) == self.page_size)):
			# end of the page
			assert self.real_total_task_number is None
			self.real_total_task_number = self.page_size * page + len(tasks)
			assert self.real_total_task_number <= self.total_task_number, "unexpected task number: %s > %s" % (self.real_total_task_number, self.total_task_number)
		self.pages[page] = tasks
		return tasks

	def __getitem__(self, n):
		return self.get_nth_task(n)

	def __iter__(self):
		class Iterator:
			def __init__(self, container):
				self.container = container
				self.current = 0
			def next(self):
				if self.container.total_task_number is None:
					self.container.hit_page(0)
				assert type(self.container.total_task_number) == int
				if self.container.real_total_task_number is None:
					if self.current < self.container.total_task_number:
						try:
							task = self.container[self.current]
						except IndexError:
							raise StopIteration()
					else:
						raise StopIteration()
				else:
					if self.current < self.container.real_total_task_number:
						task = self.container[self.current]
					else:
						raise StopIteration()
				self.current += 1
				return task
		return Iterator(self)

	def __len__(self):
		if self.real_total_task_number:
			return self.real_total_task_number
		count = 0
		for t in self:
			count += 1
		return count

class XunleiClient(object):
	default_page_size = 100
	default_bt_page_size = 9999
	def __init__(self, username=None, password=<PASSWORD>, cookie_path=None, login=True, verification_code_reader=None):
		self.attr = WithAttr(self)

		self.username = username
		self.password = password
		self.cookie_path = cookie_path
		if cookie_path:
			self.cookiejar = cookielib.LWPCookieJar()
			if os.path.exists(cookie_path):
				self.load_cookies()
		else:
			self.cookiejar = cookielib.CookieJar()

		self.page_size = self.default_page_size<commit_msg>fix limit in OnDemandTaskList (but still buggy code)<commit_after>
__all__ = ['XunleiClient']

import urllib
import urllib2
import cookielib
import re
import time
import os.path
import json
from ast import literal_eval


def retry(f_or_arg, *args):
	#retry_sleeps = [1, 1, 1]
	retry_sleeps = [1, 2, 3, 5, 10, 20, 30, 60] + [60] * 60
	def decorator(f):
		def withretry(*args, **kwargs):
			for second in retry_sleeps:
				try:
					return f(*args, **kwargs)
				except:
					import traceback
					logger.debug("Exception happened. Retrying...")
					logger.debug(traceback.format_exc())
					time.sleep(second)
			raise
		return withretry
	if callable(f_or_arg) and not args:
		return decorator(f_or_arg)
	else:
		a = f_or_arg
		assert type(a) == int
		assert not args
		retry_sleeps = [1] * a
		return decorator

class Logger:
	def stdout(self, message):
		print message
	def info(self, message):
		print message
	def debug(self, message):
		pass
	def trace(self, message):
		pass

logger = Logger()

class WithAttrSnapshot:
	def __init__(self, object, **attrs):
		self.object = object
		self.attrs = attrs
	def __enter__(self):
		self.old_attrs = []
		for k in self.attrs:
			if hasattr(self.object, k):
				self.old_attrs.append((k, True, getattr(self.object, k)))
			else:
				self.old_attrs.append((k, False, None))
		for k in self.attrs:
			setattr(self.object, k, self.attrs[k])
	def __exit__(self, exc_type, exc_val, exc_tb):
		for k, has_old_attr, v in self.old_attrs:
			if has_old_attr:
				setattr(self.object, k, v)
			else:
				delattr(self.object, k)

class WithAttr:
	def __init__(self, object):
		self.object = object
	def __call__(self, **kwargs):
		return WithAttrSnapshot(self.object, **kwargs)
	def __getattr__(self, k):
		return lambda (v): WithAttrSnapshot(self.object, **{k:v})

# TODO: write unit test
class OnDemandTaskList:
	def __init__(self, fetch_page, page_size, limit):
		self.fetch_page = fetch_page
		if limit and page_size > limit:
			page_size = limit
		self.page_size = page_size
		self.limit = limit
		self.pages = {}
		self.max_task_number = None
		self.real_total_task_number = None
		self.total_pages = None

	def is_out_of_range(self, n):
		if self.limit:
			if n >= self.limit:
				return True
		if self.max_task_number:
			if n >= self.max_task_number:
				return True
		if self.real_total_task_number:
			if n >= self.real_total_task_number:
				return True

	def check_out_of_range(self, n):
		if self.is_out_of_range(n):
			raise IndexError('task index out of range')

	def is_out_of_page(self, page):
		raise NotImplementedError()

	def get_nth_task(self, n):
		self.check_out_of_range(n)
		page = n / self.page_size
		n_in_page = n - page * self.page_size
		return self.hit_page(page)[n_in_page]

	def touch(self):
		self.hit_page(0)

	def hit_page(self, page):
		if page in self.pages:
			return self.pages[page]
		info = self.fetch_page(page, self.page_size)
		tasks = info['tasks']
		if self.max_task_number is None:
			self.max_task_number = info['total_task_number']
			if self.limit and self.max_task_number > self.limit:
				self.max_task_number = self.limit
			self.total_pages = self.max_task_number / self.page_size
			if self.max_task_number % self.page_size != 0:
				self.total_pages += 1
			if self.max_task_number == 0:
				self.real_total_task_number = 0
		if page >= self.total_pages:
			tasks = []
		elif page == self.total_pages - 1:
			if self.page_size * page + len(tasks) > self.max_task_number:
				tasks = tasks[0:self.max_task_number - self.page_size * page]
			if len(tasks) > 0:
				self.real_total_task_number = self.page_size * page + len(tasks)
			else:
				self.max_task_number -= self.page_size
				self.total_pages -= 1
				if len(self.pages.get(page-1, [])) == self.page_size:
					self.real_total_task_number = self.max_task_number
		else:
			if len(tasks) == 0:
				self.max_task_number = self.page_size * page
				self.total_pages = page
				if len(self.pages.get(page-1, [])) == self.page_size:
					self.real_total_task_number = self.max_task_number
			elif len(tasks) < self.page_size:
				self.real_total_task_number = self.page_size * page + len(tasks)
				self.max_task_number = self.real_total_task_number
				self.total_pages = page
			else:
				pass
		for i, t in enumerate(tasks):
			t['#'] = self.page_size * page + i
		self.pages[page] = tasks
		return tasks

	def __getitem__(self, n):
		return self.get_nth_task(n)

	def __iter__(self):
		class Iterator:
			def __init__(self, container):
				self.container = container
				self.current = 0
			def next(self):
				self.container.touch()
				assert type(self.container.max_task_number) == int
				if self.container.real_total_task_number is None:
					if self.current < self.container.max_task_number:
						try:
							task = self.container[self.current]
						except IndexError:
							raise StopIteration()
					else:
						raise StopIteration()
				else:
					if self.current < self.container.real_total_task_number:
						task = self.container[self.current]
					else:
						raise StopIteration()
				self.current += 1
				return task
		return Iterator(self)

	def __len__(self):
		if self.real_total_task_number:
			return self.real_total_task_number
		self.touch()
		self.hit_page(self.total_pages-1)
		if self.real_total_task_number:
			return self.real_total_task_number
		count = 0
		for t in self:
			count += 1
		return count

class XunleiClient(object):
	default_page_size = 100
	default_bt_page_size = 9999
	def __init__(self, username=None, password=<PASSWORD>, cookie_path=None, login=True, verification_code_reader=None):
		self.attr = WithAttr(self)

		self.username = username
		self.password = password
		self.cookie_path = cookie_path
		if cookie_path:
			self.cookiejar = cookielib.LWPCookieJar()
			if os.path.exists(cookie_path):
				self.load_cookies()
		else:
			self.cookiejar = cookielib.CookieJar()

		self.page_size = self.default_page_size
{'fraction_non_alphanumeric': 0.11077983788316408, 'fraction_numerical': 0.0077331594148886615, 'mean_word_length': 3.73071837814015, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2693609', 'n_tokens_mistral': 4470, 'n_tokens_neox': 3835, 'n_words': 923}