<commit_before>#! /usr/bin/env python

'''Politely (per pay-level-domain) fetch urls'''

from downpour import BaseFetcher, logger, reactor

import qr
import time						# For to implement politeness
import urlparse					# To efficiently parse URLs

class PoliteFetcher(BaseFetcher):
	# How long should we wait before making a request to the same tld?
	wait     = 2.0
	# Infinity, because there's not another easy way to access it?
	infinity = float('Inf')
	
	def __init__(self, poolSize=10, **kwargs):
		# Call the parent constructor
		BaseFetcher.__init__(self, poolSize)
		self.plds = dict((q.key, q) for q in qr.Queue.all('domain:*'))
		self.pldQueue = qr.Queue('plds')
		self.requests = qr.Queue('request', **kwargs)
		self.timer = None
	
	def __len__(self):
		return len(self.pldQueue) + len(self.requests)
		
	def getKey(self, req):
		# This actually considers the whole domain name, including subdomains, uniquely
		# This aliasing is just in case we want to change that scheme later, easily
		return 'domain:%s' % urlparse.urlparse(req.url.strip()).hostname
	
	# Event callbacks
	def onDone(self, request):
		key = self.getKey(request)
		if len(self.plds[key]):
			delay = time.time() + self.wait
			self.pldQueue.push((delay, key))
		else:
			del self.plds[key]
	
	#################
	# Insertion to our queue
	#################
	def extend(self, requests):
		count = 0
		t = time.time()
		for r in requests:
			count += 1
			key = self.getKey(r)
			try:
				self.plds[key].push(r)
			except KeyError:
				logger.debug('Making queue for %s' % r.url)
				q = qr.Queue(key)
				q.push(r)
				self.plds[key] = q
				self.pldQueue.push((t, key))
		self.remaining += count
	
	def grow(self, upto=10000):
		count = 0
		t = time.time()
		r = self.requests.pop()
		while r and count < upto:
			count += 1
			key = self.getKey(r)
			try:
				logger.debug('Read request %s' % r.url)
				self.plds[key].push(r)
			except KeyError:
				logger.debug('Making queue for %s' % key)
				q = qr.Queue(key)
				q.push(r)
				self.plds[key] = q
				self.pldQueue.push((t, key))
			r = self.requests.pop()
		self.remaining += count
		
	def pop(self):
		'''Get the next request'''
		if len(self.pldQueue) < self.poolSize:
			self.grow()
		now = time.time()
		while True:
			# Get the next plds we might want to fetch from
			next = self.pldQueue.peek()
			# If the next-fetchable is not soon enough, then wait
			if next[0] > now:
				# If we weren't waiting, then wait
				if self.timer == None:
					logger.debug('Waiting %f seconds' % (next[0] - now))
					self.timer = reactor.callLater(next[0] - now, self.serveNext)
				return None
			else:
				# Go ahead and pop this item
				next = self.pldQueue.pop()
				# Unset the timer
				self.timer = None
				try:
					return self.plds[next[1]].pop()
				except ValueError:
					# This should never happen
					logger.error('Tried to pop from non-existent pld: %s' % next[1])
					return None
				except KeyError:
					# This can happen between restarts
					self.plds[next[1]] = qr.Queue(next[1])
					return self.plds[next[1]].pop()
		return None
		
if __name__ == '__main__':
	import logging
	from downpour import BaseRequest
	
	# Turn on logging
	logger.setLevel(logging.DEBUG)

	q = qr.Queue('requests')
	with file('urls.txt') as f:
		for line in f:
			q.push(BaseRequest(line.strip()))
	
	p = PoliteFetcher(100)
	p.start()<commit_msg>No longer keeping track of all the distinct queues in python.<commit_after>#! /usr/bin/env python

'''Politely (per pay-level-domain) fetch urls'''

from downpour import BaseFetcher, logger, reactor

import qr
import time						# For to implement politeness
import urlparse					# To efficiently parse URLs

class PoliteFetcher(BaseFetcher):
	# How long should we wait before making a request to the same tld?
	wait     = 2.0
	# Infinity, because there's not another easy way to access it?
	infinity = float('Inf')
	
	def __init__(self, poolSize=10, **kwargs):
		# Call the parent constructor
		BaseFetcher.__init__(self, poolSize)
		self.pldQueue = qr.Queue('plds')
		self.requests = qr.Queue('request', **kwargs)
		self.timer = None
	
	def __len__(self):
		return len(self.pldQueue) + len(self.requests)
		
	def getKey(self, req):
		# This actually considers the whole domain name, including subdomains, uniquely
		# This aliasing is just in case we want to change that scheme later, easily
		return 'domain:%s' % urlparse.urlparse(req.url.strip()).hostname
	
	# Event callbacks
	def onDone(self, request):
		key = self.getKey(request)
		delay = time.time() + self.wait
		self.pldQueue.push((delay, key))
	
	#################
	# Insertion to our queue
	#################
	def extend(self, requests):
		count = 0
		t = time.time()
		for r in requests:
			count += 1
			key = self.getKey(r)
			qr.Queue(key).push(r)
			self.pldQueue.push((t, key))
		self.remaining += count
	
	def grow(self, upto=10000):
		count = 0
		t = time.time()
		r = self.requests.pop()
		while r and count < upto:
			count += 1
			key = self.getKey(r)
			qr.Queue(key).push(r)
			self.pldQueue.push((t, key))
			r = self.requests.pop()
		self.remaining += count
		
	def pop(self):
		'''Get the next request'''
		if len(self.pldQueue) < self.poolSize:
			self.grow()
		now = time.time()
		while True:
			# Get the next plds we might want to fetch from
			next = self.pldQueue.peek()
			# If the next-fetchable is not soon enough, then wait
			if next[0] > now:
				# If we weren't waiting, then wait
				if self.timer == None:
					logger.debug('Waiting %f seconds' % (next[0] - now))
					self.timer = reactor.callLater(next[0] - now, self.serveNext)
				return None
			else:
				# Go ahead and pop this item
				next = self.pldQueue.pop()
				# Unset the timer
				self.timer = None
				try:
					v = qr.Queue(next[1]).pop()
					if not v:
						continue
					return v
				except ValueError:
					# This should never happen
					logger.error('Tried to pop from non-existent pld: %s' % next[1])
					return None
		return None
		
if __name__ == '__main__':
	import logging
	from downpour import BaseRequest
	
	# Turn on logging
	logger.setLevel(logging.DEBUG)

	q = qr.Queue('requests')
	with file('urls.txt') as f:
		for line in f:
			q.push(BaseRequest(line.strip()))
	
	p = PoliteFetcher(100)
	p.start()
{'fraction_non_alphanumeric': 0.13389857622780355, 'fraction_numerical': 0.007198848184290513, 'mean_word_length': 3.187541862022773, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2434433', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2165, 'n_words': 648}