<commit_before>from bs4 import BeautifulSoup
import re
import requests
import sys

def _retry_on_fail(req, *args, **kwargs):
	try:
		r = req(*args, **kwargs)
		if r.status_code not in range(100, 399):
			print('Connection error, retrying... (HTTP {})'.format(r.status_code), file=sys.stderr)
			return _retry_on_fail(req, *args, **kwargs)
		else:
			return r
	except (requests.exceptions.ConnectionError, requests.packages.urllib3.exceptions.ProtocolError) as e:
		print('Connection error, retrying... ({})'.format(e.args[0]), file=sys.stderr)
		return _retry_on_fail(req, *args, **kwargs)

class Nyaa(object):
	def __init__(self, url):
		self.url = url
		self.info_url = url + '?page=view&tid='
		self.dl_url = url + '?page=download&tid='

	@property
	def last_entry(self):
		r = _retry_on_fail(requests.get, self.url)
		soup = BeautifulSoup(r.text)
		link = soup.find('tr', class_='tlistrow').find('td', class_='tlistname').a['href']
		return int(re.search('tid=([0-9]*)', link).group(1))

class NyaaEntry(object):
	def __init__(self, nyaa, nyaa_id):
		self.info_url = '{}{}'.format(nyaa.info_url, nyaa_id)
		self.download_url = '{}{}&magnet=1'.format(nyaa.dl_url, nyaa_id)

		r = _retry_on_fail(requests.get, self.info_url)
		setattr(r, 'encoding', 'utf-8')
		self.page = BeautifulSoup(r.text)
		content = self.page.find('div', class_='content').text
		if 'The torrent you are looking for does not appear to be in the database' in content:
			self.exists = False
		elif 'The torrent you are looking for has been deleted' in content:
			self.exists = False
		else:
			self.exists = True

	@property
	def category(self):
		return self.page.find('td', class_='viewcategory').find_all('a')[0].text

	@property
	def sub_category(self):
		return self.page.find('td', class_='viewcategory').find_all('a')[1].text

	@property
	def name(self):
		return self.page.find('td', class_='viewtorrentname').text

	@property
	def time(self):
		return self.page.find('td', class_='vtop').text.split(', ')

	@property
	def status(self):
		_status = self.page.find('div', class_=re.compile('content'))['class']
		if 'trusted' in _status:
			return 'trusted'
		elif 'remake' in _status:
			return 'remake'
		elif 'aplus' in _status:
			return 'a+'
		else:
			return 'normal'

	@property
	def hash(self):
		r = _retry_on_fail(requests.head, self.download_url)
		if 'Location' in r.headers:
			return re.search(r'magnet:\?xt=urn:btih:(.*)&tr=', r.headers['Location']).group(1).upper()
		else:
			return None
<commit_msg>Add timer for connection retries and improve exception handling.<commit_after>from bs4 import BeautifulSoup
import re
import requests
import sys
import time

delay = 0

def _retry_on_fail(req, *args, **kwargs):
	global delay
	try:
		r = req(*args, **kwargs)
		if r.status_code not in range(100, 399):
			delay = delay * 2 + 5
			print('Connection error, retrying in {} seconds... (HTTP {})'.format(delay, r.status_code), file=sys.stderr)
			if delay > 1800:
				print('Too many retry attempts.')
			exit(code=1)
			time.sleep(delay)
			return _retry_on_fail(req, *args, **kwargs)
		else:
			delay = 0
			return r
	except requests.exceptions.RequestException as e:
		delay = delay * 2 + 5
		print('Connection error, retrying in {} seconds... ({})'.format(delay, e.args[0].args[0]), file=sys.stderr)
		if delay > 1800:
			print('Too many retry attempts.')
			exit(code=1)
		time.sleep(delay)
		return _retry_on_fail(req, *args, **kwargs)
	except requests.packages.urllib3.exceptions.ProtocolError as e:
		delay = delay * 2 + 5
		print('Connection error, retrying in {} seconds... ({})'.format(delay, e.args[1]), file=sys.stderr)
		if delay > 1800:
			print('Too many retry attempts.')
			exit(code=1)
		time.sleep(delay)
		return _retry_on_fail(req, *args, **kwargs)

class Nyaa(object):
	def __init__(self, url):
		self.url = url
		self.info_url = url + '?page=view&tid='
		self.dl_url = url + '?page=download&tid='

	@property
	def last_entry(self):
		r = _retry_on_fail(requests.get, self.url)
		soup = BeautifulSoup(r.text)
		link = soup.find('tr', class_='tlistrow').find('td', class_='tlistname').a['href']
		return int(re.search('tid=([0-9]*)', link).group(1))

class NyaaEntry(object):
	def __init__(self, nyaa, nyaa_id):
		self.info_url = '{}{}'.format(nyaa.info_url, nyaa_id)
		self.download_url = '{}{}&magnet=1'.format(nyaa.dl_url, nyaa_id)

		r = _retry_on_fail(requests.get, self.info_url)
		setattr(r, 'encoding', 'utf-8')
		self.page = BeautifulSoup(r.text)
		content = self.page.find('div', class_='content').text
		if 'The torrent you are looking for does not appear to be in the database' in content:
			self.exists = False
		elif 'The torrent you are looking for has been deleted' in content:
			self.exists = False
		else:
			self.exists = True

	@property
	def category(self):
		return self.page.find('td', class_='viewcategory').find_all('a')[0].text

	@property
	def sub_category(self):
		return self.page.find('td', class_='viewcategory').find_all('a')[1].text

	@property
	def name(self):
		return self.page.find('td', class_='viewtorrentname').text

	@property
	def time(self):
		return self.page.find('td', class_='vtop').text.split(', ')

	@property
	def status(self):
		_status = self.page.find('div', class_=re.compile('content'))['class']
		if 'trusted' in _status:
			return 'trusted'
		elif 'remake' in _status:
			return 'remake'
		elif 'aplus' in _status:
			return 'a+'
		else:
			return 'normal'

	@property
	def hash(self):
		r = _retry_on_fail(requests.head, self.download_url)
		if 'Location' in r.headers:
			return re.search(r'magnet:\?xt=urn:btih:(.*)&tr=', r.headers['Location']).group(1).upper()
		else:
			return None

{'fraction_non_alphanumeric': 0.16334098534345753, 'fraction_numerical': 0.010418506092177291, 'mean_word_length': 4.552941176470588, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1573094', 'n_tokens_mistral': 2239, 'n_tokens_neox': 2054, 'n_words': 486}