<commit_before>from bs4 import BeautifulSoup
import requests
import os
import time
import sys

class Crawler(object):

	def __init__(self, url, directory, quiet, limit = 1):
		self.__baseUrl = url + '?p[order]=23&p[page]='
		# add '/' to end of directory name if it doesn't exists
		if directory.endswith('/'):
			self.__dir = directory
		else:
			self.__dir = directory + '/'
		# add current working directory to start of the path if '/' doesn't exists
		if not self.__dir.startswith('/'):
			self.__dir = os.getcwd() + '/' + self.__dir
		self.__quiet = quiet
		self.__limit = limit
		self.__urlList = []

	def log(self, string):
		if self.__quiet != 1:
			print(string)
		else:
			return

	def httpRequest(self, req, t = 5):
		while True:
			try:
				return requests.get(req, timeout = t)
			except requests.exceptions.HTTPError as err:
				print(err)
				print('retrying...')
				time.sleep(t)
				continue
			except requests.exceptions.ConnectionError as err:
				print(err)
				print('retrying...')
				time.sleep(t)
				continue
			except requests.exceptions.Timeout as err:
				print(err)
				print('retrying...')
				time.sleep(t)
				continue

	def getUrlList(self):
		self.log('Obtaining URLs...')
		i = 1
		for page in range(1, self.__limit + 1):
			response = self.httpRequest(self.__baseUrl + str(page))
			html = response.text
			soup = BeautifulSoup(html, 'html.parser')
			for div in soup.findAll('div', {'class': 'item-heading'}):
				link = div.find('a')
				href = link.get('href')
				title = link.string
				self.__urlList.append(href)
				self.log('Otained url [' + str(i) + ']: ' + href)
				i += 1
			# time.sleep(0.2)

	def downloadContent(self):
		self.log('Downloading HTML files...')
		prefix = 1
		for url in self.__urlList:
			response = self.httpRequest(url)
			html = response.text
			soup = BeautifulSoup(html, 'html.parser')
			parsed_url = url.split('/')
			title = parsed_url[4]
			file = self.__dir + str(prefix) + ' - ' + title
			if not os.path.exists(file):
				handle = open(file, "w")
				self.log('Downloaded file [' + str(prefix) + ']: ' + file)
				handle.write(html)
				handle.close()
			prefix += 1
			# time.sleep(0.2)<commit_msg>edit start_from and limit<commit_after>from bs4 import BeautifulSoup
import requests
import os
import time
import sys
import io

class Crawler(object):

	def __init__(self, url, directory, quiet, limit = 1, startFrom = 1):
		self.__baseUrl = url
		'''
		# add '/' to end of directory name if it doesn't exists
		if directory.endswith('/'):
			self.__dir = directory
		else:
			self.__dir = directory + '/'
		# add current working directory to start of the path if '/' doesn't exists
		if not self.__dir.startswith('/'):
			self.__dir = os.getcwd() + '/' + self.__dir
		'''
		# add '/' to end of directory name if it doesn't exists
		if directory.endswith('\\'):
			self.__dir = directory
		else:
			self.__dir = directory + '\\'
		# add current working directory to start of the path if '/' doesn't exists
		if not self.__dir.startswith('\\'):
			self.__dir = os.getcwd() + '\\' + self.__dir
		self.__quiet = quiet
		self.__limit = limit
		self.__urlList = []
		self.__from = startFrom

	def log(self, string):
		if self.__quiet != 1:
			print(string)
		else:
			return

	def httpRequest(self, req, t = 5):
		while True:
			try:
				return requests.get(req, timeout = t)
			except requests.exceptions.HTTPError as err:
				print(err)
				print('retrying...')
				time.sleep(t)
				continue
			except requests.exceptions.ConnectionError as err:
				print(err)
				print('retrying...')
				time.sleep(t)
				continue
			except requests.exceptions.Timeout as err:
				print(err)
				print('retrying...')
				time.sleep(t)
				continue

	def getUrlList(self):
		self.log('Obtaining URLs...')
		i = 1
		for page in range(self.__from, self.__from + self.__limit):
			response = self.httpRequest(self.__baseUrl + str(page))
			html = response.text
			soup = BeautifulSoup(html, 'html.parser')
			for div in soup.findAll('div', {'class': 'item-heading'}):
				link = div.find('a')
				href = link.get('href')
				title = link.string
				self.__urlList.append(href)
				self.log('Otained url [' + str(i) + ']: ' + href)
				i += 1
			# time.sleep(0.2)

	def downloadContent(self):
		self.log('Downloading HTML files...')
		encoding = 'utf-8'
		prefix = (self.__from - 1) * 20 + 1
		for url in self.__urlList:
			response = self.httpRequest(url)
			html = response.text
			soup = BeautifulSoup(html, 'html.parser')
			parsed_url = url.split('/')
			title = parsed_url[4]
			file = self.__dir + str(prefix) + ' - ' + title
			if not os.path.exists(file):
				with io.open(file, 'w', encoding = encoding) as handle:
					self.log('Downloaded file [' + str(prefix) + ']: ' + file)
					handle.write(html)
					handle.close()
			prefix += 1
			# time.sleep(0.2)
{'fraction_non_alphanumeric': 0.1477414007459594, 'fraction_numerical': 0.0072523829258184834, 'mean_word_length': 3.0393305439330542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4580992', 'n_tokens_mistral': 1948, 'n_tokens_neox': 1619, 'n_words': 431}