<commit_before>from html.parser import HTMLParser
import urllib.request

class MyHTMLParser(HTMLParser):

	links = []


	def handle_starttag(self, tag, attrs):
		if str(tag) == 'a':
			for attr in attrs:
				if attr == 'href':
					for inn in attrs:
						if 'http' in inn:
							print(inn)
							self.links.append(inn)

#Inicialização
response = urllib.request.urlopen('https://www.comidaereceitas.com.br/bolos/bolinho-de-chuva-pratico.html')
html = response.read().decode('utf-8')
parser = MyHTMLParser()
parser.feed(html)<commit_msg>Incluid array único, e processo de "caminhar" pelas urls limitado<commit_after>from html.parser import HTMLParser
import urllib.request

class MyHTMLParser(HTMLParser):

	links = []


	def handle_starttag(self, tag, attrs):
		if str(tag) == 'a':
			for attr in attrs:
				for inn in attr:
					if 'http' in inn and 'comidaereceitas.com' in inn and not 'whatsapp' in inn and not 'facebook' in inn:
						self.push(inn)

	def push(self, link):
		for ilink in self.links:
			if ilink == link:
				return

		self.links.append(link)



#Inicialização
response = urllib.request.urlopen('https://www.comidaereceitas.com.br/bolos/bolinho-de-chuva-pratico.html')
html = response.read().decode('utf-8')
parser = MyHTMLParser()
parser.feed(html)
acessos = 50
i = 0
while acessos > 0:
	link = parser.links[i]
	print(link)
	print('acessos ' + str(acessos))
	response = urllib.request.urlopen(link)
	html = response.read().decode('utf-8')
	parser.feed(html)
	i += 1
	acessos -= 1
	
print('links encontrados' + str(len(parser.links)))
{'fraction_non_alphanumeric': 0.13744380218368657, 'fraction_numerical': 0.005780346820809248, 'mean_word_length': 4.042071197411003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6224292', 'n_tokens_mistral': 629, 'n_tokens_neox': 564, 'n_words': 136}