<commit_before>from page import Page

def crawl_web(seed, max_depth = 10, max_pages = 1000):
	to_crawl, crawled = {seed}, set()
	index, graph = {}, {}

	def crawl(url, depth):
		if hasattr(crawl, 'pages'):
			crawl.pages += 1
		else:
			crawl.pages = 1

		if crawl.pages >= max_pages or depth >= max_depth or url in crawled or not url:
			crawl.pages -= 1
			return

		crawled.add(url)
		page = Page(url)
		if not page.is_valid():
			return
		index_page(index, page)
		graph[url] = page.links
		outgoing_links = page.links
		while outgoing_links or to_crawl:
			if outgoing_links:
				next_url = outgoing_links.pop()
				next_depth = depth + 1
			else:
				next_url = to_crawl.pop()
				next_depth = depth

			to_crawl.update(outgoing_links)
			crawl(next_url, next_depth)

	crawl(seed, 0)
	return (index, graph)

def index_page(index, page):
	words = page.content.split()
	for word in words:
		if word in index:
			index[word].add(page.url)
		else:
			index[word] = {page.url}

# Page Rank Algorithm where d is the damping constant<commit_msg>changed implementation of crawl_web<commit_after>from page import Page
from collections import namedtuple
import heapq
import itertools

"""
A tuple which contains
priority: the priority of the set of links in the pq (not unique)
id: the unique id of the set of links, used for tiebreakers
links: set of links
"""

Links = namedtuple('Links', ['priority', 'id', 'links'])
def crawl_web(seed, max_depth = 10, max_pages = 1000):
	crawled = set()
	crawl_queue =  [] # priority queue ensures that more "shallow" links are handled first
	index = {}
	graph = {}
	counter = itertools.count()

	"""
	Add set of links to queue of sets, crawled_queue.
	Makes sure links is not in the set of already crawled urls.
	"""
	def add_links(links, depth = 0):
		count = next(counter)
		new_links = links.difference(crawled)
		entry = Links(priority = depth, id = count, links = new_links)
		heapq.heappush(crawl_queue, entry)

	"""
	Adds all of the words in page.content to the index of words
	to sets of urls
	"""
	def index_page(page):
		words = page.content.split()
		for word in words:
			if word in index:
				index[word.lower()].add(page.url)
			else:
				index[word.lower()] = {page.url}

	add_links({seed}, 0)
	pages = 0
	while crawl_queue:
		entry = heapq.heappop(crawl_queue)
		to_crawl = entry.links
		depth = entry.priority
		while to_crawl and pages < max_pages:
			url = to_crawl.pop()
			page = Page(url)
			if page.is_valid() and not url in crawled:
				print url, depth
				pages += 1
				crawled.add(url)
				index_page(page)
				graph[url] = page.outgoing_links
				if depth < max_depth:
					add_links(page.outgoing_links, depth + 1)
	return index, graph

"""
Page Rank Algorithm where d is the damping constant, and the
calculates p_k_rank for each p in graph, where k is max_iterations
"""
{'fraction_non_alphanumeric': 0.1060979908353895, 'fraction_numerical': 0.007754670426506873, 'mean_word_length': 3.690909090909091, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6457865', 'n_tokens_mistral': 1128, 'n_tokens_neox': 969, 'n_words': 319}