<jupyter_start><jupyter_text>Crawler Web pages<jupyter_code># load dependency libraries
import requests
import os
import re
import pickle
from bs4 import BeautifulSoup
from collections import deque
domain = "uic.edu"
start_url = "https://cs.uic.edu"                

pages_folder = "../FetchedPages/"

# file extensions to ignore while crawling pages
ignore_ext = [
    '.pdf', '.doc', '.docx', '.ppt', '.pptx', '.xls', '.xlsx', '.css', '.js',
    '.aspx', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.mp4',
    '.avi', '.tar', '.gz', '.tgz', '.zip'
]

crawl_limit = 6000

# to make sure error log file is initially empty
error_file = "../error_log.txt"
f = open(error_file, "w+")
# f.truncate()
f.close()

# queue to perform BFS web traversal
url_q = deque()
url_q.append(start_url)

# list to keep track of traversed URLs
urls_crawled = []
urls_crawled.append(start_url)

# dict to track pages fetched and stored in folder
pages_crawled = {}
page_no = 0

while url_q:
    
    try:
        url = url_q.popleft()               # fetch the first URL from the queue
        rqst = requests.get(url)            # get html code of web page

        if (rqst.status_code == 200):
            
            soup = BeautifulSoup(rqst.text, 'lxml')
            tags_extracted = soup.find_all('a')                 # extract all 'a' tags from page
            
            if len(tags_extracted) != 0:                        # to reject pages which don't link to another page
                pages_crawled[page_no] = url    

                output_file = pages_folder + str(page_no)

                os.makedirs(os.path.dirname(output_file), exist_ok=True)     # create file to store html code

                with open(output_file, "w", encoding="utf-8") as file:
                    file.write(rqst.text)
                file.close()

                for tag in tags_extracted:

                    link = tag.get('href')                  

                    if link is not None and link.startswith("http") and not any(ext in link.lower() for ext in ignore_ext):
                        
                        link = link.lower()

                        link = link.split('#')[0]
                        link = link.split('?', maxsplit=1)[0]
                        link = link.rstrip('/')
                        link = link.strip()

                        if link not in urls_crawled and domain in link:
#                             print(link)
                            url_q.append(link)                 # valid URL to append to the queue
                            urls_crawled.append(link)

                if (len(pages_crawled) > crawl_limit):
                    break                                       # stop crawling when reached limit

                page_no += 1
            

    except Exception as e:
        with open(error_file, "a+") as log:                  # add error message to error log
            log.write(f"Could not connect to {url}")
            log.write(f"\nError occured: {e}\n\n")
        file.close()

        print("Could not connect to ", url)
        print("Error occured: ", e, " \n")
        continue
pickle_folder = "../PickleFiles/"
os.makedirs(pickle_folder, exist_ok=True)

# Pickling the dict of crawled pages
with open(pickle_folder + '6000_pages_crawled.pickle', 'wb') as f:
    pickle.dump(pages_crawled,f)
len(pages_crawled)
pickle_folder = "../PickleFiles/"
os.makedirs(pickle_folder, exist_ok=True)
with open(pickle_folder + '6000_pages_crawled.pickle', 'rb') as f:
    pages = pickle.load(f)
pages_crawled == pages<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11464968152866242, 'fraction_numerical': 0.006646358349487676, 'mean_word_length': 1.7827426810477658, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '519354', 'license': 'MIT', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1067, 'n_words': 302, 'path': 'JupyterNBs/Crawler.ipynb', 'repo_name': 'samujjwaal/UIC-search-engine', 'size': '3611'}