<jupyter_start><jupyter_text>02 - <NAME><jupyter_code>""" 
Crawling websites for URLs
"""
from urllib.request import urlopen, Request
import requests
from bs4 import BeautifulSoup
import datetime
import random
import re
import time

sec_timeout = 90*1   # 1 minute from now
while_timeout = time.time() + sec_timeout
random.seed(datetime.datetime.now())

def get_links(site, pageUrl):
    try:
        html = urlopen(Request('{0}{1}'.format(site, pageUrl), headers={'User-Agent': 'Webperf.se Crawler'}))
        bs = BeautifulSoup(html, 'html.parser')

        return bs.find_all('a', href=re.compile('^/')) # only URLs starting with /
    except Exception as e:
        print(site, pageUrl, '\n', e)
        return None

def check_for_redirect(url):
    try:
        r = requests.head(url, allow_redirects=True, timeout = 5)
        return r.url, r.history, r.status_code, r.headers['Content-Type']
    except:
        return None

def harvest_links(site = 'https://www.vgregion.se', initial_page = '/', max_pages = 50):
    links = get_links(site, initial_page)
    i = 0
    urls = []

    print('Looking for at most {} URLs at {} for {} seconds'.format(max_pages, site, sec_timeout))

    while len(links) > 0 and i < max_pages:
        newPage = links[random.randint(0, len(links)-1)].attrs['href']
        
        if site + newPage not in urls and 'mailto' not in newPage and '#' not in newPage and newPage != None and 'http' not in newPage and '.pdf' not in newPage and '.docx' not in newPage and '.pptx' not in newPage:
            
            check_redir = check_for_redirect(site + newPage)
            if(check_redir is not None and site in check_redir[0] and check_redir[2] == 200 and 'text/html' in check_redir[3]):
                i += 1
                print(i, newPage)
                urls.append(site + newPage)
                
                new_links = get_links(site, newPage)
                if new_links is not None:
                    links = new_links
            else:
                print('Redirection, content-type or status code error detected. URL skipped.\n', check_redir) # check_redir[0] do not work for all content-types
        
        if time.time() > while_timeout:
            break
    
    return urls

harvest_links(site='https://www.vgregion.se', max_pages=25)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11548331907613345, 'fraction_numerical': 0.011120615911035072, 'mean_word_length': 2.6040061633281972, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '104344', 'license': 'MIT', 'n_tokens_mistral': 762, 'n_tokens_neox': 725, 'n_words': 210, 'path': 'xx - samla adresser.ipynb', 'repo_name': 'Webperf-se/notebooks-diy', 'size': '2346'}