<commit_before>from bs4 import BeautifulSoup
import re, socket
from urllib.parse import urlparse
from urllib.parse import urljoin
import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse
start_url = "http://www.iitb.ac.in/"
unvisited = []
unvisited.append(start_url)
local = socket.gethostbyname(urlparse(start_url).netloc)
visited = []
pdfs = []
def crawl():
    global unvisited
    global visited
    global local
    website = unvisited.pop()
    visited.append(website)
    #-++779printlkpcc(website)
    req = urllib.request.Request(website)
    response = urllib.request.urlopen(req)
    print((response.getheader('Content-Type')))
    if  'text/html' in response.getheader('Content-Type'):
        page = response.read()
    else:
        return
    soup = BeautifulSoup(page, 'html.parser')
    #print((soup.prettify()))
    for link in soup.find_all('a'):        
        if "http" not in link:
            abs_link = urljoin(website, link.get("href"))
        else:
            abs_link = link.get('href')
        #print(abs_link)
        if abs_link not in visited:
            if re.search( '^.+\.pdf$', abs_link ):
                visited.append(abs_link)
                pdfs.append(abs_link)
            else:
                try:
                    if (abs_link not in unvisited) and (socket.gethostbyname(urlparse(abs_link).netloc) == local):
                        unvisited.append(abs_link)
                except:
                    print("Faltu IP")

while (len(unvisited) > 0):
    unvisited.sort()
    for link in unvisited:
        print(link)
    print((len(unvisited)))
    x = eval(input())
    crawl()
print("#############")
print(pdfs)<commit_msg>Prototype Ready<commit_after>from bs4 import BeautifulSoup
import re, socket
from urllib.parse import urlparse
from urllib.parse import urljoin
import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse
start_url = "http://www.iitr.ac.in/hi/departments/DPT/pages/Research+Publications.html"
unvisited = []
unvisited.append(start_url)
local = (urlparse(start_url).netloc)
# local = local.split(".")
# local = local[0]+"."+local[1]+"."+local[2]
visited = []
pdfs = []
def crawl():
    global unvisited
    global visited
    global local
    website = unvisited.pop()
    visited.append(website)
    #print(website)
    req = urllib.request.Request(website)
    try:
        response = urllib.request.urlopen(req)
    except:
        print("Not Found")
        return
    #print((response.getheader('Content-Type')))
    if  'text/html' in response.getheader('Content-Type'):
        page = response.read()
    else:
        return
    soup = BeautifulSoup(page, 'html.parser')
    #print((soup.prettify()))
    print(website)
    for link in soup.find_all('a'):        
        if "http" not in link:
            if "#" in link:
                return
            abs_link = urljoin(website, link.get("href"))
        else:
            abs_link = link.get('href')
        #print(abs_link)
        if abs_link not in visited:
            if re.search( '^.+\.pdf$', abs_link ):
                visited.append(abs_link)
                pdfs.append(abs_link)
            else:
                try:
                    ip = (urlparse(abs_link).netloc)
                    # ip = ip.split(".")
                    # ip = ip[0]+"."+ip[1]+"."+ip[2]
                    if (abs_link not in unvisited) and (ip == local):
                        unvisited.append(abs_link)
                except:
                    pass
i=0
while (len(unvisited)>0):
    # unvisited.sort()
    # for link in unvisited:
    #     print(link)
    # print((len(unvisited)))
    crawl()
    i=i+1
    print(len(pdfs), len(unvisited))
print("#############")
print(pdfs)
{'fraction_non_alphanumeric': 0.13882290841910794, 'fraction_numerical': 0.00395882818685669, 'mean_word_length': 2.270060396893874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1294038', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1224, 'n_words': 250}