<commit_before>#!/usr/bin/env python
#
#  local crawler PoC
#  basic and simple uri crawler for web pages
#  by Ark, just4fun
#  feel free to share =)
#

import re
import random
import requests
import sys
import codecs

def main():

    if len(sys.argv) != 2:
        print("Usage: {} <start_uri>".format(sys.argv[0]))
        return 1

    target_list = [codecs.encode(sys.argv[1], 'utf-8').strip(b'/')]
    external_r = re.compile(b"(https?://[\w]+\.[\w\.\-_/%]+)")
    internal_r = re.compile(b"<.*[href|src]=['\"]([\w/\\\-_\.])['\"].*>")
    base_r = re.compile(b"https?://([\w]+\.[\w\.\-_]+)")
    x = 0

    base = base_r.findall(target_list[0])[0]
    while True:
        try:
            print(target_list[x])
            data = requests.request("GET", target_list[x], timeout=1)

            tmpbase = target_list[x][:target_list[x].rfind(b'/') + 1]
            if tmpbase == b'http://' or tmpbase == b'https://':
                tmpbase = target_list[x] + b'/'
            for uri in internal_r.findall(data.content):
                new_uri = b''.join((tmpbase, uri)).strip(b'/')
                try:
                    if uri.strip(b'/') not in target_list and base_r.findall(uri)[0] == base:
                        target_list.append(new_uri)
                except:
                    if new_uri.strip(b'/') not in target_list:
                        target_list.append(new_uri)

            for uri in external_r.findall(data.content):
                if uri.strip(b'/') not in target_list and base_r.findall(uri)[0] == base:<commit_msg>corrected bug if link is dead<commit_after>#!/usr/bin/env python
#
#  local crawler PoC
#  basic and simple uri crawler for web pages
#  by Ark, just4fun
#  feel free to share =)
#

import re
import random
import requests
import sys
import codecs

def main():

    if len(sys.argv) != 2:
        print("Usage: {} <start_uri>".format(sys.argv[0]))
        return 1

    target_list = [codecs.encode(sys.argv[1], 'utf-8').strip(b'/')]
    external_r = re.compile(b"(https?://[\w]+\.[\w\.\-_/%]+)")
    internal_r = re.compile(b"<.*[href|src]=['\"]([\w/\\\-_\.])['\"].*>")
    base_r = re.compile(b"https?://([\w]+\.[\w\.\-_]+)")
    x = 0

    base = base_r.findall(target_list[0])[0]
    while True:
        try:
            data = requests.request("GET", target_list[x], timeout=1)
            if not data.ok:
                continue
            print(target_list[x])

            tmpbase = target_list[x][:target_list[x].rfind(b'/') + 1]
            if tmpbase == b'http://' or tmpbase == b'https://':
                tmpbase = target_list[x] + b'/'
            for uri in internal_r.findall(data.content):
                new_uri = b''.join((tmpbase, uri)).strip(b'/')
                try:
                    if uri.strip(b'/') not in target_list and base_r.findall(uri)[0] == base:
                        target_list.append(new_uri)
                except:
                    if new_uri.strip(b'/') not in target_list:
                        target_list.append(new_uri)

            for uri in external_r.findall(data.content):
                if uri.strip(b'/') not in target_list and base_r.findall(uri)[0] == base:
{'fraction_non_alphanumeric': 0.1785602011945929, 'fraction_numerical': 0.00817353033637221, 'mean_word_length': 1.993414863593603, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1315250', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1076, 'n_words': 224}