<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import errno
import os
import sys
try:  # Python 3
    from urllib import robotparser
    from urllib.parse import urlparse
except ImportError:  # Python 2
    import robotparser
    from urlparse import urlparse

import bs4
import requests


def main(argv=None):
    if argv is None:
        argv = sys.argv
    try:
        url = argv[1]
    except IndexError:
        print("Usage: {} URL".format(sys.argv[0]))
        sys.exit(1)
    basename = urlparse(url).hostname
    try:
        os.makedirs(basename)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(basename):
            pass
        else:
            raise
    print(process_url(url))


def process_url(url):
    if not can_robots_fetch(url):
        return []
    basename = urlparse(url).hostname
    response = requests.get(url)
    filename = os.path.join(basename, '__root__')
    if response.headers['content-type'] == 'text/html':
        filemode = 'w'
        file_content = response.text
        links = links_from_data(response.text)
    else:
        filemode = 'wb'
        file_content = response.content
        links = []
    with open(filename, filemode) as f:
        f.write(file_content)
    return links


def links_from_data(data):
    bs = bs4.BeautifulSoup(data)
    result = []
    for tag in bs.find_all('a'):
        result.append(tag['href'])
    return result


def can_robots_fetch(url):
    parsed_url = urlparse(url)
    robots_url_obj = parsed_url._replace(path='/robots.txt')
    robots_url = robots_url_obj.geturl()
    rp = robotparser.RobotFileParser(robots_url)<commit_msg>pythonify list creation<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import errno
import os
import sys
try:  # Python 3
    from urllib import robotparser
    from urllib.parse import urlparse
except ImportError:  # Python 2
    import robotparser
    from urlparse import urlparse

import bs4
import requests


def main(argv=None):
    if argv is None:
        argv = sys.argv
    try:
        url = argv[1]
    except IndexError:
        print("Usage: {} URL".format(sys.argv[0]))
        sys.exit(1)
    basename = urlparse(url).hostname
    try:
        os.makedirs(basename)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(basename):
            pass
        else:
            raise
    print(process_url(url))


def process_url(url):
    if not can_robots_fetch(url):
        return []
    basename = urlparse(url).hostname
    response = requests.get(url)
    filename = os.path.join(basename, '__root__')
    if response.headers['content-type'] == 'text/html':
        filemode = 'w'
        file_content = response.text
        links = links_from_data(response.text)
    else:
        filemode = 'wb'
        file_content = response.content
        links = []
    with open(filename, filemode) as f:
        f.write(file_content)
    return links


def links_from_data(data):
    bs = bs4.BeautifulSoup(data)
    result = [tag['href'] for tag in bs.find_all('a')]
    return result


def can_robots_fetch(url):
    parsed_url = urlparse(url)
    robots_url_obj = parsed_url._replace(path='/robots.txt')
    robots_url = robots_url_obj.geturl()
    rp = robotparser.RobotFileParser(robots_url)
{'fraction_non_alphanumeric': 0.10752038659015403, 'fraction_numerical': 0.004832376925400181, 'mean_word_length': 2.6117775354416577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '188219', 'n_tokens_mistral': 1107, 'n_tokens_neox': 1077, 'n_words': 253}