<commit_before>#!/usr/bin/env python

from argparse import ArgumentParser
from urllib import request
from pprint import pprint
import sys
from bs4 import BeautifulSoup

__version__ = '1.0.0'

def parse_args(args):
    """Parse the command line parameters."

    Returns:
        An instance of argparse.ArgumentParser that stores the command line
        parameters.
    """

    parser = ArgumentParser(description='href extractor')
    parser.add_argument('--version', action='version', version=__version__)

    parser.add_argument(
        '--format',
        default='pprint',
        help='output format'
    )

    parser.add_argument(
        'url',
        help='URL from which to extract href values')

    return parser.parse_args(args or [])

def run(args):
    """The main function.

    Args:
        args: An instance of argparse.ArgumentParser parsed in the configure
        function.

    Returns:
        None.
    """

    url = args.url
    # download the file
    with request.urlopen(url) as fin:
        data = fin.read()

    # create an instance of bs
    bs = BeautifulSoup(data, 'html.parser')
    links = bs.find_all('a')
    processed = [(i.get('href'), i.text) for i in links]

    if args.format == 'pprint':
        pprint(processed)
    elif args.format == 'csv':
        print('URL\ttext')
        for i in processed:
            print('\t'.join(i))

def main(args=sys.argv[1:]):
    sys.exit(run(parse_args(args)))

if __name__ == "__main__":
    main()<commit_msg>(extracthref.py) Improve with class SoupStrainer<commit_after>#!/usr/bin/env python

from argparse import ArgumentParser
from urllib import request
import sys
from bs4 import BeautifulSoup, SoupStrainer

__version__ = '1.0.0'

def parse_args(args):
    """Parse the command line parameters."

    Returns:
        An instance of argparse.ArgumentParser that stores the command line
        parameters.
    """

    parser = ArgumentParser(description='href extractor')
    parser.add_argument('--version', action='version', version=__version__)

    parser.add_argument(
        '--format',
        default='pprint',
        help='output format'
    )

    parser.add_argument(
        'url',
        help='URL from which to extract href values')

    return parser.parse_args(args or [])

def run(args):
    """The main function.

    Args:
        args: An instance of argparse.ArgumentParser parsed in the configure
        function.

    Returns:
        None.
    """

    url = args.url
    # download the file
    with request.urlopen(url) as fin:
        data = fin.read()

    # create an instance of bs
    links = BeautifulSoup(data, 'html.parser', parse_only=SoupStrainer("a"))
    processed = [(i.get('href'), ''.join(i.stripped_strings)) for i in links]

    if args.format == 'pprint':
        print(links.prettify())

    elif args.format == 'csv':
        print('URL\ttext')
        for i in processed:
            print('\t'.join(i))

def main(args=sys.argv[1:]):
    sys.exit(run(parse_args(args)))

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.12037945698397122, 'fraction_numerical': 0.0032711808963035655, 'mean_word_length': 2.6977025392986698, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6561148', 'n_tokens_mistral': 944, 'n_tokens_neox': 914, 'n_words': 257}