<commit_before>from dembones.collector import Collector
import dembones.urlvalidators as uv
import logging
import click


def initialise_logger(level):
    logging.basicConfig(level=level)
    return logging.getLogger("DEMBONES")


def determine_validator(vt):
    if vt == "same-domain":
        return uv.same_domain
    if vt == "same-domain-up-path":
        return uv.same_domain_up_path


@click.group()
def dembones():
    pass


@click.option("-mc,", "--max-concurrency", help="Max fetch tasks at any one time", default=5)
@click.option("-md", "--max-depth", help="Maximum recursion when collecting URLS", default=3)
@click.option("-vt", "--validate-targets", help="How to decide if we should recurse a link",
              type=click.Choice(["same-domain", "same-domain-up-path"]), default="same-domain")
@click.option("-v", "--verbose", help="Enable Verbose Mode", is_flag=True)
@click.argument("url")
def scrape(mc, md, vt, v, url):
    log = initialise_logger(logging.DEBUG if v else logging.INFO)

    validate = determine_validator(vt)

    log.info("Starting Collection")
    c = Collector(max_concurrent_fetches=mc, max_depth=md, target_validator=validate)
    c.start_collection(url)


# Simple helper for invoking in pycharm / ide for debugger
if __name__ == "__main__":
    scrape(5, 3, "same-domain", True, "https://blog.hartleybrody.com/")<commit_msg>working cli<commit_after>from dembones.collector import Collector
import dembones.urlvalidators as uv
import logging
import click


def initialise_logger(level):
    logging.basicConfig(level=level)
    return logging.getLogger("DEMBONES")


def determine_validator(vt):
    if vt == "same-domain":
        return uv.same_domain
    if vt == "same-domain-up-path":
        return uv.same_domain_up_path


@click.group()
def dembones():
    pass


@dembones.command()
@click.option("-mc,", "--max-concurrency", help="Max fetch tasks at any one time", default=5)
@click.option("-md", "--max-depth", help="Maximum recursion when collecting URLS", default=3)
@click.option("-tv", "--target-validator", help="How to decide if we should recurse a link",
              type=click.Choice(["same-domain", "same-domain-up-path"]), default="same-domain")
@click.option("-v", "--verbose", help="Enable Verbose Mode", is_flag=True)
@click.argument("url")
def scrape(max_concurrency, max_depth, target_validator, verbose, url):
    log = initialise_logger(logging.DEBUG if verbose else logging.INFO)

    validate = determine_validator(target_validator)

    log.info("Starting Collection")
    c = Collector(max_concurrent_fetches=max_concurrency, max_depth=max_depth, target_validator=validate)
    c.start_collection(url)


# Simple helper for invoking in pycharm / ide for debugger
def _debug_ide():
    scrape(5, 3, "same-domain", True, "https://blog.hartleybrody.com/")


if __name__ == "__main__":
    dembones()
{'fraction_non_alphanumeric': 0.14146680570038234, 'fraction_numerical': 0.002780674313521029, 'mean_word_length': 4.588349514563107, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6352694', 'n_tokens_mistral': 938, 'n_tokens_neox': 908, 'n_words': 224}