<commit_before>"""Utility methods for Python packages (written with puckfetcher in mind)."""

import logging
import os
import time
try:
    from urlparse import urlparse
except ImportError:
    # pylint: disable=no-name-in-module, import-error
    from urllib.parse import urlparse

import requests
from clint.textui import progress

LOG = logging.getLogger("root")

LAST_CALLED = {}

# I'm unsure if this the best way to do this.
# TODO don't support andrewmichaud.com, we should have proper local file support eventually, and
# run tests with that.
RATELIMIT_DOMAIN_WHITELIST = ["andrewmichaud.com"]


def generate_downloader(headers, args):
    """Create function to download with rate limiting and text progress."""

    def _downloader(url, dest, overwrite=True):
        # If there is a file with the name we intend to save to, assume the podcast has
        # been downloaded already.
        if os.path.isfile(dest) and not overwrite:
            LOG.info("File %s exists, not overwriting, assuming downloaded.", dest)
            print("File exists. Assuming already downloaded and skipping.")
            return

        domain = urlparse(url).netloc
        if domain == "":
            domain = "localhost"

        @rate_limited(domain, 30, args)
        def _rate_limited_download():
            response = requests.get(url, headers=headers, stream=True)
            LOG.info("Actually downloading from %s", url)
            LOG.info("Trying to save to %s", dest)

            total_length = int(response.headers.get("content-length"))
            expected_size = (total_length / 1024) + 1
            chunks = response.iter_content(chunk_size=1024)

            open(dest, "a").close()
            # per http://stackoverflow.com/a/20943461
            with open(dest, "wb") as stream:
                for chunk in progress.bar(chunks, expected_size=expected_size):
                    if not chunk:
                        return
                    stream.write(chunk)
                    stream.flush()

        _rate_limited_download()

    return _downloader


def max_clamp(val, limit):
    """Clamp int to limit."""
    return min(val, limit)


def expand(directory):
    """Apply expanduser and expandvars to directory to expand '~' and env vars."""
    temp1 = os.path.expanduser(directory)<commit_msg>Fix sub update failing due to missing directory.<commit_after>"""Utility methods for Python packages (written with puckfetcher in mind)."""

import logging
import os
import time
try:
    from urlparse import urlparse
except ImportError:
    # pylint: disable=no-name-in-module, import-error
    from urllib.parse import urlparse

import requests
from clint.textui import progress

LOG = logging.getLogger("root")

LAST_CALLED = {}

# I'm unsure if this the best way to do this.
# TODO don't support andrewmichaud.com, we should have proper local file support eventually, and
# run tests with that.
RATELIMIT_DOMAIN_WHITELIST = ["andrewmichaud.com"]


def generate_downloader(headers, args):
    """Create function to download with rate limiting and text progress."""

    def _downloader(url, dest, overwrite=True):

        domain = urlparse(url).netloc
        if domain == "":
            domain = "localhost"

        @rate_limited(domain, 30, args)
        def _rate_limited_download():
            # If there is a file with the name we intend to save to, assume the podcast has
            # been downloaded already.
            if os.path.isfile(dest) and not overwrite:
                LOG.info("File %s exists, not overwriting, assuming downloaded.", dest)
                print("File exists. Assuming already downloaded and skipping.")
                return

            # Create parent directory of file, and its parents, if they don't exist.
            parent = os.path.dirname(dest)
            if not os.path.exists(parent):
                os.makedirs(parent)

            response = requests.get(url, headers=headers, stream=True)
            LOG.info("Actually downloading from %s", url)
            LOG.info("Trying to save to %s", dest)

            total_length = int(response.headers.get("content-length"))
            expected_size = (total_length / 1024) + 1
            chunks = response.iter_content(chunk_size=1024)

            open(dest, "a").close()
            # per http://stackoverflow.com/a/20943461
            with open(dest, "wb") as stream:
                for chunk in progress.bar(chunks, expected_size=expected_size):
                    if not chunk:
                        return
                    stream.write(chunk)
                    stream.flush()

        _rate_limited_download()

    return _downloader


def max_clamp(val, limit):
    """Clamp int to limit."""
    return min(val, limit)


def expand(directory):
    """Apply expanduser and expandvars to directory to expand '~' and env vars."""
    temp1 = os.path.expanduser(directory)
{'fraction_non_alphanumeric': 0.09373729158194388, 'fraction_numerical': 0.008133387555917039, 'mean_word_length': 2.481245576786978, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6917325', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1299, 'n_words': 440}