<commit_before>import errno
import functools
import glob
import logging
import os
import shutil
from functools import partial
from multiprocessing.pool import ThreadPool
from typing import AnyStr
from typing import Dict
from typing import List
from typing import Optional
from urllib.parse import unquote
from urllib.parse import urljoin
from urllib.parse import urlparse

import requests
from bs4 import BeautifulSoup
from requests_file import FileAdapter

from maps_generator.utils.md5 import check_md5
from maps_generator.utils.md5 import md5_ext

logger = logging.getLogger("maps_generator")


def is_executable(fpath: AnyStr) -> bool:
    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)


@functools.lru_cache()
def find_executable(path: AnyStr, exe: Optional[AnyStr] = None) -> AnyStr:
    if exe is None:
        if is_executable(path):
            return path
        else:
            raise FileNotFoundError(path)
    find_pattern = f"{path}/**/{exe}"
    for name in glob.iglob(find_pattern, recursive=True):
        if is_executable(name):
            return name
    raise FileNotFoundError(f"{exe} not found in {path}")


def download_file(url: AnyStr, name: AnyStr, download_if_exists: bool = True):
    logger.info(f"Trying to download {name} from {url}.")
    if not download_if_exists and os.path.exists(name):
        logger.info(f"File {name} already exists.")
        return

    tmp_name = f"{name}__"
    os.makedirs(os.path.dirname(tmp_name), exist_ok=True)
    with requests.Session() as session:
        session.mount("file://", FileAdapter())
        with open(tmp_name, "wb") as handle:
            response = session.get(url, stream=True)
            file_length = int(response.headers["Content-Length"])
            current = 0
            max_attempts = 32
            attempts = max_attempts
            while attempts:
                for data in response.iter_content(chunk_size=4096):
                    current += len(data)
                    handle.write(data)

                if file_length == current:
                    break
<commit_msg>[generator] Fixed download_file()<commit_after>import errno
import functools
import glob
import logging
import os
import shutil
from functools import partial
from multiprocessing.pool import ThreadPool
from typing import AnyStr
from typing import Dict
from typing import List
from typing import Optional
from urllib.parse import unquote
from urllib.parse import urljoin
from urllib.parse import urlparse

import requests
from bs4 import BeautifulSoup
from requests_file import FileAdapter

from maps_generator.utils.md5 import check_md5
from maps_generator.utils.md5 import md5_ext

logger = logging.getLogger("maps_generator")


def is_executable(fpath: AnyStr) -> bool:
    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)


@functools.lru_cache()
def find_executable(path: AnyStr, exe: Optional[AnyStr] = None) -> AnyStr:
    if exe is None:
        if is_executable(path):
            return path
        else:
            raise FileNotFoundError(path)
    find_pattern = f"{path}/**/{exe}"
    for name in glob.iglob(find_pattern, recursive=True):
        if is_executable(name):
            return name
    raise FileNotFoundError(f"{exe} not found in {path}")


def download_file(url: AnyStr, name: AnyStr, download_if_exists: bool = True):
    logger.info(f"Trying to download {name} from {url}.")
    if not download_if_exists and os.path.exists(name):
        logger.info(f"File {name} already exists.")
        return

    tmp_name = f"{name}__"
    os.makedirs(os.path.dirname(tmp_name), exist_ok=True)
    with requests.Session() as session:
        session.mount("file://", FileAdapter())
        with open(tmp_name, "wb") as handle:
            response = session.get(url, stream=True)
            file_length = None
            try:
                file_length = int(response.headers["Content-Length"])
            except KeyError:
                logger.warning(
                    f"There is no attribute Content-Length in headers [{url}]: {response.headers}"
                )

            current = 0
            max_attempts = 32
            attempts = max_attempts
            while attempts:
                for data in response.iter_content(chunk_size=4096):
                    current += len(data)
                    handle.write(data)

                if file_length is None or file_length == current:
                    break

{'fraction_non_alphanumeric': 0.0867713004484305, 'fraction_numerical': 0.0053811659192825115, 'mean_word_length': 2.755050505050505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2754323', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1274, 'n_words': 373}