<commit_before>""""Vendoring script, python 3.5 with requests needed"""

# The following comment should be removed at some point in the future.
# mypy: disallow-untyped-defs=False

import os
import re
import shutil
import tarfile
import zipfile
from pathlib import Path

import invoke
import requests

FILE_WHITE_LIST = (
    'Makefile',
    'vendor.txt',
    '__init__.py',
    'README.rst',
)

# libraries that have directories with different names
LIBRARY_DIRNAMES = {
    'setuptools': 'pkg_resources',
    'msgpack-python': 'msgpack',
}

# from time to time, remove the no longer needed ones
HARDCODED_LICENSE_URLS = {
    'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE',
    'webencodings': 'https://github.com/SimonSapin/python-webencodings/raw/'
                    'master/LICENSE',
}


def drop_dir(path, **kwargs):
    shutil.rmtree(str(path), **kwargs)


def remove_all(paths):
    for path in paths:
        if path.is_dir():
            drop_dir(path)
        else:
            path.unlink()<commit_msg>Make it clearer where licenses are coming from<commit_after>""""Vendoring script, python 3.5 with requests needed"""

# The following comment should be removed at some point in the future.
# mypy: disallow-untyped-defs=False

import os
import re
import shutil
import tarfile
import zipfile
from pathlib import Path

import invoke
import requests

FILE_WHITE_LIST = (
    'Makefile',
    'vendor.txt',
    '__init__.py',
    'README.rst',
)

# libraries that have directories with different names
LIBRARY_DIRNAMES = {
    'setuptools': 'pkg_resources',
    'msgpack-python': 'msgpack',
}

# from time to time, remove the no longer needed ones
_github_license = "https://github.com/{}/raw/master/LICENSE"
HARDCODED_LICENSE_URLS = {
    'pytoml': _github_license.format('avakar/pytoml'),
    'webencodings': _github_license.format('SimonSapin/python-webencodings')
}


def drop_dir(path, **kwargs):
    shutil.rmtree(str(path), **kwargs)


def remove_all(paths):
    for path in paths:
        if path.is_dir():
            drop_dir(path)
        else:
            path.unlink()
{'fraction_non_alphanumeric': 0.11385199240986717, 'fraction_numerical': 0.0018975332068311196, 'mean_word_length': 3.5748373101952278, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2351115', 'n_tokens_mistral': 701, 'n_tokens_neox': 692, 'n_words': 183}