<commit_before>import hashlib
import http.server
import json
import logging
import os
import platform
import re
import shutil
import socketserver
import stat
import tarfile
import tempfile
from contextlib import contextmanager, ExitStack
from itertools import chain
from multiprocessing import Process
from shutil import rmtree
from typing import List

import requests
import retrying
import teamcity
import yaml
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from teamcity.messages import TeamcityServiceMessages

from pkgpanda import subprocess
from pkgpanda.exceptions import FetchError, IncompleteDownloadError, ValidationError

log = logging.getLogger(__name__)
is_windows = platform.system() == "Windows"


def is_absolute_path(path):
    if is_windows:
        # We assume one char drive letter. Sometimes its two but not often
        # pattern is <driveletter>:/string....
        if path[1] == ':':
            return True
    else:
        if path[0] == '/':
            return True
    return False


def remove_file(path):
    """removes a file. fails silently if the file does not exist"""
    if is_windows:
        # python library on Windows does not like symbolic links in directories
        # so calling out to the cmd prompt to do this fixes that.
        path = path.replace('/', '\\')
        if os.path.exists(path):
            subprocess.call(['cmd.exe', '/c', 'del', '/q', path])
    else:
        subprocess.check_call(['rm', '-f', path])


def remove_directory(path):
    """recursively removes a directory tree. fails silently if the tree does not exist"""
    if is_windows:
        # python library on Windows does not like symbolic links in directories
        # so calling out to the cmd prompt to do this fixes that.
        path = path.replace('/', '\\')
        if os.path.exists(path):
            subprocess.call(['cmd.exe', '/c', 'rmdir', '/s', '/q', path])
    else:
        subprocess.check_call(['rm', '-rf', path])


def make_directory(path):
    """Create a directory, creating intermediate directories if necessary"""
    if is_windows:
        path = path.replace('/', '\\')

    if not os.path.exists(path):
        os.makedirs(path)


def copy_file(src_path, dst_path):
    """copy a single directory item from one location to another"""
    if is_windows:
        # To make sure the copy works we are using cmd version as python
        # libraries may not handle symbolic links and other things that are
        # thrown at it.
        src = src_path.replace('/', '\\')
        dst = dst_path.replace('/', '\\')
        subprocess.check_call(['cmd.exe', '/c', 'copy', src, dst])
    else:
        subprocess.check_call(['cp', src_path, dst_path])


def copy_directory(src_path, dst_path):
    """copy recursively a directory tree from one location to another"""
    if is_windows:
        # To make sure the copy works we are using cmd version as python
        # libraries may not handle symbolic links and other things that are
        # thrown at it.
        src = src_path.replace('/', '\\')
        dst = dst_path.replace('/', '\\')
        subprocess.check_call(['cmd.exe', '/c', 'xcopy', src, dst, '/E', '/B', '/I'])
    else:
        subprocess.check_call(['cp', '-r', src_path, dst_path])


def variant_str(variant):
    """Return a string representation of variant."""
    if variant is None:
        return ''
    return variant


def variant_object(variant_str):
    """Return a variant object from its string representation."""
    if variant_str == '':
        return None
    return variant_str


def variant_name(variant):
    """Return a human-readable string representation of variant."""
    if variant is None:
        return '<default>'
    return variant


def variant_prefix(variant):
    """Return a filename prefix for variant."""
    if variant is None:
        return ''
    return variant + '.'


def variant_suffix(variant, delim='.'):
    if variant is None:
        return ''
    return delim + variant


def get_requests_retry_session(max_retries=4, backoff_factor=1, status_forcelist=None):
    status_forcelist = status_forcelist or [500, 502, 504]
    # Default max retries 4 with sleeping between retries 1s, 2s, 4s, 8s
    session = requests.Session()
    custom_retry = Retry(total=max_retries,
                         backoff_factor=backoff_factor,
                         status_forcelist=status_forcelist)
    custom_adapter = HTTPAdapter(max_retries=custom_retry)
    # Any request through this session that starts with 'http://' or 'https://'
    # will use the custom Transport Adapter created which include retries
    session.mount('http://', custom_adapter)
    session.mount('https://', custom_adapter)
    return session


def _is_incomplete_download_error(exception):
    return isinstance(exception, IncompleteDownloadError)


@retrying.retry(
    stop_max_attempt_number=3,
    wait_random_min=1000,
    wait_random_max=2000,
    retry_on_exception=_is_incomplete_download_error)
def _download_remote_file(out_filename, url):
    with open(out_filename, "wb") as f:
        r = get_requests_retry_session().get(url, stream=True)
        r.raise_for_status()

        total_bytes_read = 0
        for chunk in r.iter_content(chunk_size=4096):
            f.write(chunk)
            total_bytes_read += len(chunk)

        if 'content-length' in r.headers:
            content_length = int(r.headers['content-length'])
            if total_bytes_read != content_length:
                raise IncompleteDownloadError(url, total_bytes_read, content_length)

        return r


def download(out_filename, url, work_dir, rm_on_error=True):
    assert os.path.isabs(out_filename)
    assert os.path.isabs(work_dir)
    work_dir = work_dir.rstrip('/')

    # Strip off whitespace to make it so scheme matching doesn't fail because
    # of simple user whitespace.
    url = url.strip()

    # Handle file:// urls specially since requests doesn't know about them.
    try:
        if url.startswith('file://'):
            src_filename = url[len('file://'):]
            if not os.path.isabs(src_filename):
                src_filename = work_dir + '/' + src_filename
            shutil.copyfile(src_filename, out_filename)
        else:
            _download_remote_file(out_filename, url)
    except Exception as fetch_exception:
        if rm_on_error:
            rm_passed = False

            # try / except so if remove fails we don't get an exception during an exception.
            # Sets rm_passed to true so if this fails we can include a special error message in the
            # FetchError
            try:
                os.remove(out_filename)
                rm_passed = True
            except Exception:
                pass
        else:
            rm_passed = True

        raise FetchError(url, out_filename, fetch_exception, rm_passed) from fetch_exception


def download_atomic(out_filename, url, work_dir):
    assert os.path.isabs(out_filename)
    tmp_filename = out_filename + '.tmp'
    try:
        download(tmp_filename, url, work_dir)
        shutil.move(tmp_filename, out_filename)
    except FetchError:
        try:
            os.remove(tmp_filename)
        except:
            pass
        raise


def extract_tarball(path, target):
    """Extract the tarball into target.

    If there are any errors, delete the folder being extracted to.
    """
    # TODO(cmaloney): Validate extraction will pass before unpacking as much as possible.
    # TODO(cmaloney): Unpack into a temporary directory then move into place to
    # prevent partial extraction from ever laying around on the filesystem.
    try:
        assert os.path.exists(path), "Path doesn't exist but should: {}".format(path)
        make_directory(target)

        # TODO(tweidner): https://jira.mesosphere.com/browse/DCOS-48220
        # Make this cross-platform via Python's tarfile module once
        # https://bugs.python.org/issue21872 is fixed.
        if is_windows:
            subprocess.check_call(['bsdtar', '-xf', path, '-C', target])
        else:
            subprocess.check_call(['tar', '-xf', path, '-C', target])

    except:
        # If there are errors, we can't really cope since we are already in an error state.
        rmtree(target, ignore_errors=True)
        raise


def load_json(filename):
    try:
        with open(filename) as f:
            return json.load(f)
    except ValueError as ex:
        raise ValueError("Invalid JSON in {0}: {1}".format(filename, ex)) from ex


class YamlParseError(Exception):
    pass


def load_yaml(filename):
    try:
        with open(filename) as f:
            return yaml.safe_load(f)
    except yaml.YAMLError as ex:
        raise YamlParseError("Invalid YAML in {}: {}".format(filename, ex)) from ex


def write_yaml(filename, data, **kwargs):
    dumped_yaml = yaml.safe_dump(data, **kwargs)
    write_string(filename, dumped_yaml)


def make_file(name):
    with open(name, 'a'):
        pass


def write_json(filename, data):
    dumped_json = json_prettyprint(data=data)
    write_string(filename, dumped_json)


def write_string(filename, data):
    """
    Write a string to a file.
    Overwrite any data in that file.

    We use an atomic write practice of creating a temporary file and then
    moving that temporary file to the given ``filename``. This prevents race
    conditions such as the file being read by another process after it is
    opened here but not yet written to.

    It also prevents us from creating or truncating a file before we fail to
    write data to it because of low disk space.

    If no file already exists at ``filename``, the new file is created with
    permissions 0o644.
    """
    prefix = os.path.basename(filename)
    tmp_file_dir = os.path.dirname(os.path.realpath(filename))
    fd, temporary_filename = tempfile.mkstemp(prefix=prefix, dir=tmp_file_dir)

    try:
        permissions = os.stat(filename).st_mode
    except FileNotFoundError:
        log.debug("File %s does not exist, creating", filename)
        permissions = 0o644

    try:
        try:
            os.write(fd, data.encode())
        finally:
            os.close(fd)
        os.chmod(temporary_filename, stat.S_IMODE(permissions))
        os.replace(temporary_filename, filename)
    except Exception:
        os.remove(temporary_filename)
        raise


def load_string(filename):
    with open(filename) as f:
        return f.read().strip()


def json_prettyprint(data):
    return json.dumps(
        data,
        sort_keys=True,
        indent=2,
        separators=(',', ':'),
    )


def if_exists(fn, *args, **kwargs):
    try:
        return fn(*args, **kwargs)
    except FileNotFoundError as e:
        log.debug(e)
        return None


def sha1(filename):
    hasher = hashlib.sha1()

    with open(filename, 'rb') as fh:
        while 1:
            buf = fh.read(4096)
            if not buf:
                break
            hasher.update(buf)

    return hasher.hexdigest()


def expect_folder(path, files):
    path_contents = os.listdir(path)
    assert set(path_contents) == set(files)


def expect_fs(folder, contents):
    if isinstance(contents, list):
        expect_folder(folder, contents)
    elif isinstance(contents, dict):
        expect_folder(folder, contents.keys())

        for path in iter(contents):
            if contents[path] is not None:
                expect_fs(os.path.join(folder, path), contents[path])
    else:
        raise ValueError("Invalid type {0} passed to expect_fs".format(type(contents)))


def _tar_filter(tar_info: tarfile.TarInfo) -> tarfile.TarInfo:
    tar_info.uid = 0
    tar_info.gid = 0
    return tar_info


def make_tar(result_filename, change_folder):
    tar_mode = ''
    if is_windows:
        tar_mode = 'w:gz'
    else:
        tar_mode = 'w:xz'

    with tarfile.open(name=str(result_filename), mode=tar_mode) as tar:
        tar.add(name=str(change_folder), arcname='./', filter=_tar_filter)


def rewrite_symlinks(root, old_prefix, new_prefix):
    log.info("Rewrite symlinks in %s from %s to %s", root, old_prefix, new_prefix)
    # Find the symlinks and rewrite them from old_prefix to new_prefix
    # All symlinks not beginning with old_prefix are ignored because
    # packages may contain arbitrary symlinks.
    for root_dir, dirs, files in os.walk(root):
        for name in chain(files, dirs):
            full_path = os.path.join(root_dir, name)
            if os.path.islink(full_path):
                # Rewrite old_prefix to new_prefix if present.
                target = os.readlink(full_path)
                if target.startswith(old_prefix):
                    new_target = os.path.join(new_prefix, target[len(old_prefix) + 1:].lstrip('/'))
                    log.debug("Remove the old link (%s) and write a new one (%s).", target, new_target)
                    os.remove(full_path)
                    os.symlink(new_target, full_path)


def check_forbidden_services(path, services):
    """Check if package contains systemd services that may break DC/OS

    This functions checks the contents of systemd's unit file dirs and
    throws the exception if there are reserved services inside.

    Args:
        path: path where the package contents are
        services: list of reserved services to look for

    Raises:
        ValidationError: Reserved serice names were found inside the package
    """
    services_dir_regexp = re.compile(r'dcos.target.wants(?:_.+)?')
    forbidden_srv_set = set(services)
    pkg_srv_set = set()

    for direntry in os.listdir(path):
        if not services_dir_regexp.match(direntry):
            continue
        pkg_srv_set.update(set(os.listdir(os.path.join(path, direntry))))

    found_units = forbidden_srv_set.intersection(pkg_srv_set)

    if found_units:
        msg = "Reverved unit names found: " + ','.join(found_units)
        raise ValidationError(msg)<commit_msg>Removed extra logging that cause build log to explode.<commit_after>import hashlib
import http.server
import json
import logging
import os
import platform
import re
import shutil
import socketserver
import stat
import tarfile
import tempfile
from contextlib import contextmanager, ExitStack
from itertools import chain
from multiprocessing import Process
from shutil import rmtree
from typing import List

import requests
import retrying
import teamcity
import yaml
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from teamcity.messages import TeamcityServiceMessages

from pkgpanda import subprocess
from pkgpanda.exceptions import FetchError, IncompleteDownloadError, ValidationError

log = logging.getLogger(__name__)
is_windows = platform.system() == "Windows"


def is_absolute_path(path):
    if is_windows:
        # We assume one char drive letter. Sometimes its two but not often
        # pattern is <driveletter>:/string....
        if path[1] == ':':
            return True
    else:
        if path[0] == '/':
            return True
    return False


def remove_file(path):
    """removes a file. fails silently if the file does not exist"""
    if is_windows:
        # python library on Windows does not like symbolic links in directories
        # so calling out to the cmd prompt to do this fixes that.
        path = path.replace('/', '\\')
        if os.path.exists(path):
            subprocess.call(['cmd.exe', '/c', 'del', '/q', path])
    else:
        subprocess.check_call(['rm', '-f', path])


def remove_directory(path):
    """recursively removes a directory tree. fails silently if the tree does not exist"""
    if is_windows:
        # python library on Windows does not like symbolic links in directories
        # so calling out to the cmd prompt to do this fixes that.
        path = path.replace('/', '\\')
        if os.path.exists(path):
            subprocess.call(['cmd.exe', '/c', 'rmdir', '/s', '/q', path])
    else:
        subprocess.check_call(['rm', '-rf', path])


def make_directory(path):
    """Create a directory, creating intermediate directories if necessary"""
    if is_windows:
        path = path.replace('/', '\\')

    if not os.path.exists(path):
        os.makedirs(path)


def copy_file(src_path, dst_path):
    """copy a single directory item from one location to another"""
    if is_windows:
        # To make sure the copy works we are using cmd version as python
        # libraries may not handle symbolic links and other things that are
        # thrown at it.
        src = src_path.replace('/', '\\')
        dst = dst_path.replace('/', '\\')
        subprocess.check_call(['cmd.exe', '/c', 'copy', src, dst])
    else:
        subprocess.check_call(['cp', src_path, dst_path])


def copy_directory(src_path, dst_path):
    """copy recursively a directory tree from one location to another"""
    if is_windows:
        # To make sure the copy works we are using cmd version as python
        # libraries may not handle symbolic links and other things that are
        # thrown at it.
        src = src_path.replace('/', '\\')
        dst = dst_path.replace('/', '\\')
        subprocess.check_call(['cmd.exe', '/c', 'xcopy', src, dst, '/E', '/B', '/I'])
    else:
        subprocess.check_call(['cp', '-r', src_path, dst_path])


def variant_str(variant):
    """Return a string representation of variant."""
    if variant is None:
        return ''
    return variant


def variant_object(variant_str):
    """Return a variant object from its string representation."""
    if variant_str == '':
        return None
    return variant_str


def variant_name(variant):
    """Return a human-readable string representation of variant."""
    if variant is None:
        return '<default>'
    return variant


def variant_prefix(variant):
    """Return a filename prefix for variant."""
    if variant is None:
        return ''
    return variant + '.'


def variant_suffix(variant, delim='.'):
    if variant is None:
        return ''
    return delim + variant


def get_requests_retry_session(max_retries=4, backoff_factor=1, status_forcelist=None):
    status_forcelist = status_forcelist or [500, 502, 504]
    # Default max retries 4 with sleeping between retries 1s, 2s, 4s, 8s
    session = requests.Session()
    custom_retry = Retry(total=max_retries,
                         backoff_factor=backoff_factor,
                         status_forcelist=status_forcelist)
    custom_adapter = HTTPAdapter(max_retries=custom_retry)
    # Any request through this session that starts with 'http://' or 'https://'
    # will use the custom Transport Adapter created which include retries
    session.mount('http://', custom_adapter)
    session.mount('https://', custom_adapter)
    return session


def _is_incomplete_download_error(exception):
    return isinstance(exception, IncompleteDownloadError)


@retrying.retry(
    stop_max_attempt_number=3,
    wait_random_min=1000,
    wait_random_max=2000,
    retry_on_exception=_is_incomplete_download_error)
def _download_remote_file(out_filename, url):
    with open(out_filename, "wb") as f:
        r = get_requests_retry_session().get(url, stream=True)
        r.raise_for_status()

        total_bytes_read = 0
        for chunk in r.iter_content(chunk_size=4096):
            f.write(chunk)
            total_bytes_read += len(chunk)

        if 'content-length' in r.headers:
            content_length = int(r.headers['content-length'])
            if total_bytes_read != content_length:
                raise IncompleteDownloadError(url, total_bytes_read, content_length)

        return r


def download(out_filename, url, work_dir, rm_on_error=True):
    assert os.path.isabs(out_filename)
    assert os.path.isabs(work_dir)
    work_dir = work_dir.rstrip('/')

    # Strip off whitespace to make it so scheme matching doesn't fail because
    # of simple user whitespace.
    url = url.strip()

    # Handle file:// urls specially since requests doesn't know about them.
    try:
        if url.startswith('file://'):
            src_filename = url[len('file://'):]
            if not os.path.isabs(src_filename):
                src_filename = work_dir + '/' + src_filename
            shutil.copyfile(src_filename, out_filename)
        else:
            _download_remote_file(out_filename, url)
    except Exception as fetch_exception:
        if rm_on_error:
            rm_passed = False

            # try / except so if remove fails we don't get an exception during an exception.
            # Sets rm_passed to true so if this fails we can include a special error message in the
            # FetchError
            try:
                os.remove(out_filename)
                rm_passed = True
            except Exception:
                pass
        else:
            rm_passed = True

        raise FetchError(url, out_filename, fetch_exception, rm_passed) from fetch_exception


def download_atomic(out_filename, url, work_dir):
    assert os.path.isabs(out_filename)
    tmp_filename = out_filename + '.tmp'
    try:
        download(tmp_filename, url, work_dir)
        shutil.move(tmp_filename, out_filename)
    except FetchError:
        try:
            os.remove(tmp_filename)
        except:
            pass
        raise


def extract_tarball(path, target):
    """Extract the tarball into target.

    If there are any errors, delete the folder being extracted to.
    """
    # TODO(cmaloney): Validate extraction will pass before unpacking as much as possible.
    # TODO(cmaloney): Unpack into a temporary directory then move into place to
    # prevent partial extraction from ever laying around on the filesystem.
    try:
        assert os.path.exists(path), "Path doesn't exist but should: {}".format(path)
        make_directory(target)

        # TODO(tweidner): https://jira.mesosphere.com/browse/DCOS-48220
        # Make this cross-platform via Python's tarfile module once
        # https://bugs.python.org/issue21872 is fixed.
        if is_windows:
            subprocess.check_call(['bsdtar', '-xf', path, '-C', target])
        else:
            subprocess.check_call(['tar', '-xf', path, '-C', target])

    except:
        # If there are errors, we can't really cope since we are already in an error state.
        rmtree(target, ignore_errors=True)
        raise


def load_json(filename):
    try:
        with open(filename) as f:
            return json.load(f)
    except ValueError as ex:
        raise ValueError("Invalid JSON in {0}: {1}".format(filename, ex)) from ex


class YamlParseError(Exception):
    pass


def load_yaml(filename):
    try:
        with open(filename) as f:
            return yaml.safe_load(f)
    except yaml.YAMLError as ex:
        raise YamlParseError("Invalid YAML in {}: {}".format(filename, ex)) from ex


def write_yaml(filename, data, **kwargs):
    dumped_yaml = yaml.safe_dump(data, **kwargs)
    write_string(filename, dumped_yaml)


def make_file(name):
    with open(name, 'a'):
        pass


def write_json(filename, data):
    dumped_json = json_prettyprint(data=data)
    write_string(filename, dumped_json)


def write_string(filename, data):
    """
    Write a string to a file.
    Overwrite any data in that file.

    We use an atomic write practice of creating a temporary file and then
    moving that temporary file to the given ``filename``. This prevents race
    conditions such as the file being read by another process after it is
    opened here but not yet written to.

    It also prevents us from creating or truncating a file before we fail to
    write data to it because of low disk space.

    If no file already exists at ``filename``, the new file is created with
    permissions 0o644.
    """
    prefix = os.path.basename(filename)
    tmp_file_dir = os.path.dirname(os.path.realpath(filename))
    fd, temporary_filename = tempfile.mkstemp(prefix=prefix, dir=tmp_file_dir)

    try:
        permissions = os.stat(filename).st_mode
    except FileNotFoundError:
        log.debug("File %s does not exist, creating", filename)
        permissions = 0o644

    try:
        try:
            os.write(fd, data.encode())
        finally:
            os.close(fd)
        os.chmod(temporary_filename, stat.S_IMODE(permissions))
        os.replace(temporary_filename, filename)
    except Exception:
        os.remove(temporary_filename)
        raise


def load_string(filename):
    with open(filename) as f:
        return f.read().strip()


def json_prettyprint(data):
    return json.dumps(
        data,
        sort_keys=True,
        indent=2,
        separators=(',', ':'),
    )


def if_exists(fn, *args, **kwargs):
    try:
        return fn(*args, **kwargs)
    except FileNotFoundError as e:
        log.debug(e)
        return None


def sha1(filename):
    hasher = hashlib.sha1()

    with open(filename, 'rb') as fh:
        while 1:
            buf = fh.read(4096)
            if not buf:
                break
            hasher.update(buf)

    return hasher.hexdigest()


def expect_folder(path, files):
    path_contents = os.listdir(path)
    assert set(path_contents) == set(files)


def expect_fs(folder, contents):
    if isinstance(contents, list):
        expect_folder(folder, contents)
    elif isinstance(contents, dict):
        expect_folder(folder, contents.keys())

        for path in iter(contents):
            if contents[path] is not None:
                expect_fs(os.path.join(folder, path), contents[path])
    else:
        raise ValueError("Invalid type {0} passed to expect_fs".format(type(contents)))


def _tar_filter(tar_info: tarfile.TarInfo) -> tarfile.TarInfo:
    tar_info.uid = 0
    tar_info.gid = 0
    return tar_info


def make_tar(result_filename, change_folder):
    tar_mode = ''
    if is_windows:
        tar_mode = 'w:gz'
    else:
        tar_mode = 'w:xz'

    with tarfile.open(name=str(result_filename), mode=tar_mode) as tar:
        tar.add(name=str(change_folder), arcname='./', filter=_tar_filter)


def rewrite_symlinks(root, old_prefix, new_prefix):
    log.info("Rewrite symlinks in %s from %s to %s", root, old_prefix, new_prefix)
    # Find the symlinks and rewrite them from old_prefix to new_prefix
    # All symlinks not beginning with old_prefix are ignored because
    # packages may contain arbitrary symlinks.
    for root_dir, dirs, files in os.walk(root):
        for name in chain(files, dirs):
            full_path = os.path.join(root_dir, name)
            if os.path.islink(full_path):
                # Rewrite old_prefix to new_prefix if present.
                target = os.readlink(full_path)
                if target.startswith(old_prefix):
                    new_target = os.path.join(new_prefix, target[len(old_prefix) + 1:].lstrip('/'))
                    os.remove(full_path)
                    os.symlink(new_target, full_path)


def check_forbidden_services(path, services):
    """Check if package contains systemd services that may break DC/OS

    This functions checks the contents of systemd's unit file dirs and
    throws the exception if there are reserved services inside.

    Args:
        path: path where the package contents are
        services: list of reserved services to look for

    Raises:
        ValidationError: Reserved serice names were found inside the package
    """
    services_dir_regexp = re.compile(r'dcos.target.wants(?:_.+)?')
    forbidden_srv_set = set(services)
    pkg_srv_set = set()

    for direntry in os.listdir(path):
        if not services_dir_regexp.match(direntry):
            continue
        pkg_srv_set.update(set(os.listdir(os.path.join(path, direntry))))

    found_units = forbidden_srv_set.intersection(pkg_srv_set)

    if found_units:
        msg = "Reverved unit names found: " + ','.join(found_units)
        raise ValidationError(msg)
{'fraction_non_alphanumeric': 0.09663308114310301, 'fraction_numerical': 0.004661336010613503, 'mean_word_length': 2.728111215078198, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1498084', 'n_tokens_mistral': 8303, 'n_tokens_neox': 7970, 'n_words': 2512}