<commit_before># Py3 compatibility
from __future__ import print_function
from __future__ import unicode_literals

import os
import sys
import subprocess
import contextlib
from tempfile import NamedTemporaryFile
from passlib.apache import HtpasswdFile


@contextlib.contextmanager
def set_env(key, value):
    exists = key in os.environ
    original_value = os.environ.get(key, '')
    os.environ[key] = value
    yield
    os.environ[key] = original_value
    if not exists:
        del os.environ[key]


def pip_install(*args):
    '''
    Run pip install ...

    Explicitly ignores user's config.
    '''
    pip_cmd = os.path.join(os.path.dirname(sys.executable), 'pip')
    with set_env('PIP_CONFIG_FILE', os.devnull):
        cmd = [pip_cmd, 'install'] + list(args)
        print(' '.join(cmd))
        subprocess.call(cmd)


def make_htpasswd(filename, username, password):
    ht = HtpasswdFile(path=filename, new=True)
    ht.set_password(username, password)
    ht.save()


def pypi_server(
    directory,
    username,
    password,
    interface='0.0.0.0',  # all interfaces
    port='8080',
    volatile=False,       # allow package overwrites?
):
    '''
    Run pypi-server.
    '''
    pypi_srv = os.path.join(os.path.dirname(sys.executable), 'pypi-server')
    with NamedTemporaryFile() as password_file:
        make_htpasswd(password_file.name, username, password)

        cmd = [
            pypi_srv,
            '--interface', interface,
            '--port', port,
            '--passwords', password_file.name,
            '--disable-fallback',
        ] + (
            ['--overwrite'] if volatile else []
        ) + [directory]

        try:
            retval = subprocess.call(cmd)
            print(retval)
        except KeyboardInterrupt:
            pass


def write_file(path, content):
    try:
        os.makedirs(os.path.dirname(path))<commit_msg>serve directory - with ctrl-c to exit<commit_after># Py3 compatibility
from __future__ import print_function
from __future__ import unicode_literals

import os
import sys
import subprocess
import signal
import contextlib
from tempfile import NamedTemporaryFile
from passlib.apache import HtpasswdFile


@contextlib.contextmanager
def set_env(key, value):
    exists = key in os.environ
    original_value = os.environ.get(key, '')
    os.environ[key] = value
    yield
    os.environ[key] = original_value
    if not exists:
        del os.environ[key]


def pip_install(*args):
    '''
    Run pip install ...

    Explicitly ignores user's config.
    '''
    pip_cmd = os.path.join(os.path.dirname(sys.executable), 'pip')
    with set_env('PIP_CONFIG_FILE', os.devnull):
        cmd = [pip_cmd, 'install'] + list(args)
        print(' '.join(cmd))
        subprocess.call(cmd)


def make_htpasswd(filename, username, password):
    ht = HtpasswdFile(path=filename, new=True)
    ht.set_password(username, password)
    ht.save()


def pypi_server(
    directory,
    username,
    password,
    interface='0.0.0.0',  # all interfaces
    port='8080',
    volatile=False,       # allow package overwrites?
):
    '''
    Run pypi-server.
    '''
    pypi_srv = os.path.join(os.path.dirname(sys.executable), 'pypi-server')
    with NamedTemporaryFile() as password_file:
        make_htpasswd(password_file.name, username, password)

        cmd = [
            pypi_srv,
            '--interface', interface,
            '--port', port,
            '--passwords', password_file.name,
            '--disable-fallback',
        ] + (
            ['--overwrite'] if volatile else []
        ) + [directory]

        process = subprocess.Popen(cmd)
        try:
            process.wait()
        except KeyboardInterrupt:
            os.kill(process.pid, signal.SIGHUP)
            process.wait()
        print()


def write_file(path, content):
    try:
        os.makedirs(os.path.dirname(path))
{'fraction_non_alphanumeric': 0.11698015975264107, 'fraction_numerical': 0.004637979902087091, 'mean_word_length': 2.5387420237010025, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7007501', 'n_tokens_mistral': 1254, 'n_tokens_neox': 1213, 'n_words': 269}