<commit_before>#!/usr/bin/env python

from __future__ import print_function

import copy
import operator
import os
import platform
import subprocess
import sys

try:
    from configparser import Error as ConfigParserError, SafeConfigParser
except ImportError:
    from ConfigParser import Error as ConfigParserError, SafeConfigParser

from collections import defaultdict
from distutils.util import strtobool


__author__ = '<NAME>'
__license__ = 'BSD License'
__script__ = 'bootstrapper'
__version__ = '0.2.1'


CONFIG = {
    __script__: {
        'env': 'env',
        'requirements': 'requirements.txt',
        'quiet': False,
    },
    'pip': {
        'download_cache': '~/.{0}/pip-cache/'.format(__script__),
    },
    'virtualenv': {},
}
DEFAULT_CONFIG = 'bootstrap.cfg'
IS_PY3 = sys.version_info[0] == 3
IS_WINDOWS = platform.system() == 'Windows'

iteritems = lambda seq: seq.items() if IS_PY3 else seq.iteritems()
iterkeys = lambda seq: seq.keys() if IS_PY3 else seq.iterkeys()
string_types = (bytes, str) if IS_PY3 else basestring


def check_pre_requirements(pre_requirements):
    """
    Check all necessary system requirements to exist.
    """
    pre_requirements = set(pre_requirements or [])
    pre_requirements.add('virtualenv')

    for requirement in pre_requirements:
        if not which(requirement):
            error('Requirement {0!r} is not found in system'.
                  format(requirement))

    return True


def config_to_args(config):
    """
    Convert config dict to arguments list.
    """
    result = []

    for key, value in iteritems(config):
        if value is False:
            continue

        key = key.replace('_', '-')

        if value is not True:
            result.append('--{0}={1}'.format(key, str(value)))
        else:
            result.append('--{0}'.format(key))

    return result


def create_env(env, args, recreate=False, quiet=False):
    """
    Create virtual environment.
    """
    cmd = None

    if not quiet:
        print('== Step 1. Create virtual environment ==')

    if recreate or not os.path.isdir(env):
        cmd = 'virtualenv {0} {1}'.format(args, env)

    if not cmd and not quiet:
        print('Virtual environment {0!r} already created, done...'.format(env))

    if cmd:
        run_cmd(cmd, echo=not quiet)

    if not quiet:
        print()

    return True


def error(message, code=None):
    """
    Print error message and exit with error code.
    """
    print('ERROR: {0}. Exit...'.format(message.rstrip('.')))
    sys.exit(code or 1)


def install(env, requirements, args, quiet=False):
    """
    Install library or project into virtual environment.
    """
    if os.path.isfile(requirements):
        args += ' -r {0}'.format(requirements)
        label = 'project'
    else:
        args += ' -e .'
        label = 'library'

    if not quiet:
        print('== Step 2. Install {0} =='.format(label))

    pip_cmd(env, 'install {0}'.format(args), echo=not quiet)

    if not quiet:
        print()

    return True


def main(*args):
    """
    Bootstrap Python projects and libraries with virtualenv and pip.

    Also check system requirements before bootstrap and run post bootstrap
    hook if any.
    """
    try:
        # Create parser, read arguments from direct input or command line
        args = parse_args(args or sys.argv[1:])

        # Initialize bootstrapper instance Read current config from file
        config = read_config(args.config, args)
        bootstrap = config[__script__]

        # Check pre-requirements
        check_pre_requirements(bootstrap['pre_requirements'])

        # Create virtual environment
        env_args = prepare_args(config['virtualenv'], bootstrap)
        create_env(bootstrap['env'],
                   env_args,
                   bootstrap['recreate'],
                   bootstrap['quiet'])

        # And install library or project here
        pip_args = prepare_args(config['pip'], bootstrap)
        install(bootstrap['env'],
                bootstrap['requirements'],
                pip_args,
                bootstrap['quiet'])

        # Run post-bootstrap hook
        run_hook(bootstrap['hook'], bootstrap, bootstrap['quiet'])

        # All OK!
        if not bootstrap['quiet']:
            print('All OK!')
    except BaseException as err:
        filename = os.path.join('~', __script__, '{0}.log'.format(__script__))
        filename = os.path.expanduser(filename)

        with open(filename, 'a+') as handler:
            traceback.print_exc(file=handler)

        message = ('User aborted workflow'
                   if isinstance(err, KeyboardInterrupt)
                   else 'Unexcepted error catched')
        print('ERROR: {0}. Exit...'.format(message))
        print('Full log stored to ~/.{0}/{0}.log'.format(__script__))

        # True means error happened, exit code: 1
        return True
<commit_msg>Fix default path to pip cache directory and path to bootstrapper log file.<commit_after>#!/usr/bin/env python

from __future__ import print_function

import copy
import operator
import os
import platform
import subprocess
import sys

try:
    from configparser import Error as ConfigParserError, SafeConfigParser
except ImportError:
    from ConfigParser import Error as ConfigParserError, SafeConfigParser

from collections import defaultdict
from distutils.util import strtobool


__author__ = '<NAME>'
__license__ = 'BSD License'
__script__ = 'bootstrapper'
__version__ = '0.2.1'


CONFIG = {
    __script__: {
        'env': 'env',
        'requirements': 'requirements.txt',
        'quiet': False,
    },
    'pip': {
        'download_cache': os.path.expanduser(
            os.path.join('~', '.{0}'.format(__script__), 'pip-cache')
        ),
    },
    'virtualenv': {},
}
DEFAULT_CONFIG = 'bootstrap.cfg'
IS_PY3 = sys.version_info[0] == 3
IS_WINDOWS = platform.system() == 'Windows'

iteritems = lambda seq: seq.items() if IS_PY3 else seq.iteritems()
iterkeys = lambda seq: seq.keys() if IS_PY3 else seq.iterkeys()
string_types = (bytes, str) if IS_PY3 else basestring


def check_pre_requirements(pre_requirements):
    """
    Check all necessary system requirements to exist.
    """
    pre_requirements = set(pre_requirements or [])
    pre_requirements.add('virtualenv')

    for requirement in pre_requirements:
        if not which(requirement):
            error('Requirement {0!r} is not found in system'.
                  format(requirement))

    return True


def config_to_args(config):
    """
    Convert config dict to arguments list.
    """
    result = []

    for key, value in iteritems(config):
        if value is False:
            continue

        key = key.replace('_', '-')

        if value is not True:
            result.append('--{0}={1}'.format(key, str(value)))
        else:
            result.append('--{0}'.format(key))

    return result


def create_env(env, args, recreate=False, quiet=False):
    """
    Create virtual environment.
    """
    cmd = None

    if not quiet:
        print('== Step 1. Create virtual environment ==')

    if recreate or not os.path.isdir(env):
        cmd = 'virtualenv {0} {1}'.format(args, env)

    if not cmd and not quiet:
        print('Virtual environment {0!r} already created, done...'.format(env))

    if cmd:
        run_cmd(cmd, echo=not quiet)

    if not quiet:
        print()

    return True


def error(message, code=None):
    """
    Print error message and exit with error code.
    """
    print('ERROR: {0}. Exit...'.format(message.rstrip('.')))
    sys.exit(code or 1)


def install(env, requirements, args, quiet=False):
    """
    Install library or project into virtual environment.
    """
    if os.path.isfile(requirements):
        args += ' -r {0}'.format(requirements)
        label = 'project'
    else:
        args += ' -e .'
        label = 'library'

    if not quiet:
        print('== Step 2. Install {0} =='.format(label))

    pip_cmd(env, 'install {0}'.format(args), echo=not quiet)

    if not quiet:
        print()

    return True


def main(*args):
    """
    Bootstrap Python projects and libraries with virtualenv and pip.

    Also check system requirements before bootstrap and run post bootstrap
    hook if any.
    """
    try:
        # Create parser, read arguments from direct input or command line
        args = parse_args(args or sys.argv[1:])

        # Initialize bootstrapper instance Read current config from file
        config = read_config(args.config, args)
        bootstrap = config[__script__]

        # Check pre-requirements
        check_pre_requirements(bootstrap['pre_requirements'])

        # Create virtual environment
        env_args = prepare_args(config['virtualenv'], bootstrap)
        create_env(bootstrap['env'],
                   env_args,
                   bootstrap['recreate'],
                   bootstrap['quiet'])

        # And install library or project here
        pip_args = prepare_args(config['pip'], bootstrap)
        install(bootstrap['env'],
                bootstrap['requirements'],
                pip_args,
                bootstrap['quiet'])

        # Run post-bootstrap hook
        run_hook(bootstrap['hook'], bootstrap, bootstrap['quiet'])

        # All OK!
        if not bootstrap['quiet']:
            print('All OK!')
    except BaseException as err:
        filename = os.path.expanduser(
            os.path.join('~',
                         '.{0}'.format(__script__),
                         '{0}.log'.format(__script__)
        )

        with open(filename, 'a+') as handler:
            traceback.print_exc(file=handler)

        message = ('User aborted workflow'
                   if isinstance(err, KeyboardInterrupt)
                   else 'Unexcepted error catched')
        print('ERROR: {0}. Exit...'.format(message))
        print('Full log stored to ~/.{0}/{0}.log'.format(__script__))

        # True means error happened, exit code: 1
        return True

{'fraction_non_alphanumeric': 0.11804721888755502, 'fraction_numerical': 0.006102440976390556, 'mean_word_length': 2.4496204278812974, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2525127', 'n_tokens_mistral': 2927, 'n_tokens_neox': 2780, 'n_words': 824}