<commit_before>#!/usr/bin/env python

# Accelerator for pip, the Python package manager.
#
# Author: <NAME> <<EMAIL>>
# Last Change: April 11, 2013
# URL: https://github.com/paylogic/pip-accel

"""
Usage: pip-accel [ARGUMENTS TO PIP]

The pip-accel program is a wrapper for pip, the Python package manager. It
accelerates the usage of pip to initialize Python virtual environments given
one or more requirements files. The pip-accel command supports all subcommands
and options supported by pip, however it is only useful for the "pip install"
subcommand.

For more information please refer to the GitHub project page
at https://github.com/paylogic/pip-accel
"""

import os
import os.path
import re
import shutil
import sys
import tarfile
import time
import urllib
import urlparse

# Whether a user is directly looking at the output.
INTERACTIVE = os.isatty(1)

# Check if the operator requested verbose output.
VERBOSE = '-v' in sys.argv

# The main loop of pip-accel retries at most this many times to counter pip errors
# due to connectivity issues with PyPi and/or linked distribution websites.
MAX_RETRIES = 10

# Select the location of the download cache and other files based on the user
# running the pip-accel command (root goes to /var/lib, otherwise ~/.pip-accel).
if os.getuid() == 0:
    download_cache = '/root/.pip/download-cache'
    source_index = '/var/lib/pip-accel/sources'
    binary_index = '/var/lib/pip-accel/binaries'
else:
    download_cache = os.path.expanduser(os.environ.get('PIP_DOWNLOAD_CACHE', '~/.pip/download-cache'))
    source_index = os.path.expanduser('~/.pip-accel/sources')
    binary_index = os.path.expanduser('~/.pip-accel/binaries')

def main():
    """
    Main logic of the pip-accel command.
    """
    arguments = sys.argv[1:]
    if not arguments:
        message("%s\n", __doc__.strip(), prefix=False)
        sys.exit(0)
    # If no install subcommand is given we pass the command line straight
    # to pip without any changes and exit immediately afterwards.
    if 'install' not in arguments:
        sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments))
    main_timer = Timer()
    # Create all required directories on the fly.
    for directory in [download_cache, source_index, binary_index]:
        if not os.path.isdir(directory):
            os.makedirs(directory)
    # Execute "pip install" in a loop in order to retry after intermittent
    # error responses from servers (which can happen quite frequently).
    for i in xrange(1, MAX_RETRIES):
        have_source_dists, dependencies = unpack_source_dists(arguments)
        if not have_source_dists:
            download_source_dists(arguments)
        elif not dependencies:
            message("No dependencies found in pip's output, probably there's nothing to do.\n")
            return
        else:
            if build_binary_dists(dependencies) and install_dependencies(dependencies):
                message("Done! Took %s to install %i packages.\n", main_timer, len(dependencies))
            return
    message("External command failed %i times, aborting!\n" % MAX_RETRIES)
    sys.exit(1)

def unpack_source_dists(original_arguments):
    """
    Check whether we have local source distributions available for all
    dependencies and find the names and versions of all dependencies.
    Returns a tuple of two values:

    - The first value is True if all dependencies are available as local source
      distributions, False otherwise.

    - When the first value is True, the second value is a list of tuples,
      otherwise the second value is None. Each tuple contains three values,
      in this order: (package-name, package-version, directory). The third
      value points to an existing directory containing the unpacked sources.
    """
    unpack_timer = Timer()
    message("Unpacking local source distributions ..\n")
    # Create a shallow copy of the original argument
    # list that includes the --no-install option.<commit_msg>Make the output more human friendly<commit_after>#!/usr/bin/env python

# Accelerator for pip, the Python package manager.
#
# Author: <NAME> <<EMAIL>>
# Last Change: April 12, 2013
# URL: https://github.com/paylogic/pip-accel

"""
Usage: pip-accel [ARGUMENTS TO PIP]

The pip-accel program is a wrapper for pip, the Python package manager. It
accelerates the usage of pip to initialize Python virtual environments given
one or more requirements files. The pip-accel command supports all subcommands
and options supported by pip, however it is only useful for the "pip install"
subcommand.

For more information please refer to the GitHub project page
at https://github.com/paylogic/pip-accel
"""

import os
import os.path
import re
import shutil
import sys
import tarfile
import time
import urllib
import urlparse

# Whether a user is directly looking at the output.
INTERACTIVE = os.isatty(1)

# Check if the operator requested verbose output.
VERBOSE = '-v' in sys.argv

# The main loop of pip-accel retries at most this many times to counter pip errors
# due to connectivity issues with PyPi and/or linked distribution websites.
MAX_RETRIES = 10

# Select the location of the download cache and other files based on the user
# running the pip-accel command (root goes to /var/lib, otherwise ~/.pip-accel).
if os.getuid() == 0:
    download_cache = '/root/.pip/download-cache'
    source_index = '/var/lib/pip-accel/sources'
    binary_index = '/var/lib/pip-accel/binaries'
else:
    download_cache = os.path.expanduser(os.environ.get('PIP_DOWNLOAD_CACHE', '~/.pip/download-cache'))
    source_index = os.path.expanduser('~/.pip-accel/sources')
    binary_index = os.path.expanduser('~/.pip-accel/binaries')

def main():
    """
    Main logic of the pip-accel command.
    """
    arguments = sys.argv[1:]
    if not arguments:
        message("%s\n", __doc__.strip(), prefix=False)
        sys.exit(0)
    # If no install subcommand is given we pass the command line straight
    # to pip without any changes and exit immediately afterwards.
    if 'install' not in arguments:
        sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments))
    main_timer = Timer()
    # Create all required directories on the fly.
    for directory in [download_cache, source_index, binary_index]:
        if not os.path.isdir(directory):
            os.makedirs(directory)
    # Execute "pip install" in a loop in order to retry after intermittent
    # error responses from servers (which can happen quite frequently).
    for i in xrange(1, MAX_RETRIES):
        have_source_dists, dependencies = unpack_source_dists(arguments)
        if not have_source_dists:
            download_source_dists(arguments)
        elif not dependencies:
            message("No dependencies found in pip's output, probably there's nothing to do.\n")
            return
        else:
            if build_binary_dists(dependencies) and install_dependencies(dependencies):
                message("Done! Took %s to install %i package%s.\n", main_timer, len(dependencies), len(dependencies) != 1 and 's' or '')
            return
    message("External command failed %i times, aborting!\n" % MAX_RETRIES)
    sys.exit(1)

def unpack_source_dists(original_arguments):
    """
    Check whether we have local source distributions available for all
    dependencies and find the names and versions of all dependencies.
    Returns a tuple of two values:

    - The first value is True if all dependencies are available as local source
      distributions, False otherwise.

    - When the first value is True, the second value is a list of tuples,
      otherwise the second value is None. Each tuple contains three values,
      in this order: (package-name, package-version, directory). The third
      value points to an existing directory containing the unpacked sources.
    """
    unpack_timer = Timer()
    message("Unpacking local source distributions ..\n")
    # Create a shallow copy of the original argument
    # list that includes the --no-install option.
{'fraction_non_alphanumeric': 0.08593847303524722, 'fraction_numerical': 0.003611906837713289, 'mean_word_length': 3.37125748502994, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4280666', 'n_tokens_mistral': 2297, 'n_tokens_neox': 2206, 'n_words': 887}