<commit_before>#!/usr/bin/env python
from __future__ import with_statement

import argparse
import hashlib
import os
import sys
import tempfile
import shutil
import logging

from logging import getLogger, StreamHandler

# Update here and in setup.py
VERSION = '1.0.0rc5-dev'

try:
    import boto  # noqa
    import boto.s3.connection
    import boto.exception
except ImportError:
    boto = None  # noqa

from virtualenv import (  # noqa
    call_subprocess,
    create_bootstrap_script,
)

logger = getLogger(__name__)

# http://www.astro.keele.ac.uk/oldusers/rno/Computing/File_magic.html
MAGIC_NUM = {
            # magic code, offset
    'ELF': ('.ELF', 0),
    'GZIP': ('\x1f\x8b', 0),
    'BZIP': ('\x42\x5a', 0),
    'TAR': ('ustar', 257),
}


class Namespace(argparse._AttributeHolder):
    def __init__(self):
        super(Namespace, self).__init__()
        self._sensitive_arguments = set()

    def _get_kwargs(self):
        '''
        Exclude arguments which are defined as being sensitive
        '''
        kwargs = super(Namespace, self)._get_kwargs()
        return [
            (name, value) for name, value in kwargs
            if name not in self.sensitive_arguments
        ]

    @property
    def sensitive_arguments(self):
        return self._sensitive_arguments

    def add_sensitive_arguments(self, *args):
        '''
        Define arguments as being "sensitive." That is, when this object is
        converted to text, these attributes are not included.

        `args` is a list of strings that indicate the attribute name.
        '''
        self._sensitive_arguments |= set(args)


argparse.Namespace = Namespace


def rmtree(path):
    try:
        if os.path.islink(path):
            os.unlink(path)
        elif os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.unlink(path)
        return True
    except OSError, why:
        logger.warn(
            'Failed to remove %s. '
            'Make sure you have permissions to this path. '
            '%s' % (path, why)
        )
        return False


# Helper method to determine the actual type of the file without relying on the
# file extension
def get_type(path):
    with open(path) as f:
        for file_type, magic in MAGIC_NUM.items():
            f.seek(magic[1])
            if magic[0] == f.read(len(magic[0])):
                    return file_type
    return None


class Terrarium(object):
    def __init__(self, args):
        self.args = args
        self._requirements = None
        self._digest = None

    @property
    def digest(self):
        if self._digest is not None:
            return self._digest
        m = hashlib.new(self.args.digest_type)
        m.update('\n'.join(self.requirements))
        self._digest = m.hexdigest()
        return self._digest

    @property
    def requirements(self):
        if self._requirements is not None:
            return self._requirements
        lines = []
        for arg in self.args.reqs:
            if os.path.exists(arg):
                with open(arg, 'r') as f:
                    for line in f.readlines():
                        line = line.strip()
                        if line and not line.startswith('#'):
                            lines.append(line)
        self._requirements = sorted(lines)
        return self._requirements

    def restore_previously_backed_up_environment(self):
        backup = self.get_backup_location()
        if not self.environment_exists(backup):
            logger.info(
                'Failed to restore backup. It doesn\'t appear to exist at %s',
                backup,
            )
            return 1

        target = self.get_target_location()
        if os.path.isdir(target):
            logger.info('Deleting environment at %s', target)
            rmtree(target)

        logger.info('Renaming %s to %s', backup, target)
        os.rename(backup, target)
        return 0

    def get_target_location(self):
        return os.path.abspath(self.args.target)

    def get_backup_location(self, target=None):
        if target is None:
            target = self.get_target_location()
        return ''.join([target, self.args.backup_suffix])

    def environment_exists(self, env):
        return os.path.exists(os.path.join(
            env,
            'bin',
            'activate',
        ))

    def install(self):
        logger.debug('Running install')

        old_target = self.get_target_location()
        old_target_backup = self.get_backup_location()
        new_target = old_target
        prompt = os.path.basename(new_target)

        # Are we building a new environment, or replacing an existing one?
        old_target_exists = self.environment_exists(old_target)

        if old_target_exists and not self.args.replace:
            args = [
                'bash',
                '-c',
                'source {}/bin/activate && pip install -r {}'.format(old_target, ' '.join(self.args.reqs))
            ]
            call_subprocess(args)
            logger.info('Virtual environment exists, re-installing requirements...')
            return 0

        if old_target_exists:<commit_msg>commenting out command execution<commit_after>#!/usr/bin/env python
from __future__ import with_statement

import argparse
import hashlib
import os
import sys
import tempfile
import shutil
import logging

from logging import getLogger, StreamHandler

# Update here and in setup.py
VERSION = '1.0.0rc5-dev'

try:
    import boto  # noqa
    import boto.s3.connection
    import boto.exception
except ImportError:
    boto = None  # noqa

from virtualenv import (  # noqa
    call_subprocess,
    create_bootstrap_script,
)

logger = getLogger(__name__)

# http://www.astro.keele.ac.uk/oldusers/rno/Computing/File_magic.html
MAGIC_NUM = {
            # magic code, offset
    'ELF': ('.ELF', 0),
    'GZIP': ('\x1f\x8b', 0),
    'BZIP': ('\x42\x5a', 0),
    'TAR': ('ustar', 257),
}


class Namespace(argparse._AttributeHolder):
    def __init__(self):
        super(Namespace, self).__init__()
        self._sensitive_arguments = set()

    def _get_kwargs(self):
        '''
        Exclude arguments which are defined as being sensitive
        '''
        kwargs = super(Namespace, self)._get_kwargs()
        return [
            (name, value) for name, value in kwargs
            if name not in self.sensitive_arguments
        ]

    @property
    def sensitive_arguments(self):
        return self._sensitive_arguments

    def add_sensitive_arguments(self, *args):
        '''
        Define arguments as being "sensitive." That is, when this object is
        converted to text, these attributes are not included.

        `args` is a list of strings that indicate the attribute name.
        '''
        self._sensitive_arguments |= set(args)


argparse.Namespace = Namespace


def rmtree(path):
    try:
        if os.path.islink(path):
            os.unlink(path)
        elif os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.unlink(path)
        return True
    except OSError, why:
        logger.warn(
            'Failed to remove %s. '
            'Make sure you have permissions to this path. '
            '%s' % (path, why)
        )
        return False


# Helper method to determine the actual type of the file without relying on the
# file extension
def get_type(path):
    with open(path) as f:
        for file_type, magic in MAGIC_NUM.items():
            f.seek(magic[1])
            if magic[0] == f.read(len(magic[0])):
                    return file_type
    return None


class Terrarium(object):
    def __init__(self, args):
        self.args = args
        self._requirements = None
        self._digest = None

    @property
    def digest(self):
        if self._digest is not None:
            return self._digest
        m = hashlib.new(self.args.digest_type)
        m.update('\n'.join(self.requirements))
        self._digest = m.hexdigest()
        return self._digest

    @property
    def requirements(self):
        if self._requirements is not None:
            return self._requirements
        lines = []
        for arg in self.args.reqs:
            if os.path.exists(arg):
                with open(arg, 'r') as f:
                    for line in f.readlines():
                        line = line.strip()
                        if line and not line.startswith('#'):
                            lines.append(line)
        self._requirements = sorted(lines)
        return self._requirements

    def restore_previously_backed_up_environment(self):
        backup = self.get_backup_location()
        if not self.environment_exists(backup):
            logger.info(
                'Failed to restore backup. It doesn\'t appear to exist at %s',
                backup,
            )
            return 1

        target = self.get_target_location()
        if os.path.isdir(target):
            logger.info('Deleting environment at %s', target)
            rmtree(target)

        logger.info('Renaming %s to %s', backup, target)
        os.rename(backup, target)
        return 0

    def get_target_location(self):
        return os.path.abspath(self.args.target)

    def get_backup_location(self, target=None):
        if target is None:
            target = self.get_target_location()
        return ''.join([target, self.args.backup_suffix])

    def environment_exists(self, env):
        return os.path.exists(os.path.join(
            env,
            'bin',
            'activate',
        ))

    def install(self):
        logger.debug('Running install')

        old_target = self.get_target_location()
        old_target_backup = self.get_backup_location()
        new_target = old_target
        prompt = os.path.basename(new_target)

        # Are we building a new environment, or replacing an existing one?
        old_target_exists = self.environment_exists(old_target)

        if old_target_exists and not self.args.replace:
            args = [
                'bash',
                '-c',
                'source {}/bin/activate && pip install -r {}'.format(old_target, ' '.join(self.args.reqs))
            ]
            # call_subprocess(args)
            logger.info('Virtual environment exists, re-installing requirements...')
            return 0

        if old_target_exists:
{'fraction_non_alphanumeric': 0.09736562771398244, 'fraction_numerical': 0.0042458747466949725, 'mean_word_length': 1.991053391053391, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083839', 'n_tokens_mistral': 3042, 'n_tokens_neox': 2917, 'n_words': 796}