<commit_before>import os
import sys
import glob
import tarfile
import pkg_resources
import ConfigParser
from cStringIO import StringIO

from pkgtools.pkg import Dir as DirTools, EggDir
from pyg.scripts import script_args
from pyg.locations import EASY_INSTALL, INSTALL_DIR, BIN, USER_SITE
from pyg.utils import TempDir, ZipFile, call_setup, run_setup, name_from_egg, ext
from pyg.log import logger


__all__ = ['PygError', 'InstallationError', 'AlreadyInstalled', 'Version', 'ReqSet',
           'Egg', 'Archive', 'Dir', 'Bundle', 'Binary', 'args_manager']


## A generic error thrown by Pyg
class PygError(Exception):
    pass

## An error thrown when the installation goes wrong
class InstallationError(PygError):
    pass

## This isn't really an error but it is useful when Pyg is used as a library
class AlreadyInstalled(InstallationError):
    pass


#class Version(object): ## OLD!! Does not work properly!
#    def __init__(self, v):
#        self.v = v
#        while self.v[-1] == 0:
#            self.v = self.v[:-2]
#
#    def __str__(self):
#        return self.v
#
#    def __repr__(self):
#        return 'Version({0})'.format(self.v)
#
#    def __eq__(self, other):
#        if len(self.v) != len(other.v):
#            return False
#        return self.v == other.v
#
#    def __ge__(self, other):
#        return self.v >= other.v
#
#    def __gt__(self, other):
#        return self.v > other.v
#
#    def __le__(self, other):
#        return self.v <= other.v
#
#    def __lt__(self, other):
#        return self.v < other.v


class Version(object):
    '''
    This class implements a Version object with comparison methods::

        >>> Version('0.1') < Version('0.1.1')
        True
        >>> Version('0.1') == Version('0.1')
        True
        >>> Version('0.1') >= Version('0.1')
        True
        >>> Version('0.1b') > Version('0.1')
        False
        >>> Version('0.1b') > Version('0.1a')
        True
    '''

    def __init__(self, v):
        self._v = v
        self.v = pkg_resources.parse_version(v)

    def __str__(self):
        return str(self._v)

    def __repr__(self):
        return 'Version({0})'.format(self._v)

    def __str__(self):
        return self._v

    def __eq__(self, o):
        return self.v == o.v

    def __ge__(self, o):
        return self.v >= o.v

    def __gt__(self, o):
        return self.v > o.v

    def __le__(self, o):
        return self.v <= o.v

    def __lt__(self, o):
        return self.v < o.v


class ReqSet(object):
    '''
    A requirement set, used by :class:`~pyg.types.Archive` and :class:`~pyg.types.Egg` to keep trace of package's requirements.
    '''

    def __init__(self, comes_from):
        self._reqs = set()
        self.comes_from = comes_from

    def __iter__(self):
        for r in self.reqs:
            yield r

    def __nonzero__(self):
        return bool(self.reqs)

    def __bool__(self):
        return bool(self.reqs)

    def add(self, r):
        self._reqs.add(r)

    @ property
    def reqs(self):
        return self._reqs


class Egg(object):
    '''
    This class represent a Python Egg object. It needs a file-like object with egg data.

    :param fobj: the file-like object with egg data
    :param eggname: the egg name (for example ``pyg-0.1.2-py2.7.egg``)
    :param reqset: a :class:`~pyg.types.ReqSet` object used to store requirements
    :param packname: the package name. If the egg name is ``pyg-0.1.2-py2.7.egg``, *packname* should be ``pyg``
    '''

    def __init__(self, fobj, eggname, reqset, packname=None):
        self.fobj = fobj
        self.eggname = os.path.basename(eggname)
        self.reqset = reqset
        self.packname = packname or name_from_egg(eggname)
        self.idir = args_manager['install']['install_dir']
        if args_manager['install']['user']:
            self.idir = USER_SITE

    def install(self):
        eggpath = os.path.join(self.idir, self.eggname)
        if os.path.exists(eggpath):
            logger.info('{0} is already installed', self.packname)
            raise AlreadyInstalled
        logger.info('Installing {0} egg file', self.packname)
        with ZipFile(self.fobj) as z:
            z.extractall(eggpath)
        logger.info('Adding egg file to sys.path')
        with open(EASY_INSTALL) as f: ## TODO: Fix the opening mode to read and write simultaneously
            lines = f.readlines()
        with open(EASY_INSTALL, 'w') as f:
            try:
                f.writelines(lines[:-1])
                f.write('./' + self.eggname + '\n')
                f.write(lines[-1])
            ## When using this file for the first time
            except IndexError:
                pass
        dist = EggDir(eggpath)

        ## Install scripts in the setuptools' way
        if not args_manager['install']['no_scripts']:
            for name, content, mode in script_args(dist):
                logger.info('Installing {0} script to {1}', name, BIN)
                target = os.path.join(BIN, name)
                with open(target, 'w' + mode) as f:
                    f.write(content)
                    os.chmod(target, 0755)
        else:
            logger.info('Scripts not installed')
        logger.info('Looking for requirements...')
        try:
            for req in dist.file('requires.txt'):
                self.reqset.add(req)
        except KeyError:
            logger.debug('requires.txt not found')


class Dir(object):
    '''
    To install the package this class needs the *path*, a temporary directory's path, and a :class:`ReqSet` object
    to store requirements.
    '''

    def __init__(self, path, name, tempdir, reqset=None):
        self.path = path
        self.name = name
        self.tempdir = tempdir
        self.reqset = reqset

    def install(self):
        '''
        Given a pathname containing a :file:`setup.py` file, install the package.
        First runs `setup.py egg_info` to find out requirements, then runs ``setup.py install``.
        '''

        if self.reqset is not None:
            logger.info('Running setup.py egg_info for {0}', self.name)
            call_setup(self.path, ['egg_info', '--egg-base', self.tempdir])
            try:
                dist = DirTools(glob.glob(os.path.join(self.path, '*egg-info'))[0])
            except (IndexError, ValueError):
                pass
            else:
                try:
                    for r in dist.file('requires.txt'):
                        self.reqset.add(r)
                except (KeyError, ConfigParser.MissingSectionHeaderError):
                    logger.debug('debug: requires.txt not found')
                try:
                    for r in dist.file('dependency_links.txt'):
                        self.reqset.add(r)
                except (KeyError, ConfigParser.MissingSectionHeaderError):
                    logger.debug('debug: dependency_links.txt not found')
        args = []
        if args_manager['install']['install_dir'] != INSTALL_DIR:
            args += ['--prefix', args_manager['install']['install_dir']]
        if args_manager['install']['no_scripts']:
            args += ['--install-scripts', self.tempdir]
        if args_manager['install']['no_data']:
            args += ['--install-data', self.tempdir]
        if args_manager['install']['user']:
            args += ['--user']
        run_setup(self.path, self.name, args=args, exc=InstallationError)
<commit_msg>Commenting out some suspect code, was trying to install URLs...<commit_after>import os
import sys
import glob
import tarfile
import pkg_resources
import ConfigParser
from cStringIO import StringIO

from pkgtools.pkg import Dir as DirTools, EggDir
from pyg.scripts import script_args
from pyg.locations import EASY_INSTALL, INSTALL_DIR, BIN, USER_SITE
from pyg.utils import TempDir, ZipFile, call_setup, run_setup, name_from_egg, ext
from pyg.log import logger


__all__ = ['PygError', 'InstallationError', 'AlreadyInstalled', 'Version', 'ReqSet',
           'Egg', 'Archive', 'Dir', 'Bundle', 'Binary', 'args_manager']


## A generic error thrown by Pyg
class PygError(Exception):
    pass

## An error thrown when the installation goes wrong
class InstallationError(PygError):
    pass

## This isn't really an error but it is useful when Pyg is used as a library
class AlreadyInstalled(InstallationError):
    pass


#class Version(object): ## OLD!! Does not work properly!
#    def __init__(self, v):
#        self.v = v
#        while self.v[-1] == 0:
#            self.v = self.v[:-2]
#
#    def __str__(self):
#        return self.v
#
#    def __repr__(self):
#        return 'Version({0})'.format(self.v)
#
#    def __eq__(self, other):
#        if len(self.v) != len(other.v):
#            return False
#        return self.v == other.v
#
#    def __ge__(self, other):
#        return self.v >= other.v
#
#    def __gt__(self, other):
#        return self.v > other.v
#
#    def __le__(self, other):
#        return self.v <= other.v
#
#    def __lt__(self, other):
#        return self.v < other.v


class Version(object):
    '''
    This class implements a Version object with comparison methods::

        >>> Version('0.1') < Version('0.1.1')
        True
        >>> Version('0.1') == Version('0.1')
        True
        >>> Version('0.1') >= Version('0.1')
        True
        >>> Version('0.1b') > Version('0.1')
        False
        >>> Version('0.1b') > Version('0.1a')
        True
    '''

    def __init__(self, v):
        self._v = v
        self.v = pkg_resources.parse_version(v)

    def __str__(self):
        return str(self._v)

    def __repr__(self):
        return 'Version({0})'.format(self._v)

    def __str__(self):
        return self._v

    def __eq__(self, o):
        return self.v == o.v

    def __ge__(self, o):
        return self.v >= o.v

    def __gt__(self, o):
        return self.v > o.v

    def __le__(self, o):
        return self.v <= o.v

    def __lt__(self, o):
        return self.v < o.v


class ReqSet(object):
    '''
    A requirement set, used by :class:`~pyg.types.Archive` and :class:`~pyg.types.Egg` to keep trace of package's requirements.
    '''

    def __init__(self, comes_from):
        self._reqs = set()
        self.comes_from = comes_from

    def __iter__(self):
        for r in self.reqs:
            yield r

    def __nonzero__(self):
        return bool(self.reqs)

    def __bool__(self):
        return bool(self.reqs)

    def add(self, r):
        self._reqs.add(r)

    @ property
    def reqs(self):
        return self._reqs


class Egg(object):
    '''
    This class represent a Python Egg object. It needs a file-like object with egg data.

    :param fobj: the file-like object with egg data
    :param eggname: the egg name (for example ``pyg-0.1.2-py2.7.egg``)
    :param reqset: a :class:`~pyg.types.ReqSet` object used to store requirements
    :param packname: the package name. If the egg name is ``pyg-0.1.2-py2.7.egg``, *packname* should be ``pyg``
    '''

    def __init__(self, fobj, eggname, reqset, packname=None):
        self.fobj = fobj
        self.eggname = os.path.basename(eggname)
        self.reqset = reqset
        self.packname = packname or name_from_egg(eggname)
        self.idir = args_manager['install']['install_dir']
        if args_manager['install']['user']:
            self.idir = USER_SITE

    def install(self):
        eggpath = os.path.join(self.idir, self.eggname)
        if os.path.exists(eggpath):
            logger.info('{0} is already installed', self.packname)
            raise AlreadyInstalled
        logger.info('Installing {0} egg file', self.packname)
        with ZipFile(self.fobj) as z:
            z.extractall(eggpath)
        logger.info('Adding egg file to sys.path')
        with open(EASY_INSTALL) as f: ## TODO: Fix the opening mode to read and write simultaneously
            lines = f.readlines()
        with open(EASY_INSTALL, 'w') as f:
            try:
                f.writelines(lines[:-1])
                f.write('./' + self.eggname + '\n')
                f.write(lines[-1])
            ## When using this file for the first time
            except IndexError:
                pass
        dist = EggDir(eggpath)

        ## Install scripts in the setuptools' way
        if not args_manager['install']['no_scripts']:
            for name, content, mode in script_args(dist):
                logger.info('Installing {0} script to {1}', name, BIN)
                target = os.path.join(BIN, name)
                with open(target, 'w' + mode) as f:
                    f.write(content)
                    os.chmod(target, 0755)
        else:
            logger.info('Scripts not installed')
        logger.info('Looking for requirements...')
        try:
            for req in dist.file('requires.txt'):
                self.reqset.add(req)
        except KeyError:
            logger.debug('requires.txt not found')


class Dir(object):
    '''
    To install the package this class needs the *path*, a temporary directory's path, and a :class:`ReqSet` object
    to store requirements.
    '''

    def __init__(self, path, name, tempdir, reqset=None):
        self.path = path
        self.name = name
        self.tempdir = tempdir
        self.reqset = reqset

    def install(self):
        '''
        Given a pathname containing a :file:`setup.py` file, install the package.
        First runs `setup.py egg_info` to find out requirements, then runs ``setup.py install``.
        '''

        if self.reqset is not None:
            logger.info('Running setup.py egg_info for {0}', self.name)
            call_setup(self.path, ['egg_info', '--egg-base', self.tempdir])
            try:
                dist = DirTools(glob.glob(os.path.join(self.path, '*egg-info'))[0])
            except (IndexError, ValueError):
                pass
            else:
                try:
                    for r in dist.file('requires.txt'):
                        self.reqset.add(r)
                except (KeyError, ConfigParser.MissingSectionHeaderError):
                    logger.debug('debug: requires.txt not found')
                #try:
                #    for r in dist.file('dependency_links.txt'):
                #        self.reqset.add(r)
                #except (KeyError, ConfigParser.MissingSectionHeaderError):
                    #logger.debug('debug: dependency_links.txt not found')
        args = []
        if args_manager['install']['install_dir'] != INSTALL_DIR:
            args += ['--prefix', args_manager['install']['install_dir']]
        if args_manager['install']['no_scripts']:
            args += ['--install-scripts', self.tempdir]
        if args_manager['install']['no_data']:
            args += ['--install-data', self.tempdir]
        if args_manager['install']['user']:
            args += ['--user']
        run_setup(self.path, self.name, args=args, exc=InstallationError)

{'fraction_non_alphanumeric': 0.12731900073672225, 'fraction_numerical': 0.006429576049829215, 'mean_word_length': 2.128430756337733, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241966', 'n_tokens_mistral': 4681, 'n_tokens_neox': 4487, 'n_words': 1241}