<commit_before>#!/usr/bin/env python
#
#   Copyright 2014 Telenor Digital AS
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

"""Bootstrap installation of transmute package.

When imported (or directly executed), this module will make the latest transmute
package available, downloading it from PyPI as needed. Dowloaded packages are
stored in a local cache and will be reused in subsequent runs.

This can also be used to bootstrap applications that use the transmute package.
In this case, the module becomes the application's main executable. A couple of
customization points are provided to support this use case with minimal effort:

    requirements: global variable listing packages to be fetched from PyPI.
    main(): placeholder for application specific logic. If the module is
        executed as __main__ script, this gets called after packages in
        requirements have been updated and added to sys.path.

Additionally, bootstrap_starting(), bootstrap_succeeded() and bootstrap_failed()
are called at specific points in the bootstrapping process.

The following example could be used to load latest 'foobar' from PyPI and invoke
the foobar.cli.main() entry point:

    requirements = [ 'foobar' ]
    def main():
        import foobar.cli
        return foobar.cli.main()

"""


################################################################################
# Customization points to support running module as main application script.
################################################################################

requirements = [ 'transmute' ]

def main():
    """Called when module is '__main__', after successful bootstrap."""
    ### ~~~ Your code here ~~~ ###

def bootstrap_starting():
    """Called before attempting to load/download packages."""

def bootstrap_succeeded():
    """Called after (updated) packages have been added to sys.path."""
    global _bootstrapped_by_transmute
    _bootstrapped_by_transmute = True

def bootstrap_failed():
    """Called when bootstrap fails to download packages from PyPI or load them
    from the local cache.
    """
    raise RuntimeError("Unable to load 'transmute' package.")


################################################################################
# transmute.bootstrap code follows.
################################################################################

import os
import os.path
import sys

def _chunk_read(file, chunk_size=16*1024):
    """Read file one chunk at a time."""

    while True:
        chunk = file.read(chunk_size)
        if chunk == '':
            break
        yield chunk

def _md5(filename):
    """Compute MD5 hash of a file."""

    import hashlib

    h = hashlib.md5()
    try:
        with open(filename) as file:
            for chunk in _chunk_read(file):
                h.update(chunk)
    except: pass

    return h.hexdigest()

def _copy(source, destination, chunk_size):
    """Read source into destination, one chunk at a time."""

    for chunk in _chunk_read(source, chunk_size):
        destination.write(chunk)

def _download(url, filename, md5sum):
    """Download url and save to filename, verify MD5 hash of content.

    Content is initially saved to a temporary file and the content's MD5 hash is
    verified before the file is atomically renamed to its desired filename.
    """
    import contextlib
    import tempfile
    import urllib2

    dirname = os.path.dirname(filename)
    dst = tempfile.NamedTemporaryFile(suffix='.download', dir=dirname)
    with contextlib.closing(dst):
        with contextlib.closing(urllib2.urlopen(url)) as src:
            _copy(src, dst, 4 * 1024)
        dst.flush()

        if _md5(dst.name) != md5sum:
            raise RuntimeError("MD5 hash of local file does not match "
                    "expected value, upon downloading %s" % url)

        # Depend on non-documented implementation of NamedTemporaryFile, a
        # shame, but prettier than the alternative
        dst.delete = False

        os.rename(dst.name, filename)

def require(baskets, *requirements):
    """Satisfy requirements from given baskets."""

    import pkg_resources

    requirements = list(pkg_resources.parse_requirements(requirements))

    environment = pkg_resources.Environment([])
    for basket in baskets:
        basket.fill_environment(environment, requirements)

    working_set = pkg_resources.WorkingSet([])
    for dist in working_set.resolve(requirements, env=environment):
        dist._transmute_basket.make_local(dist)
        working_set.add(dist)

    for entry in sys.path:
        if entry not in working_set.entries:
            working_set.add_entry(entry)

    sys.path[:] = working_set.entries


class Basket(object):
    """A container for Python Eggs."""

    _cache_dir = os.path.expanduser('~/.python-transmute/cache')

    def __init__(self, path=None, url=None):
        assert (path is None) != (url is None)

        if path is None:
            path = self._prepare_cache(url)

        self.path = os.path.join(path, '') # Keep trailing separator!
        self.distributions = {}

        self.add(*os.listdir(self.path))

    def _prepare_cache(self, url):
        import urllib

        path = os.path.join(self._cache_dir, urllib.quote(url, ''))
        if not os.path.isdir(path):
            os.makedirs(path)
        return path

    @classmethod
    def _is_egg(cls, filename):
        return filename[-4] == '.' \
                and filename[-3:].lower() == 'egg'

    def get_distribution(self, egg, **metadata):
        from pkg_resources import Distribution

        dist = Distribution.from_location(self.path + egg, egg)
        dist._transmute_basket = self
        dist._transmute_metadata = metadata<commit_msg>Allow packages in sys.path to satisfy bootstrap requirements<commit_after>#!/usr/bin/env python
#
#   Copyright 2014 Telenor Digital AS
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

"""Bootstrap installation of transmute package.

When imported (or directly executed), this module will make the latest transmute
package available, downloading it from PyPI as needed. Dowloaded packages are
stored in a local cache and will be reused in subsequent runs.

This can also be used to bootstrap applications that use the transmute package.
In this case, the module becomes the application's main executable. A couple of
customization points are provided to support this use case with minimal effort:

    requirements: global variable listing packages to be fetched from PyPI.
    main(): placeholder for application specific logic. If the module is
        executed as __main__ script, this gets called after packages in
        requirements have been updated and added to sys.path.

Additionally, bootstrap_starting(), bootstrap_succeeded() and bootstrap_failed()
are called at specific points in the bootstrapping process.

The following example could be used to load latest 'foobar' from PyPI and invoke
the foobar.cli.main() entry point:

    requirements = [ 'foobar' ]
    def main():
        import foobar.cli
        return foobar.cli.main()

"""


################################################################################
# Customization points to support running module as main application script.
################################################################################

requirements = [ 'transmute' ]

def main():
    """Called when module is '__main__', after successful bootstrap."""
    ### ~~~ Your code here ~~~ ###

def bootstrap_starting():
    """Called before attempting to load/download packages."""

def bootstrap_succeeded():
    """Called after (updated) packages have been added to sys.path."""
    global _bootstrapped_by_transmute
    _bootstrapped_by_transmute = True

def bootstrap_failed():
    """Called when bootstrap fails to download packages from PyPI or load them
    from the local cache.
    """
    raise RuntimeError("Unable to load 'transmute' package.")


################################################################################
# transmute.bootstrap code follows.
################################################################################

import os
import os.path
import sys

def _chunk_read(file, chunk_size=16*1024):
    """Read file one chunk at a time."""

    while True:
        chunk = file.read(chunk_size)
        if chunk == '':
            break
        yield chunk

def _md5(filename):
    """Compute MD5 hash of a file."""

    import hashlib

    h = hashlib.md5()
    try:
        with open(filename) as file:
            for chunk in _chunk_read(file):
                h.update(chunk)
    except: pass

    return h.hexdigest()

def _copy(source, destination, chunk_size):
    """Read source into destination, one chunk at a time."""

    for chunk in _chunk_read(source, chunk_size):
        destination.write(chunk)

def _download(url, filename, md5sum):
    """Download url and save to filename, verify MD5 hash of content.

    Content is initially saved to a temporary file and the content's MD5 hash is
    verified before the file is atomically renamed to its desired filename.
    """
    import contextlib
    import tempfile
    import urllib2

    dirname = os.path.dirname(filename)
    dst = tempfile.NamedTemporaryFile(suffix='.download', dir=dirname)
    with contextlib.closing(dst):
        with contextlib.closing(urllib2.urlopen(url)) as src:
            _copy(src, dst, 4 * 1024)
        dst.flush()

        if _md5(dst.name) != md5sum:
            raise RuntimeError("MD5 hash of local file does not match "
                    "expected value, upon downloading %s" % url)

        # Depend on non-documented implementation of NamedTemporaryFile, a
        # shame, but prettier than the alternative
        dst.delete = False

        os.rename(dst.name, filename)

def require(baskets, *requirements):
    """Satisfy requirements from given baskets."""

    import pkg_resources

    requirements = list(pkg_resources.parse_requirements(requirements))

    environment = pkg_resources.Environment()
    for basket in baskets:
        basket.fill_environment(environment, requirements)

    working_set = pkg_resources.WorkingSet([])
    for dist in working_set.resolve(requirements, env=environment):
        if hasattr(dist, '_transmute_basket'):
            dist._transmute_basket.make_local(dist)
        working_set.add(dist)

    for entry in sys.path:
        if entry not in working_set.entries:
            working_set.add_entry(entry)

    sys.path[:] = working_set.entries


class Basket(object):
    """A container for Python Eggs."""

    _cache_dir = os.path.expanduser('~/.python-transmute/cache')

    def __init__(self, path=None, url=None):
        assert (path is None) != (url is None)

        if path is None:
            path = self._prepare_cache(url)

        self.path = os.path.join(path, '') # Keep trailing separator!
        self.distributions = {}

        self.add(*os.listdir(self.path))

    def _prepare_cache(self, url):
        import urllib

        path = os.path.join(self._cache_dir, urllib.quote(url, ''))
        if not os.path.isdir(path):
            os.makedirs(path)
        return path

    @classmethod
    def _is_egg(cls, filename):
        return filename[-4] == '.' \
                and filename[-3:].lower() == 'egg'

    def get_distribution(self, egg, **metadata):
        from pkg_resources import Distribution, EGG_DIST

        dist = Distribution.from_location(self.path + egg, egg)
        dist._transmute_basket = self
        dist._transmute_metadata = metadata

        # Prefer local packages.
        dist.precedence = EGG_DIST - 0.1
{'fraction_non_alphanumeric': 0.13335437544385703, 'fraction_numerical': 0.005207922354612168, 'mean_word_length': 3.271654870239299, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3270752', 'n_tokens_mistral': 3459, 'n_tokens_neox': 3292, 'n_words': 1208}