<commit_before>import os
from contextlib import contextmanager
from distutils.dir_util import copy_tree
from importlib import import_module
from shutil import rmtree
from tempfile import mkdtemp


@contextmanager
def expose_modules(*modulenames):
    """

    :type modulenames: list
    """
    tmpdir = mkdtemp()

    def modulename_to_srcdst(m):
        mod = import_module(m)
        src_ = os.path.dirname(os.path.abspath(mod.__file__))
        dst_ = os.path.join(tmpdir, os.path.basename(src_))
        return src_, dst_

    try:
        copypaths = dict(map(modulename_to_srcdst, modulenames))
        [copy_tree(src, dst) for (src, dst) in copypaths.iteritems()]
        yield tmpdir
    finally:
        rmtree(tmpdir, ignore_errors=True)


def mergeenv(moreenv=None, pythonpaths=None):
    if moreenv is None:
        moreenv = {}
    if pythonpaths is None:
        pythonpaths = []
    env = merge_dicts(os.environ, moreenv)
    env["PYTHONDONTWRITEBYTECODE"] = "true"<commit_msg>Py3 compatibility<commit_after>import os
from contextlib import contextmanager
from distutils.dir_util import copy_tree
from importlib import import_module
from shutil import rmtree
from tempfile import mkdtemp


@contextmanager
def expose_modules(*modulenames):
    """

    :type modulenames: list
    """
    tmpdir = mkdtemp()

    def modulename_to_srcdst(m):
        mod = import_module(m)
        src_ = os.path.dirname(os.path.abspath(mod.__file__))
        dst_ = os.path.join(tmpdir, os.path.basename(src_))
        return src_, dst_

    try:
        copypaths = dict(map(modulename_to_srcdst, modulenames))
        [copy_tree(src, dst) for (src, dst) in copypaths.items()]
        yield tmpdir
    finally:
        rmtree(tmpdir, ignore_errors=True)


def mergeenv(moreenv=None, pythonpaths=None):
    if moreenv is None:
        moreenv = {}
    if pythonpaths is None:
        pythonpaths = []
    env = merge_dicts(os.environ, moreenv)
    env["PYTHONDONTWRITEBYTECODE"] = "true"
{'fraction_non_alphanumeric': 0.096579476861167, 'fraction_numerical': 0.0005030181086519115, 'mean_word_length': 3.2866379310344827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7031089', 'n_tokens_mistral': 670, 'n_tokens_neox': 657, 'n_words': 152}