<commit_before>#!/usr/bin/env python

from argparse import ArgumentParser, RawDescriptionHelpFormatter
try:
    from collections.abc import Mapping, Sequence
except ImportError:
    from collections import Mapping, Sequence
from copy import deepcopy
import ctypes
from distutils.spawn import find_executable # pylint:disable=no-name-in-module,import-error
import errno
from importlib import import_module
from json import load as json_loadf
import inspect
import os
import signal
from subprocess import Popen
import sys

try:
    input = raw_input # pylint:disable=redefined-builtin
except NameError:
    pass


def merge_dict(result, *dicts):
    for data in dicts:
        for k, v in data.items():
            if k in result:
                try:
                    is_str = isinstance(v, basestring)
                except NameError:
                    is_str = isinstance(v, str)

                if isinstance(v, Mapping):
                    # Will raise exception if result[k] is not a Mapping
                    merge_dict(result[k], v)
                elif isinstance(v, Sequence) and not is_str:
                    result[k].extend(deepcopy(v))
                else:
                    result[k] = deepcopy(v)
            else:
                result[k] = deepcopy(v)


def flatten_dict(prefix, data):
    # see http://stackoverflow.com/a/6036037
    for key in data:
        item = data[key]
        if prefix:
            new_prefix = prefix + '_' + key.upper()
        else:
            new_prefix = key.upper()

        try:
            is_str = isinstance(item, basestring)
        except NameError:
            is_str = isinstance(item, str)

        if isinstance(item, Mapping):
            for child_item in flatten_dict(new_prefix, item):
                yield child_item
        elif isinstance(item, Sequence) and not is_str:
            yield (new_prefix, os.pathsep.join(item))
        else:
            yield (new_prefix, item)


class Loader():

    config = None

    _modules = None
    _python_bin = None
    _CSL = None

    def __init__(self, config):
        self.config = config
        self._modules = []

        if os.name == 'nt':
            os.execvp = self._win_execvp

            if not hasattr(os, 'symlink'):
                os.symlink = self._win_symlink


    def setup_project_env(self, project=None, variant=None):
        project = self.config['project'] = project or\
                os.environ.get('PROJECT_NAME',
                self.config.get('default_project'))

        variant = self.config['variant'] = variant or\
                os.environ.get('PROJECT_VARIANT',
                self.config.get('default_variant', 'default'))

        if self.config.get('variant_is_production') or\
                variant in ['prod', 'production', 'staging', 'stage']:
            is_prod = '1'
        else:
            is_prod = '0'

        os.environ['PROJECT_NAME'] = project
        os.environ['PROJECT_VARIANT'] = variant
        os.environ['PROJECT_VARIANT_IS_PRODUCTION'] = is_prod
        return project, variant


    def setup_shell_env(self, data=None):
        config = self.get_project_config()
        env = config.get('shell_env', {})
        for key, value in (data or {}).items():
            env[key] = value

        # Expand PYTHONPATH relative to project directory
        for idx, value in enumerate(env.get('PYTHONPATH', [])):
            env['PYTHONPATH'][idx] = os.path.join(self.config['work_dir'],
                    value)
        for key, value in flatten_dict('', env):
            os.environ[key] = value


    def setup_virtualenv(self):
        venv_dir = self.get_virtualenv_dir()
        venv_bin_dir = os.path.join(venv_dir, 'bin')
        venv_type = self.config.get('virtualenv_type', 'python')

        self._python_bin = None

        if venv_type in ('venv', 'virtualenv'):
            os.environ['VIRTUAL_ENV'] = venv_dir
            if 'PYTHONHOME' in os.environ:
                del os.environ['PYTHONHOME']<commit_msg>support PATH in shell_env, fix PYTHONPATH<commit_after>#!/usr/bin/env python

from argparse import ArgumentParser, RawDescriptionHelpFormatter
try:
    from collections.abc import Mapping, Sequence
except ImportError:
    from collections import Mapping, Sequence
from copy import deepcopy
import ctypes
from distutils.spawn import find_executable # pylint:disable=no-name-in-module,import-error
import errno
from importlib import import_module
from json import load as json_loadf
import inspect
import os
import signal
from subprocess import Popen
import sys

try:
    input = raw_input # pylint:disable=redefined-builtin
except NameError:
    pass


def merge_dict(result, *dicts):
    for data in dicts:
        for k, v in data.items():
            if k in result:
                try:
                    is_str = isinstance(v, basestring)
                except NameError:
                    is_str = isinstance(v, str)

                if isinstance(v, Mapping):
                    # Will raise exception if result[k] is not a Mapping
                    merge_dict(result[k], v)
                elif isinstance(v, Sequence) and not is_str:
                    result[k].extend(deepcopy(v))
                else:
                    result[k] = deepcopy(v)
            else:
                result[k] = deepcopy(v)


def flatten_dict(prefix, data):
    # see http://stackoverflow.com/a/6036037
    for key in data:
        item = data[key]
        if prefix:
            new_prefix = prefix + '_' + key.upper()
        else:
            new_prefix = key.upper()

        try:
            is_str = isinstance(item, basestring)
        except NameError:
            is_str = isinstance(item, str)

        if isinstance(item, Mapping):
            for child_item in flatten_dict(new_prefix, item):
                yield child_item
        elif isinstance(item, Sequence) and not is_str:
            yield (new_prefix, os.pathsep.join(item))
        else:
            yield (new_prefix, item)


class Loader():

    config = None

    _modules = None
    _python_bin = None
    _CSL = None

    def __init__(self, config):
        self.config = config
        self._modules = []

        if os.name == 'nt':
            os.execvp = self._win_execvp

            if not hasattr(os, 'symlink'):
                os.symlink = self._win_symlink


    def setup_project_env(self, project=None, variant=None):
        project = self.config['project'] = project or\
                os.environ.get('PROJECT_NAME',
                self.config.get('default_project'))

        variant = self.config['variant'] = variant or\
                os.environ.get('PROJECT_VARIANT',
                self.config.get('default_variant', 'default'))

        if self.config.get('variant_is_production') or\
                variant in ['prod', 'production', 'staging', 'stage']:
            is_prod = '1'
        else:
            is_prod = '0'

        os.environ['PROJECT_NAME'] = project
        os.environ['PROJECT_VARIANT'] = variant
        os.environ['PROJECT_VARIANT_IS_PRODUCTION'] = is_prod
        return project, variant


    def setup_shell_env(self, data=None):
        config = self.get_project_config()
        env = config.get('shell_env', {})
        for key, value in (data or {}).items():
            env[key] = value

        # Expand PATH relative to project directory
        paths = [self.expand_path(x) for x in env.pop('PATH', [])]
        if paths:
            paths += os.environ.get('PATH', '').split(os.pathsep)
            os.environ['PATH'] = os.pathsep.join(paths)

        # Expand PYTHONPATH relative to project directory
        paths = [os.expand_path(x) for x in env.pop('PYTHONPATH', [])]
        if paths:
            paths += os.environ.get('PYTHONPATH', '').split(os.pathsep)
            os.environ['PYTHONPATH'] = os.pathsep.join(paths)

        for key, value in flatten_dict('', env):
            os.environ[key] = value


    def setup_virtualenv(self):
        venv_dir = self.get_virtualenv_dir()
        venv_bin_dir = os.path.join(venv_dir, 'bin')
        venv_type = self.config.get('virtualenv_type', 'python')

        self._python_bin = None

        if venv_type in ('venv', 'virtualenv'):
            os.environ['VIRTUAL_ENV'] = venv_dir
            if 'PYTHONHOME' in os.environ:
                del os.environ['PYTHONHOME']
{'fraction_non_alphanumeric': 0.10274463007159905, 'fraction_numerical': 0.0021479713603818618, 'mean_word_length': 2.0531876138433516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6224455', 'n_tokens_mistral': 2520, 'n_tokens_neox': 2424, 'n_words': 618}