<commit_before>import os
import sys
import logging
import importlib
from collections import namedtuple
from contextlib import contextmanager

logger = logging.getLogger(__name__)


def import_task(module_name, class_name, function_name, path=None):
    """Find and return the function for given function name."""
    namespace = import_module(module_name, path)
    if class_name:
        cls = getattr(namespace, class_name)
        namespace = cls

    return getattr(namespace, function_name)


def import_module(module_name, path=None):
    """Import module by searching main module, current working directory and
    Python path.

    """
    main_module, main_module_name = get_main_module()
    if module_name == main_module_name:
        return main_module

    if path is None:
        path = os.getcwd()

    with custom_path(path):
        return importlib.import_module(module_name)


def import_class_str(s):
    module, cls = s.rsplit('.', 1)
    module = import_module(module)
    return getattr(module, cls)


@contextmanager
def custom_path(path):
    if path in sys.path:
        yield
    else:
        sys.path.insert(0, path)
        try:
            yield
        finally:
            sys.path.remove(path)


def get_main_module():
    """Returns main module and module name pair."""
    main_module = sys.modules['__main__']
    if not hasattr(main_module, '__file__'):
        # if run from interactive shell
        return None, None
    filename = os.path.basename(main_module.__file__)
    module_name, ext = os.path.splitext(filename)
    MainModule = namedtuple('MainModule', ['module', 'name'])
    return MainModule(module=main_module, name=module_name)<commit_msg>refactor importer<commit_after>import os
import sys
import logging
import importlib
from collections import namedtuple

logger = logging.getLogger(__name__)


def import_task(module_name, class_name, function_name):
    """Find and return the function for given function name."""
    namespace = import_module(module_name)
    if class_name:
        namespace = getattr(namespace, class_name)
    return getattr(namespace, function_name)


def import_module(module_name):
    """Import module by searching main module, current working directory and
    Python path.

    """
    module, main_module_name = get_main_module()
    if module_name == main_module_name:
        return module
    return importlib.import_module(module_name)


def import_class_str(s):
    module, cls = s.rsplit('.', 1)
    module = import_module(module)
    return getattr(module, cls)


def get_main_module():
    """Returns main module and module name pair."""
    if not hasattr(main_module, '__file__'):
        # if run from interactive shell
        return None, None
    main_filename = os.path.basename(main_module.__file__)
    module_name, ext = os.path.splitext(main_filename)
    return FakeModule(module=main_module, name=module_name)


FakeModule = namedtuple('MainModule', ['module', 'name'])
main_module = sys.modules['__main__']
{'fraction_non_alphanumeric': 0.09211398277004638, 'fraction_numerical': 0.0009940357852882703, 'mean_word_length': 3.709828393135725, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3348335', 'n_tokens_mistral': 883, 'n_tokens_neox': 870, 'n_words': 246}