<commit_before>"""
Utilities

    :copyright: 2008 by <NAME> <<EMAIL>>
    :license: MIT (See ``LICENSE`` file for more details)

"""
import os
import sys
import stat


def wrap_sequence(obj, sequence_type=tuple):
    """
    Return a tuple whose item is obj.
    If obj is already a list or tuple, it is returned unchanged.
    """
    if isinstance(obj, (list, tuple)):
        return obj
    else:
        return sequence_type((obj,))

def compile_python_source(filepath, optimization=False):
    """
    Compile a source file to byte-code and write out
    the byte-code cache file. The source code is loaded
    from the file name *filepath*. The byte-code is cached
    in the normal manner (*filepath* + 'c' or 'o' if
    optimization is enabled).
    """
    from os import spawnv
    from sys import executable, platform

    args = [executable]
    if optimization:
        args.append('-O')
    args += ['-m', 'py_compile']

    if isinstance(filepath, (tuple, list)):
        args.extend(filepath)
    else:
        args.append(filepath)

    if platform == "win32":
        args = ['"%s"' % arg for arg in args]
    return spawnv(os.P_WAIT, executable, args)


# ----------------------------------------------------------------
# File browser
# ----------------------------------------------------------------
def collect_files(filepath_or_list, ignore_list=None):
    """
    ``collect_files()`` generates the file names in a directory tree.
    Note: ``collect_files()`` will not visit symbolic links to
    subdirectories. *ignore_list* argument is ignore filename patterns
    (using fnmatch).
    """
    import fnmatch

    def ignore(filename):
        if ignore_list:
            for pattern in ignore_list:
                if fnmatch.fnmatch(filename, pattern):
                    return True
        return False

    for filepath in wrap_sequence(filepath_or_list):

        if ignore(os.path.basename(filepath)):
            continue

        if not os.path.exists(filepath):
            from errno import ENOENT
            raise IOError(ENOENT, "No such file or directory", filepath)
        elif not os.path.isdir(filepath):
            yield filepath
        else:
            # pylint: disable-msg=W0612
            for dirpath, dirnames, filenames in os.walk(filepath):
                # For in-place deletion (avoids copying the list),
                # Don't delete anything earlier in the list than
                # the current element through.
                for i, dirname in enumerate(reversed(dirnames)):
                    if ignore(dirname):
                        # don't visit ignore directory
                        del dirnames[-(i+1)]
                for filename in filenames:
                    if not ignore(filename):
                        yield os.path.join(dirpath, filename)


# ----------------------------------------------------------------
# Path utilities
# ----------------------------------------------------------------
def python_module_file(filepath):
    """
    Return ``True`` if path is an existing Python source
    file (*.py*). This follows symbolic links.
    """
    return (python_source_file(filepath) or
            python_compiled_file(filepath))

def python_source_file(filepath):
    """
    Return ``True`` if path is an existing Python source
    file (*.py*). This follows symbolic links.
    """
    try:
        st = os.stat(filepath)
    except (TypeError, os.error):
        return False
    else:
        return (stat.S_ISREG(st.st_mode) and
                filepath.endswith('.py'))

def python_compiled_file(filepath):
    """
    Return ``True`` if path is an existing Python compiled
    bytecode file (*.pyc* or *.pyo*). This follows symbolic links.
    """
    try:
        st = os.stat(filepath)
    except (TypeError, os.error):
        return False
    else:
        return (stat.S_ISREG(st.st_mode) and
                   (filepath.endswith('.pyc') or
                    filepath.endswith('.pyo')))

def python_package(dirpath):
    """
    Return ``True`` if *dirpath* is an existing directory that is
    Python package directory (contains ``__init__.py[co]``).
    """
    from os.path import isdir
    return isdir(dirpath) and python_module_exists(dirpath, '__init__')

def python_module_exists(dirpath, modulename):
    """
    Return ``True`` if *dirpath* refers to an existing directory that
    contains Python module named *modulename*.
    """
    from os.path import isdir, isfile, join
    return (isdir(dirpath) and
                (isfile(join(dirpath, '%s.py' % modulename)) or
                 isfile(join(dirpath, '%s.pyc' % modulename)) or
                 isfile(join(dirpath, '%s.pyo' % modulename))))


# ----------------------------------------------------------------
# Working with modules
# ----------------------------------------------------------------
def import_module(modulename):
    """
    Return loaded module if module is already loaded in
    ``sys.modules``, otherwise load module and return it.
    """
    if modulename not in sys.modules:
        __import__(modulename)
    return sys.modules[modulename]

## ``resolve_modulename`` is major bottleneck of performance :-(<commit_msg>Improved ModuleNameResolver<commit_after>"""
Utilities

    :copyright: 2008 by <NAME> <<EMAIL>>
    :license: MIT (See ``LICENSE`` file for more details)

"""
import os
import sys
import stat


def wrap_sequence(obj, sequence_type=tuple):
    """
    Return a tuple whose item is obj.
    If obj is already a list or tuple, it is returned unchanged.
    """
    if isinstance(obj, (list, tuple)):
        return obj
    else:
        return sequence_type((obj,))

def compile_python_source(filepath, optimization=False):
    """
    Compile a source file to byte-code and write out
    the byte-code cache file. The source code is loaded
    from the file name *filepath*. The byte-code is cached
    in the normal manner (*filepath* + 'c' or 'o' if
    optimization is enabled).
    """
    from os import spawnv
    from sys import executable, platform

    args = [executable]
    if optimization:
        args.append('-O')
    args += ['-m', 'py_compile']

    if isinstance(filepath, (tuple, list)):
        args.extend(filepath)
    else:
        args.append(filepath)

    if platform == "win32":
        args = ['"%s"' % arg for arg in args]
    return spawnv(os.P_WAIT, executable, args)


# ----------------------------------------------------------------
# File browser
# ----------------------------------------------------------------
def collect_files(filepath_or_list, ignore_list=None):
    """
    ``collect_files()`` generates the file names in a directory tree.
    Note: ``collect_files()`` will not visit symbolic links to
    subdirectories. *ignore_list* argument is ignore filename patterns
    (using fnmatch).
    """
    import fnmatch

    def ignore(filename):
        if ignore_list:
            for pattern in ignore_list:
                if fnmatch.fnmatch(filename, pattern):
                    return True
        return False

    for filepath in wrap_sequence(filepath_or_list):

        if ignore(os.path.basename(filepath)):
            continue

        if not os.path.exists(filepath):
            from errno import ENOENT
            raise IOError(ENOENT, "No such file or directory", filepath)
        elif not os.path.isdir(filepath):
            yield filepath
        else:
            # pylint: disable-msg=W0612
            for dirpath, dirnames, filenames in os.walk(filepath):
                # For in-place deletion (avoids copying the list),
                # Don't delete anything earlier in the list than
                # the current element through.
                for i, dirname in enumerate(reversed(dirnames)):
                    if ignore(dirname):
                        # don't visit ignore directory
                        del dirnames[-(i+1)]
                for filename in filenames:
                    if not ignore(filename):
                        yield os.path.join(dirpath, filename)


# ----------------------------------------------------------------
# Path utilities
# ----------------------------------------------------------------
def python_module_file(filepath):
    """
    Return ``True`` if path is an existing Python source
    file (*.py*). This follows symbolic links.
    """
    return (python_source_file(filepath) or
            python_compiled_file(filepath))

def python_source_file(filepath):
    """
    Return ``True`` if path is an existing Python source
    file (*.py*). This follows symbolic links.
    """
    try:
        st = os.stat(filepath)
    except (TypeError, os.error):
        return False
    else:
        return (stat.S_ISREG(st.st_mode) and
                filepath.endswith('.py'))

def python_compiled_file(filepath):
    """
    Return ``True`` if path is an existing Python compiled
    bytecode file (*.pyc* or *.pyo*). This follows symbolic links.
    """
    try:
        st = os.stat(filepath)
    except (TypeError, os.error):
        return False
    else:
        return (stat.S_ISREG(st.st_mode) and
                   (filepath.endswith('.pyc') or
                    filepath.endswith('.pyo')))

def python_package(dirpath):
    """
    Return ``True`` if *dirpath* is an existing directory that is
    Python package directory (contains ``__init__.py[co]``).
    """
    # os.path.isdir() check is not needed
    return python_module_exists(dirpath, '__init__')

def python_module_exists(dirpath, modulename):
    """
    Return ``True`` if *dirpath* refers to an existing directory that
    contains Python module named *modulename*.
    """
    # os.path.isdir() check is not needed
    from os.path import isfile, join
    return (isfile(join(dirpath, '%s.py' % modulename)) or
                 isfile(join(dirpath, '%s.pyc' % modulename)) or
                 isfile(join(dirpath, '%s.pyo' % modulename)))


# ----------------------------------------------------------------
# Working with modules
# ----------------------------------------------------------------
def import_module(modulename):
    """
    Return loaded module if module is already loaded in
    ``sys.modules``, otherwise load module and return it.
    """
    if modulename not in sys.modules:
        __import__(modulename)
    return sys.modules[modulename]

## ``resolve_modulename`` is major bottleneck of performance :-(
{'fraction_non_alphanumeric': 0.16954984230144318, 'fraction_numerical': 0.0021026474242569052, 'mean_word_length': 1.859016393442623, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6766776', 'n_tokens_mistral': 2850, 'n_tokens_neox': 2725, 'n_words': 892}