<commit_before>import binascii
import hmac
import importlib
import inspect
import os
import random
import string
from types import ModuleType


random = random.SystemRandom()


NOT_SET = type('NOT_SET', (), {
    '__bool__': (lambda self: False),
    '__str__': (lambda self: 'NOT_SET'),
    '__repr__': (lambda self: 'NOT_SET'),
    '__copy__': (lambda self: self),
})()
"""A ``None``-ish constant for use where ``None`` may be a valid value."""


def fully_qualified_name(obj):
    """Get the fully qualified name for an object.

    Returns the object's module name joined with its qualified name. If
    the object is a module, its name is returned.

    >>> fully_qualified_name(object)
    'builtins.object'
    >>> import tangled.util
    >>> fully_qualified_name(tangled.util)
    'tangled.util'

    """
    if isinstance(obj, ModuleType):
        return obj.__name__
    return '{}.{}'.format(obj.__module__, obj.__qualname__)


def caller_package(level=2):
    frame = inspect.stack()[level][0]
    package = frame.f_globals['__package__']
    return package


def load_object(obj, obj_name=None, package=None, level=2):
    """Load an object.

    ``obj`` may be an object or a string that points to an object. If
    it's a string, the object will be loaded and returned from the
    specified path. If it's any other type of object, it will be
    returned as is.

    The format of a path string is either 'package.module' to load a
    module or 'package.module:object' to load an object from a module.

    The object name can be passed via ``obj_name`` instead of in the
    path (if the name is passed both ways, the name in the path will
    win).

    Examples::

        >>> load_object('tangled.util:load_object')
        <function load_object at ...>
        >>> load_object('tangled.util', 'load_object')
        <function load_object at ...>
        >>> load_object('tangled.util:load_object', 'IGNORED')
        <function load_object at ...>
        >>> load_object('.util:load_object', package='tangled')
        <function load_object at ...>
        >>> load_object('.:load_object', package='tangled.util')
        <function load_object at ...>
        >>> load_object(':load_object', package='tangled.util')
        <function load_object at ...>
        >>> load_object(load_object)
        <function load_object at ...>
        >>> load_object(load_object, 'IGNORED', 'IGNORED', 'IGNORED')
        <function load_object at ...>

    """
    if isinstance(obj, str):
        if ':' in obj:
            module_name, obj_name = obj.split(':')
            if not module_name:
                module_name = '.'
        else:
            module_name = obj
        if module_name.startswith('.') and package is None:
            package = caller_package(level)
        obj = importlib.import_module(module_name, package)
        if obj_name:
            attrs = obj_name.split('.')
            for attr in attrs:
                obj = getattr(obj, attr)
    return obj


def asset_path(path, *rel_path):
    """Get absolute path to asset in package.

    ``path`` can be just a package name like 'tangled.web' or it can be
    a package name and a relative file system path like
    'tangled.web:some/path'.

    If ``rel_path`` is passed, it will be appended to the base rel. path
    in ``path``.

    Examples::

        >>> asset_path('tangled.util')
        '.../tangled/tangled'
        >>> asset_path('tangled.util:')
        '.../tangled/tangled'
        >>> asset_path('tangled.util:x')
        '.../tangled/tangled/x'
        >>> asset_path('tangled.util', 'x')
        '.../tangled/tangled/x'
        >>> asset_path('tangled.util:x', 'y')
        '.../tangled/tangled/x/y'
<commit_msg>Add is_module_path() utility function<commit_after>import binascii
import hmac
import importlib
import inspect
import os
import random
import string
from types import ModuleType


random = random.SystemRandom()


NOT_SET = type('NOT_SET', (), {
    '__bool__': (lambda self: False),
    '__str__': (lambda self: 'NOT_SET'),
    '__repr__': (lambda self: 'NOT_SET'),
    '__copy__': (lambda self: self),
})()
"""A ``None``-ish constant for use where ``None`` may be a valid value."""


def fully_qualified_name(obj):
    """Get the fully qualified name for an object.

    Returns the object's module name joined with its qualified name. If
    the object is a module, its name is returned.

    >>> fully_qualified_name(object)
    'builtins.object'
    >>> import tangled.util
    >>> fully_qualified_name(tangled.util)
    'tangled.util'

    """
    if isinstance(obj, ModuleType):
        return obj.__name__
    return '{}.{}'.format(obj.__module__, obj.__qualname__)


def caller_package(level=2):
    frame = inspect.stack()[level][0]
    package = frame.f_globals['__package__']
    return package


def load_object(obj, obj_name=None, package=None, level=2):
    """Load an object.

    ``obj`` may be an object or a string that points to an object. If
    it's a string, the object will be loaded and returned from the
    specified path. If it's any other type of object, it will be
    returned as is.

    The format of a path string is either 'package.module' to load a
    module or 'package.module:object' to load an object from a module.

    The object name can be passed via ``obj_name`` instead of in the
    path (if the name is passed both ways, the name in the path will
    win).

    Examples::

        >>> load_object('tangled.util:load_object')
        <function load_object at ...>
        >>> load_object('tangled.util', 'load_object')
        <function load_object at ...>
        >>> load_object('tangled.util:load_object', 'IGNORED')
        <function load_object at ...>
        >>> load_object('.util:load_object', package='tangled')
        <function load_object at ...>
        >>> load_object('.:load_object', package='tangled.util')
        <function load_object at ...>
        >>> load_object(':load_object', package='tangled.util')
        <function load_object at ...>
        >>> load_object(load_object)
        <function load_object at ...>
        >>> load_object(load_object, 'IGNORED', 'IGNORED', 'IGNORED')
        <function load_object at ...>

    """
    if isinstance(obj, str):
        if ':' in obj:
            module_name, obj_name = obj.split(':')
            if not module_name:
                module_name = '.'
        else:
            module_name = obj
        if module_name.startswith('.') and package is None:
            package = caller_package(level)
        obj = importlib.import_module(module_name, package)
        if obj_name:
            attrs = obj_name.split('.')
            for attr in attrs:
                obj = getattr(obj, attr)
    return obj


def is_module_path(path) -> bool:
    """Is ``path`` a module path like ``package.module``?

    Examples::

        >>> is_module_path('package')
        True
        >>> is_module_path('package.module')
        True
        >>> is_module_path('.module')
        True
        >>> is_module_path('package.module:obj')
        False
        >>> is_module_path('a/b')
        False
        >>> is_module_path('/a/b')
        False

    """
    return all(p.isidentifier() for p in path.lstrip('.').split('.'))


def asset_path(path, *rel_path):
    """Get absolute path to asset in package.

    ``path`` can be just a package name like 'tangled.web' or it can be
    a package name and a relative file system path like
    'tangled.web:some/path'.

    If ``rel_path`` is passed, it will be appended to the base rel. path
    in ``path``.

    Examples::

        >>> asset_path('tangled.util')
        '.../tangled/tangled'
        >>> asset_path('tangled.util:')
        '.../tangled/tangled'
        >>> asset_path('tangled.util:x')
        '.../tangled/tangled/x'
        >>> asset_path('tangled.util', 'x')
        '.../tangled/tangled/x'
        >>> asset_path('tangled.util:x', 'y')
        '.../tangled/tangled/x/y'

{'fraction_non_alphanumeric': 0.1340606972661149, 'fraction_numerical': 0.0007524454477050414, 'mean_word_length': 2.574630210667862, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 134, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2920471', 'n_tokens_mistral': 2437, 'n_tokens_neox': 2374, 'n_words': 718}