<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""This is a modification of sphinx.apidoc by David.Zuber.
It uses jinja templates to render the rst files.

Parses a directory tree looking for Python modules and packages and creates
ReST files appropriately to create code documentation with Sphinx.

This is derived form the "sphinx-apidoc" script, which is:

  Copyright 2007-2014 by the Sphinx team, see http://sphinx-doc.org/latest/authors.html.
"""
import os
import inspect
import pkgutil
import pkg_resources
import shutil

import jinja2
from sphinx.util.osutil import walk
from sphinx.ext import autosummary


INITPY = '__init__.py'
PY_SUFFIXES = set(['.py', '.pyx'])
TEMPLATE_DIR = 'templates'
"""Built-in template dir for jinjaapi rendering"""
AUTOSUMMARYTEMPLATE_DIR = 'autosummarytemplates'
"""Templates for autosummary"""
MODULE_TEMPLATE_NAME = 'jinjaapi_module.rst'
"""Name of the template that is used for rendering modules."""
PACKAGE_TEMPLATE_NAME = 'jinjaapi_package.rst'
"""Name of the template that is used for rendering packages."""


def prepare_dir(app, directory, delete=False):
    """Create apidoc dir, delete contents if delete is True.

    :param app: the sphinx app
    :type app: :class:`sphinx.application.Sphinx`
    :param directory: the apidoc directory. you can use relative paths here
    :type directory: str
    :param delete: if True, deletes the contents of apidoc. This acts like an override switch.
    :type delete: bool
    :returns: None
    :rtype: None
    :raises: None
    """
    app.info("Preparing output directories for jinjaapidoc.")
    if os.path.exists(directory):
        if delete:
            app.debug("Deleting dir %s", directory)
            shutil.rmtree(directory)
            app.debug("Creating dir %s", directory)
            os.mkdir(directory)
    else:
        app.debug("Creating %s", directory)
        os.mkdir(directory)


def make_loader(template_dirs):
    """Return a new :class:`jinja2.FileSystemLoader` that uses the template_dirs

    :param template_dirs: directories to search for templates
    :type template_dirs: None | :class:`list`
    :returns: a new loader
    :rtype: :class:`jinja2.FileSystemLoader`
    :raises: None
    """
    return jinja2.FileSystemLoader(searchpath=template_dirs)


def make_environment(loader):
    """Return a new :class:`jinja2.Environment` with the given loader

    :param loader: a jinja2 loader
    :type loader: :class:`jinja2.BaseLoader`
    :returns: a new environment
    :rtype: :class:`jinja2.Environment`
    :raises: None
    """
    return jinja2.Environment(loader=loader)


def makename(package, module):
    """Join package and module with a dot.

    Package or Module can be empty.

    :param package: the package name
    :type package: :class:`str`
    :param module: the module name
    :type module: :class:`str`
    :returns: the joined name
    :rtype: :class:`str`
    :raises: :class:`AssertionError`, if both package and module are empty
    """
    # Both package and module can be None/empty.
    assert package or module, "Specify either package or module"
    if package:
        name = package
        if module:
            name += '.' + module
    else:
        name = module
    return name


def write_file(app, name, text, dest, suffix, dryrun, force):
    """Write the output file for module/package <name>.

    :param app: the sphinx app
    :type app: :class:`sphinx.application.Sphinx`
    :param name: the file name without file extension
    :type name: :class:`str`
    :param text: the content of the file
    :type text: :class:`str`
    :param dest: the output directory
    :type dest: :class:`str`
    :param suffix: the file extension
    :type suffix: :class:`str`
    :param dryrun: If True, do not create any files, just log the potential location.
    :type dryrun: :class:`bool`
    :param force: Overwrite existing files
    :type force: :class:`bool`
    :returns: None
    :raises: None
    """
    fname = os.path.join(dest, '%s.%s' % (name, suffix))
    if dryrun:
        app.info('Would create file %s.' % fname)
        return
    if not force and os.path.isfile(fname):
        app.info('File %s already exists, skipping.' % fname)
    else:
        app.info('Creating file %s.' % fname)
        f = open(fname, 'w')
        try:
            f.write(text)
            relpath = os.path.relpath(fname, start=app.env.srcdir)
            abspath = os.sep + relpath
            docpath = app.env.relfn2path(abspath)[0]
            docpath = docpath.rsplit(os.path.extsep, 1)[0]
            app.debug2('Adding document %s' % docpath)
            app.env.found_docs.add(docpath)
        finally:
            f.close()


def import_name(app, name):
    """Import the given name and return name, obj, parent, mod_name

    :param name: name to import
    :type name: str
    :returns: the imported object or None
    :rtype: object | None
    :raises: None
    """
    try:
        app.debug2('Importing %r', name)
        name, obj = autosummary.import_by_name(name)[:2]
        app.debug2('Imported %s', obj)
        return obj
    except ImportError as e:
        app.warn("Jinjapidoc failed to import %r: %s", name, e)


def get_members(app, mod, typ, include_public=None):
    """Return the memebrs of mod of the given type

    :param app: the sphinx app
    :type app: :class:`sphinx.application.Sphinx`
    :param mod: the module with members
    :type mod: module
    :param typ: the typ, ``'class'``, ``'function'``, ``'exception'``, ``'data'``, ``'members'``
    :type typ: str
    :param include_public: list of private members to include to plublics
    :type include_public: list | None
    :returns: None
    :rtype: None
    :raises: None
    """
    include_public = include_public or []
    tests = {'class': lambda x: inspect.isclass(x) and not issubclass(x, BaseException),
             'function': lambda x: inspect.isfunction(x),
             'exception': lambda x: inspect.isclass(x) and issubclass(x, BaseException),
             'data': lambda x: not inspect.ismodule(x) and not inspect.isclass(x) and not inspect.isfunction(x),
             'members': lambda x: True}
    items = []
    for name in dir(mod):
        i = getattr(mod, name)
        inspect.ismodule(i)

        if tests.get(typ, lambda x: False)(i):
            items.append(name)
    public = [x for x in items
              if x in include_public or not x.startswith('_')]
    app.debug2('Got members of %s of type %s: public %s and %s', mod, typ, public, items)
    return public, items


def _get_submodules(app, module):<commit_msg>ignore classes and functions that are from another module<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""This is a modification of sphinx.apidoc by David.Zuber.
It uses jinja templates to render the rst files.

Parses a directory tree looking for Python modules and packages and creates
ReST files appropriately to create code documentation with Sphinx.

This is derived form the "sphinx-apidoc" script, which is:

  Copyright 2007-2014 by the Sphinx team, see http://sphinx-doc.org/latest/authors.html.
"""
import os
import inspect
import pkgutil
import pkg_resources
import shutil

import jinja2
from sphinx.util.osutil import walk
from sphinx.ext import autosummary


INITPY = '__init__.py'
PY_SUFFIXES = set(['.py', '.pyx'])
TEMPLATE_DIR = 'templates'
"""Built-in template dir for jinjaapi rendering"""
AUTOSUMMARYTEMPLATE_DIR = 'autosummarytemplates'
"""Templates for autosummary"""
MODULE_TEMPLATE_NAME = 'jinjaapi_module.rst'
"""Name of the template that is used for rendering modules."""
PACKAGE_TEMPLATE_NAME = 'jinjaapi_package.rst'
"""Name of the template that is used for rendering packages."""


def prepare_dir(app, directory, delete=False):
    """Create apidoc dir, delete contents if delete is True.

    :param app: the sphinx app
    :type app: :class:`sphinx.application.Sphinx`
    :param directory: the apidoc directory. you can use relative paths here
    :type directory: str
    :param delete: if True, deletes the contents of apidoc. This acts like an override switch.
    :type delete: bool
    :returns: None
    :rtype: None
    :raises: None
    """
    app.info("Preparing output directories for jinjaapidoc.")
    if os.path.exists(directory):
        if delete:
            app.debug("Deleting dir %s", directory)
            shutil.rmtree(directory)
            app.debug("Creating dir %s", directory)
            os.mkdir(directory)
    else:
        app.debug("Creating %s", directory)
        os.mkdir(directory)


def make_loader(template_dirs):
    """Return a new :class:`jinja2.FileSystemLoader` that uses the template_dirs

    :param template_dirs: directories to search for templates
    :type template_dirs: None | :class:`list`
    :returns: a new loader
    :rtype: :class:`jinja2.FileSystemLoader`
    :raises: None
    """
    return jinja2.FileSystemLoader(searchpath=template_dirs)


def make_environment(loader):
    """Return a new :class:`jinja2.Environment` with the given loader

    :param loader: a jinja2 loader
    :type loader: :class:`jinja2.BaseLoader`
    :returns: a new environment
    :rtype: :class:`jinja2.Environment`
    :raises: None
    """
    return jinja2.Environment(loader=loader)


def makename(package, module):
    """Join package and module with a dot.

    Package or Module can be empty.

    :param package: the package name
    :type package: :class:`str`
    :param module: the module name
    :type module: :class:`str`
    :returns: the joined name
    :rtype: :class:`str`
    :raises: :class:`AssertionError`, if both package and module are empty
    """
    # Both package and module can be None/empty.
    assert package or module, "Specify either package or module"
    if package:
        name = package
        if module:
            name += '.' + module
    else:
        name = module
    return name


def write_file(app, name, text, dest, suffix, dryrun, force):
    """Write the output file for module/package <name>.

    :param app: the sphinx app
    :type app: :class:`sphinx.application.Sphinx`
    :param name: the file name without file extension
    :type name: :class:`str`
    :param text: the content of the file
    :type text: :class:`str`
    :param dest: the output directory
    :type dest: :class:`str`
    :param suffix: the file extension
    :type suffix: :class:`str`
    :param dryrun: If True, do not create any files, just log the potential location.
    :type dryrun: :class:`bool`
    :param force: Overwrite existing files
    :type force: :class:`bool`
    :returns: None
    :raises: None
    """
    fname = os.path.join(dest, '%s.%s' % (name, suffix))
    if dryrun:
        app.info('Would create file %s.' % fname)
        return
    if not force and os.path.isfile(fname):
        app.info('File %s already exists, skipping.' % fname)
    else:
        app.info('Creating file %s.' % fname)
        f = open(fname, 'w')
        try:
            f.write(text)
            relpath = os.path.relpath(fname, start=app.env.srcdir)
            abspath = os.sep + relpath
            docpath = app.env.relfn2path(abspath)[0]
            docpath = docpath.rsplit(os.path.extsep, 1)[0]
            app.debug2('Adding document %s' % docpath)
            app.env.found_docs.add(docpath)
        finally:
            f.close()


def import_name(app, name):
    """Import the given name and return name, obj, parent, mod_name

    :param name: name to import
    :type name: str
    :returns: the imported object or None
    :rtype: object | None
    :raises: None
    """
    try:
        app.debug2('Importing %r', name)
        name, obj = autosummary.import_by_name(name)[:2]
        app.debug2('Imported %s', obj)
        return obj
    except ImportError as e:
        app.warn("Jinjapidoc failed to import %r: %s", name, e)


def get_members(app, mod, typ, include_public=None):
    """Return the memebrs of mod of the given type

    :param app: the sphinx app
    :type app: :class:`sphinx.application.Sphinx`
    :param mod: the module with members
    :type mod: module
    :param typ: the typ, ``'class'``, ``'function'``, ``'exception'``, ``'data'``, ``'members'``
    :type typ: str
    :param include_public: list of private members to include to plublics
    :type include_public: list | None
    :returns: None
    :rtype: None
    :raises: None
    """
    include_public = include_public or []
    tests = {'class': lambda x: inspect.isclass(x) and not issubclass(x, BaseException) and x.__module__ == mod.__name__,
             'function': lambda x: inspect.isfunction(x) and x.__module__ == mod.__name__,
             'exception': lambda x: inspect.isclass(x) and issubclass(x, BaseException) and x.__module__ == mod.__name__,
             'data': lambda x: not inspect.ismodule(x) and not inspect.isclass(x) and not inspect.isfunction(x),
             'members': lambda x: True}
    items = []
    for name in dir(mod):
        i = getattr(mod, name)
        inspect.ismodule(i)

        if tests.get(typ, lambda x: False)(i):
            items.append(name)
    public = [x for x in items
              if x in include_public or not x.startswith('_')]
    app.debug2('Got members of %s of type %s: public %s and %s', mod, typ, public, items)
    return public, items


def _get_submodules(app, module):
{'fraction_non_alphanumeric': 0.11301293071231033, 'fraction_numerical': 0.004036176096868226, 'mean_word_length': 2.995222454464019, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5293041', 'n_tokens_mistral': 4245, 'n_tokens_neox': 4116, 'n_words': 1388}