<commit_before>"""Load setuptools configuration from ``pyproject.toml`` files"""
import os
import sys
from contextlib import contextmanager
from functools import partial
from typing import Union
import json

from setuptools.errors import OptionError, FileError
from distutils import log

from . import expand as _expand

_Path = Union[str, os.PathLike]


def load_file(filepath: _Path):
    try:
        from setuptools.extern import tomli
    except ImportError:  # Bootstrap problem (?) diagnosed by test_distutils_adoption
        sys_path = sys.path.copy()
        try:
            from setuptools import _vendor
            sys.path.append(_vendor.__path__[0])
            import tomli
        finally:
            sys.path = sys_path

    with open(filepath, "rb") as file:
        return tomli.load(file)


def validate(config: dict, filepath: _Path):
    from setuptools.extern import _validate_pyproject
    from setuptools.extern._validate_pyproject import fastjsonschema_exceptions

    try:
        return _validate_pyproject.validate(config)
    except fastjsonschema_exceptions.JsonSchemaValueException as ex:
        msg = [f"Schema: {ex}"]
        if ex.value:
            msg.append(f"Given value:\n{json.dumps(ex.value, indent=2)}")
        if ex.rule:
            msg.append(f"Offending rule: {json.dumps(ex.rule, indent=2)}")
        if ex.definition:
            msg.append(f"Definition:\n{json.dumps(ex.definition, indent=2)}")

        log.error("\n\n".join(msg) + "\n")
        raise


def read_configuration(filepath, expand=True, ignore_option_errors=False):
    """Read given configuration file and returns options from it as a dict.

    :param str|unicode filepath: Path to configuration file in the ``pyproject.toml``
        format.

    :param bool expand: Whether to expand directives and other computed values
        (i.e. post-process the given configuration)

    :param bool ignore_option_errors: Whether to silently ignore
        options, values of which could not be resolved (e.g. due to exceptions
        in directives such as file:, attr:, etc.).
        If False exceptions are propagated as expected.

    :rtype: dict
    """
    filepath = os.path.abspath(filepath)

    if not os.path.isfile(filepath):
        raise FileError(f"Configuration file {filepath!r} does not exist.")

    asdict = load_file(filepath) or {}
    project_table = asdict.get("project")
    tool_table = asdict.get("tool", {}).get("setuptools")
    if not asdict or not(project_table or tool_table):
        return {}  # User is not using pyproject to configure setuptools

    with _ignore_errors(ignore_option_errors):
        validate(asdict, filepath)

    if expand:
        root_dir = os.path.dirname(filepath)
        return expand_configuration(asdict, root_dir, ignore_option_errors)

    return asdict


def expand_configuration(config, root_dir=None, ignore_option_errors=False):
    """Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...)
    find their final values.

    :param dict config: Dict containing the configuration for the distribution
    :param str root_dir: Top-level directory for the distribution/project
        (the same directory where ``pyproject.toml`` is place)
    :param bool ignore_option_errors: see :func:`read_configuration`

    :rtype: dict
    """
    root_dir = root_dir or os.getcwd()
    project_cfg = config.get("project", {})<commit_msg>Make include_package_data=True for `pyproject.toml` configs<commit_after>"""Load setuptools configuration from ``pyproject.toml`` files"""
import os
import sys
from contextlib import contextmanager
from functools import partial
from typing import Union
import json

from setuptools.errors import OptionError, FileError
from distutils import log

from . import expand as _expand

_Path = Union[str, os.PathLike]


def load_file(filepath: _Path):
    try:
        from setuptools.extern import tomli
    except ImportError:  # Bootstrap problem (?) diagnosed by test_distutils_adoption
        sys_path = sys.path.copy()
        try:
            from setuptools import _vendor
            sys.path.append(_vendor.__path__[0])
            import tomli
        finally:
            sys.path = sys_path

    with open(filepath, "rb") as file:
        return tomli.load(file)


def validate(config: dict, filepath: _Path):
    from setuptools.extern import _validate_pyproject
    from setuptools.extern._validate_pyproject import fastjsonschema_exceptions

    try:
        return _validate_pyproject.validate(config)
    except fastjsonschema_exceptions.JsonSchemaValueException as ex:
        msg = [f"Schema: {ex}"]
        if ex.value:
            msg.append(f"Given value:\n{json.dumps(ex.value, indent=2)}")
        if ex.rule:
            msg.append(f"Offending rule: {json.dumps(ex.rule, indent=2)}")
        if ex.definition:
            msg.append(f"Definition:\n{json.dumps(ex.definition, indent=2)}")

        log.error("\n\n".join(msg) + "\n")
        raise


def read_configuration(filepath, expand=True, ignore_option_errors=False):
    """Read given configuration file and returns options from it as a dict.

    :param str|unicode filepath: Path to configuration file in the ``pyproject.toml``
        format.

    :param bool expand: Whether to expand directives and other computed values
        (i.e. post-process the given configuration)

    :param bool ignore_option_errors: Whether to silently ignore
        options, values of which could not be resolved (e.g. due to exceptions
        in directives such as file:, attr:, etc.).
        If False exceptions are propagated as expected.

    :rtype: dict
    """
    filepath = os.path.abspath(filepath)

    if not os.path.isfile(filepath):
        raise FileError(f"Configuration file {filepath!r} does not exist.")

    asdict = load_file(filepath) or {}
    project_table = asdict.get("project")
    tool_table = asdict.get("tool", {}).get("setuptools")
    if not asdict or not(project_table or tool_table):
        return {}  # User is not using pyproject to configure setuptools

    # There is an overall sense in the community that making include_package_data=True
    # the default would be an improvement.
    # `ini2toml` backfills include_package_data=False when nothing is explicitly given,
    # therefore setting a default here is backwards compatible.
    tool_table.setdefault("include-package-data", True)

    with _ignore_errors(ignore_option_errors):
        validate(asdict, filepath)

    if expand:
        root_dir = os.path.dirname(filepath)
        return expand_configuration(asdict, root_dir, ignore_option_errors)

    return asdict


def expand_configuration(config, root_dir=None, ignore_option_errors=False):
    """Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...)
    find their final values.

    :param dict config: Dict containing the configuration for the distribution
    :param str root_dir: Top-level directory for the distribution/project
        (the same directory where ``pyproject.toml`` is place)
    :param bool ignore_option_errors: see :func:`read_configuration`

    :rtype: dict
    """
    root_dir = root_dir or os.getcwd()
    project_cfg = config.get("project", {})
{'fraction_non_alphanumeric': 0.08952959028831563, 'fraction_numerical': 0.0012415505586977514, 'mean_word_length': 3.2975696502667455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228548', 'n_tokens_mistral': 2097, 'n_tokens_neox': 2042, 'n_words': 639}