<commit_before># Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import itertools
import logging
from typing import cast

from pants.backend.python.dependency_inference import import_parser, module_mapper
from pants.backend.python.dependency_inference.import_parser import (
    ParsedPythonImports,
    ParsePythonImportsRequest,
)
from pants.backend.python.dependency_inference.module_mapper import PythonModule, PythonModuleOwners
from pants.backend.python.target_types import PythonSources, PythonTestsSources
from pants.backend.python.util_rules import ancestor_files, pex
from pants.backend.python.util_rules.ancestor_files import AncestorFiles, AncestorFilesRequest
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.core.util_rules import stripped_source_files
from pants.engine.addresses import Address
from pants.engine.internals.graph import Owners, OwnersRequest
from pants.engine.rules import Get, MultiGet, SubsystemRule, rule
from pants.engine.target import (
    Dependencies,
    DependenciesRequest,
    ExplicitlyProvidedDependencies,
    HydratedSources,
    HydrateSourcesRequest,
    InferDependenciesRequest,
    InferredDependencies,
    WrappedTarget,
)
from pants.engine.unions import UnionRule
from pants.option.global_options import OwnersNotFoundBehavior
from pants.option.subsystem import Subsystem
from pants.python.python_setup import PythonSetup

logger = logging.getLogger(__name__)


class PythonInferSubsystem(Subsystem):
    options_scope = "python-infer"
    help = "Options controlling which dependencies will be inferred for Python targets."

    @classmethod
    def register_options(cls, register):
        super().register_options(register)
        register(
            "--imports",
            default=True,
            type=bool,
            help=(
                "Infer a target's imported dependencies by parsing import statements from sources."
            ),
        )
        register(
            "--string-imports",
            default=False,
            type=bool,
            help=(
                "Infer a target's dependencies based on strings that look like dynamic "
                "dependencies, such as Django settings files expressing dependencies as strings. "
                "To ignore any false positives, put `!{bad_address}` in the `dependencies` field "
                "of your target."
            ),
        )
        register(
            "--string-imports-min-dots",
            default=2,
            type=int,
            help=(
                "If --string-imports is True, treat valid-looking strings with at least this many "
                "dots in them as potential dynamic dependencies. E.g., `'foo.bar.Baz'` will be "
                "treated as a potential dependency if this option is set to 2 but not if set to 3."
            ),
        )
        register(
            "--inits",
            default=False,
            type=bool,
            help=(
                "Infer a target's dependencies on any `__init__.py` files in the packages "
                "it is located in (recursively upward in the directory structure).\n\nEven if this "
                "is disabled, Pants will still include any ancestor `__init__.py` files, only they "
                "will not be 'proper' dependencies, e.g. they will not show up in "
                "`./pants dependencies` and their own dependencies will not be used.\n\nIf you "
                "have empty `__init__.py` files, it's safe to leave this option off; otherwise, "
                "you should enable this option."
            ),
        )
        register(
            "--conftests",
            default=True,
            type=bool,
            help=(
                "Infer a test target's dependencies on any conftest.py files in the current "
                "directory and ancestor directories."
            ),
        )
        register(
            "--entry-points",
            default=True,
            type=bool,
            help=(
                "Infer dependencies on targets' entry points, e.g. `pex_binary`'s "
                "`entry_point` field, `python_awslambda`'s `handler` field and "
                "`python_distribution`'s `entry_points` field."
            ),
        )

    @property
    def imports(self) -> bool:
        return cast(bool, self.options.imports)

    @property
    def string_imports(self) -> bool:
        return cast(bool, self.options.string_imports)

    @property
    def string_imports_min_dots(self) -> int:
        return cast(bool, self.options.string_imports_min_dots)

    @property
    def inits(self) -> bool:
        return cast(bool, self.options.inits)
<commit_msg>[Internal] Fix type typo. (#13119)<commit_after># Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import itertools
import logging
from typing import cast

from pants.backend.python.dependency_inference import import_parser, module_mapper
from pants.backend.python.dependency_inference.import_parser import (
    ParsedPythonImports,
    ParsePythonImportsRequest,
)
from pants.backend.python.dependency_inference.module_mapper import PythonModule, PythonModuleOwners
from pants.backend.python.target_types import PythonSources, PythonTestsSources
from pants.backend.python.util_rules import ancestor_files, pex
from pants.backend.python.util_rules.ancestor_files import AncestorFiles, AncestorFilesRequest
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.core.util_rules import stripped_source_files
from pants.engine.addresses import Address
from pants.engine.internals.graph import Owners, OwnersRequest
from pants.engine.rules import Get, MultiGet, SubsystemRule, rule
from pants.engine.target import (
    Dependencies,
    DependenciesRequest,
    ExplicitlyProvidedDependencies,
    HydratedSources,
    HydrateSourcesRequest,
    InferDependenciesRequest,
    InferredDependencies,
    WrappedTarget,
)
from pants.engine.unions import UnionRule
from pants.option.global_options import OwnersNotFoundBehavior
from pants.option.subsystem import Subsystem
from pants.python.python_setup import PythonSetup

logger = logging.getLogger(__name__)


class PythonInferSubsystem(Subsystem):
    options_scope = "python-infer"
    help = "Options controlling which dependencies will be inferred for Python targets."

    @classmethod
    def register_options(cls, register):
        super().register_options(register)
        register(
            "--imports",
            default=True,
            type=bool,
            help=(
                "Infer a target's imported dependencies by parsing import statements from sources."
            ),
        )
        register(
            "--string-imports",
            default=False,
            type=bool,
            help=(
                "Infer a target's dependencies based on strings that look like dynamic "
                "dependencies, such as Django settings files expressing dependencies as strings. "
                "To ignore any false positives, put `!{bad_address}` in the `dependencies` field "
                "of your target."
            ),
        )
        register(
            "--string-imports-min-dots",
            default=2,
            type=int,
            help=(
                "If --string-imports is True, treat valid-looking strings with at least this many "
                "dots in them as potential dynamic dependencies. E.g., `'foo.bar.Baz'` will be "
                "treated as a potential dependency if this option is set to 2 but not if set to 3."
            ),
        )
        register(
            "--inits",
            default=False,
            type=bool,
            help=(
                "Infer a target's dependencies on any `__init__.py` files in the packages "
                "it is located in (recursively upward in the directory structure).\n\nEven if this "
                "is disabled, Pants will still include any ancestor `__init__.py` files, only they "
                "will not be 'proper' dependencies, e.g. they will not show up in "
                "`./pants dependencies` and their own dependencies will not be used.\n\nIf you "
                "have empty `__init__.py` files, it's safe to leave this option off; otherwise, "
                "you should enable this option."
            ),
        )
        register(
            "--conftests",
            default=True,
            type=bool,
            help=(
                "Infer a test target's dependencies on any conftest.py files in the current "
                "directory and ancestor directories."
            ),
        )
        register(
            "--entry-points",
            default=True,
            type=bool,
            help=(
                "Infer dependencies on targets' entry points, e.g. `pex_binary`'s "
                "`entry_point` field, `python_awslambda`'s `handler` field and "
                "`python_distribution`'s `entry_points` field."
            ),
        )

    @property
    def imports(self) -> bool:
        return cast(bool, self.options.imports)

    @property
    def string_imports(self) -> bool:
        return cast(bool, self.options.string_imports)

    @property
    def string_imports_min_dots(self) -> int:
        return cast(int, self.options.string_imports_min_dots)

    @property
    def inits(self) -> bool:
        return cast(bool, self.options.inits)

{'fraction_non_alphanumeric': 0.07808930425752855, 'fraction_numerical': 0.0023883696780893043, 'mean_word_length': 2.323326432022084, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3434569', 'n_tokens_mistral': 2552, 'n_tokens_neox': 2405, 'n_words': 766}