<commit_before>"""Finders try to find right section for passed module name"""
import importlib.machinery
import inspect
import os
import os.path
import re
import sys
import sysconfig
from abc import ABCMeta, abstractmethod
from fnmatch import fnmatch
from functools import lru_cache
from glob import glob
from typing import Dict, Iterable, Iterator, List, Optional, Pattern, Sequence, Tuple, Type

from . import sections
from .settings import Config
from .utils import chdir, exists_case_sensitive

try:
    from pipreqs import pipreqs

except ImportError:
    pipreqs = None

try:
    from pip_api import parse_requirements

except ImportError:
    parse_requirements = None

try:
    from requirementslib import Pipfile

except ImportError:
    Pipfile = None

KNOWN_SECTION_MAPPING: Dict[str, str] = {
    sections.STDLIB: "STANDARD_LIBRARY",
    sections.FUTURE: "FUTURE_LIBRARY",
    sections.FIRSTPARTY: "FIRST_PARTY",
    sections.THIRDPARTY: "THIRD_PARTY",
}


class BaseFinder(metaclass=ABCMeta):
    def __init__(self, config: Config) -> None:
        self.config = config

    @abstractmethod
    def find(self, module_name: str) -> Optional[str]:
        raise NotImplementedError


class ForcedSeparateFinder(BaseFinder):
    def find(self, module_name: str) -> Optional[str]:
        for forced_separate in self.config.forced_separate:
            # Ensure all forced_separate patterns will match to end of string
            path_glob = forced_separate
            if not forced_separate.endswith("*"):
                path_glob = "%s*" % forced_separate

            if fnmatch(module_name, path_glob) or fnmatch(module_name, "." + path_glob):
                return forced_separate
        return None


class LocalFinder(BaseFinder):
    def find(self, module_name: str) -> Optional[str]:
        if module_name.startswith("."):
            return "LOCALFOLDER"
        return None


class KnownPatternFinder(BaseFinder):
    def __init__(self, config: Config) -> None:
        super().__init__(config)

        self.known_patterns: List[Tuple[Pattern[str], str]] = []
        for placement in reversed(config.sections):
            known_placement = KNOWN_SECTION_MAPPING.get(placement, placement).lower()
            config_key = f"known_{known_placement}"
            known_patterns = list(
                getattr(self.config, config_key, self.config.known_other.get(known_placement, []))
            )
            known_patterns = [
                pattern
                for known_pattern in known_patterns
                for pattern in self._parse_known_pattern(known_pattern)
            ]
            for known_pattern in known_patterns:
                regexp = "^" + known_pattern.replace("*", ".*").replace("?", ".?") + "$"
                self.known_patterns.append((re.compile(regexp), placement))

    @staticmethod
    def _parse_known_pattern(pattern: str) -> List[str]:
        """Expand pattern if identified as a directory and return found sub packages"""
        if pattern.endswith(os.path.sep):
            patterns = [
                filename
                for filename in os.listdir(pattern)
                if os.path.isdir(os.path.join(pattern, filename))
            ]
        else:
            patterns = [pattern]

        return patterns

    def find(self, module_name: str) -> Optional[str]:
        # Try to find most specific placement instruction match (if any)
        parts = module_name.split(".")
        module_names_to_check = (".".join(parts[:first_k]) for first_k in range(len(parts), 0, -1))
        for module_name_to_check in module_names_to_check:
            for pattern, placement in self.known_patterns:
                if pattern.match(module_name_to_check):
                    return placement
        return None


class PathFinder(BaseFinder):
    def __init__(self, config: Config, path: str = ".") -> None:
        super().__init__(config)

        # restore the original import path (i.e. not the path to bin/isort)
        root_dir = os.path.abspath(path)
        src_dir = f"{root_dir}/src"
        self.paths = [root_dir, src_dir]

        # virtual env
        self.virtual_env = self.config.virtual_env or os.environ.get("VIRTUAL_ENV")
        if self.virtual_env:
            self.virtual_env = os.path.realpath(self.virtual_env)
        self.virtual_env_src = ""
        if self.virtual_env:
            self.virtual_env_src = f"{self.virtual_env}/src/"
            for venv_path in glob(f"{self.virtual_env}/lib/python*/site-packages"):
                if venv_path not in self.paths:
                    self.paths.append(venv_path)
            for nested_venv_path in glob(f"{self.virtual_env}/lib/python*/*/site-packages"):
                if nested_venv_path not in self.paths:
                    self.paths.append(nested_venv_path)
            for venv_src_path in glob(f"{self.virtual_env}/src/*"):
                if os.path.isdir(venv_src_path):
                    self.paths.append(venv_src_path)

        # conda
        self.conda_env = self.config.conda_env or os.environ.get("CONDA_PREFIX") or ""
        if self.conda_env:
            self.conda_env = os.path.realpath(self.conda_env)
            for conda_path in glob(f"{self.conda_env}/lib/python*/site-packages"):
                if conda_path not in self.paths:
                    self.paths.append(conda_path)
            for nested_conda_path in glob(f"{self.conda_env}/lib/python*/*/site-packages"):
                if nested_conda_path not in self.paths:
                    self.paths.append(nested_conda_path)

        # handle case-insensitive paths on windows
        self.stdlib_lib_prefix = os.path.normcase(sysconfig.get_paths()["stdlib"])
        if self.stdlib_lib_prefix not in self.paths:
            self.paths.append(self.stdlib_lib_prefix)

        # add system paths
        for system_path in sys.path[1:]:
            if system_path not in self.paths:
                self.paths.append(system_path)

    def find(self, module_name: str) -> Optional[str]:
        for prefix in self.paths:
            package_path = "/".join((prefix, module_name.split(".")[0]))
            is_module = (
                exists_case_sensitive(package_path + ".py")
                or any(
                    exists_case_sensitive(package_path + ext_suffix)
                    for ext_suffix in importlib.machinery.EXTENSION_SUFFIXES
                )
                or exists_case_sensitive(package_path + "/__init__.py")
            )
            is_package = exists_case_sensitive(package_path) and os.path.isdir(package_path)
            if is_module or is_package:
                if (
                    "site-packages" in prefix
                    or "dist-packages" in prefix
                    or (self.virtual_env and self.virtual_env_src in prefix)
                ):
                    return sections.THIRDPARTY
                elif os.path.normcase(prefix) == self.stdlib_lib_prefix:
                    return sections.STDLIB
                elif self.conda_env and self.conda_env in prefix:
                    return sections.THIRDPARTY
                if os.getcwd() in package_path:
                    return sections.FIRSTPARTY
                elif os.path.normcase(prefix).startswith(self.stdlib_lib_prefix):
                    return sections.STDLIB  # pragma: no cover - edge case for one OS. Hard to test.
                return self.config.default_section
        return None


class ReqsBaseFinder(BaseFinder):<commit_msg>Add support for src_paths inside of finders<commit_after>"""Finders try to find right section for passed module name"""
import importlib.machinery
import inspect
import os
import os.path
import re
import sys
import sysconfig
from abc import ABCMeta, abstractmethod
from fnmatch import fnmatch
from functools import lru_cache
from glob import glob
from typing import Dict, Iterable, Iterator, List, Optional, Pattern, Sequence, Tuple, Type

from . import sections
from .settings import Config
from .utils import chdir, exists_case_sensitive

try:
    from pipreqs import pipreqs

except ImportError:
    pipreqs = None

try:
    from pip_api import parse_requirements

except ImportError:
    parse_requirements = None

try:
    from requirementslib import Pipfile

except ImportError:
    Pipfile = None

KNOWN_SECTION_MAPPING: Dict[str, str] = {
    sections.STDLIB: "STANDARD_LIBRARY",
    sections.FUTURE: "FUTURE_LIBRARY",
    sections.FIRSTPARTY: "FIRST_PARTY",
    sections.THIRDPARTY: "THIRD_PARTY",
}


class BaseFinder(metaclass=ABCMeta):
    def __init__(self, config: Config) -> None:
        self.config = config

    @abstractmethod
    def find(self, module_name: str) -> Optional[str]:
        raise NotImplementedError


class ForcedSeparateFinder(BaseFinder):
    def find(self, module_name: str) -> Optional[str]:
        for forced_separate in self.config.forced_separate:
            # Ensure all forced_separate patterns will match to end of string
            path_glob = forced_separate
            if not forced_separate.endswith("*"):
                path_glob = "%s*" % forced_separate

            if fnmatch(module_name, path_glob) or fnmatch(module_name, "." + path_glob):
                return forced_separate
        return None


class LocalFinder(BaseFinder):
    def find(self, module_name: str) -> Optional[str]:
        if module_name.startswith("."):
            return "LOCALFOLDER"
        return None


class KnownPatternFinder(BaseFinder):
    def __init__(self, config: Config) -> None:
        super().__init__(config)

        self.known_patterns: List[Tuple[Pattern[str], str]] = []
        for placement in reversed(config.sections):
            known_placement = KNOWN_SECTION_MAPPING.get(placement, placement).lower()
            config_key = f"known_{known_placement}"
            known_patterns = list(
                getattr(self.config, config_key, self.config.known_other.get(known_placement, []))
            )
            known_patterns = [
                pattern
                for known_pattern in known_patterns
                for pattern in self._parse_known_pattern(known_pattern)
            ]
            for known_pattern in known_patterns:
                regexp = "^" + known_pattern.replace("*", ".*").replace("?", ".?") + "$"
                self.known_patterns.append((re.compile(regexp), placement))

    @staticmethod
    def _parse_known_pattern(pattern: str) -> List[str]:
        """Expand pattern if identified as a directory and return found sub packages"""
        if pattern.endswith(os.path.sep):
            patterns = [
                filename
                for filename in os.listdir(pattern)
                if os.path.isdir(os.path.join(pattern, filename))
            ]
        else:
            patterns = [pattern]

        return patterns

    def find(self, module_name: str) -> Optional[str]:
        # Try to find most specific placement instruction match (if any)
        parts = module_name.split(".")
        module_names_to_check = (".".join(parts[:first_k]) for first_k in range(len(parts), 0, -1))
        for module_name_to_check in module_names_to_check:
            for pattern, placement in self.known_patterns:
                if pattern.match(module_name_to_check):
                    return placement
        return None


class PathFinder(BaseFinder):
    def __init__(self, config: Config, path: str = ".") -> None:
        super().__init__(config)

        # restore the original import path (i.e. not the path to bin/isort)
        root_dir = os.path.abspath(path)
        src_dir = f"{root_dir}/src"
        self.paths = [root_dir, src_dir]

        # virtual env
        self.virtual_env = self.config.virtual_env or os.environ.get("VIRTUAL_ENV")
        if self.virtual_env:
            self.virtual_env = os.path.realpath(self.virtual_env)
        self.virtual_env_src = ""
        if self.virtual_env:
            self.virtual_env_src = f"{self.virtual_env}/src/"
            for venv_path in glob(f"{self.virtual_env}/lib/python*/site-packages"):
                if venv_path not in self.paths:
                    self.paths.append(venv_path)
            for nested_venv_path in glob(f"{self.virtual_env}/lib/python*/*/site-packages"):
                if nested_venv_path not in self.paths:
                    self.paths.append(nested_venv_path)
            for venv_src_path in glob(f"{self.virtual_env}/src/*"):
                if os.path.isdir(venv_src_path):
                    self.paths.append(venv_src_path)

        # conda
        self.conda_env = self.config.conda_env or os.environ.get("CONDA_PREFIX") or ""
        if self.conda_env:
            self.conda_env = os.path.realpath(self.conda_env)
            for conda_path in glob(f"{self.conda_env}/lib/python*/site-packages"):
                if conda_path not in self.paths:
                    self.paths.append(conda_path)
            for nested_conda_path in glob(f"{self.conda_env}/lib/python*/*/site-packages"):
                if nested_conda_path not in self.paths:
                    self.paths.append(nested_conda_path)

        # handle case-insensitive paths on windows
        self.stdlib_lib_prefix = os.path.normcase(sysconfig.get_paths()["stdlib"])
        if self.stdlib_lib_prefix not in self.paths:
            self.paths.append(self.stdlib_lib_prefix)

        # add system paths
        for system_path in sys.path[1:]:
            if system_path not in self.paths:
                self.paths.append(system_path)

    def find(self, module_name: str) -> Optional[str]:
        for prefix in self.paths:
            package_path = "/".join((prefix, module_name.split(".")[0]))
            is_module = (
                exists_case_sensitive(package_path + ".py")
                or any(
                    exists_case_sensitive(package_path + ext_suffix)
                    for ext_suffix in importlib.machinery.EXTENSION_SUFFIXES
                )
                or exists_case_sensitive(package_path + "/__init__.py")
            )
            is_package = exists_case_sensitive(package_path) and os.path.isdir(package_path)
            if is_module or is_package:
                if (
                    "site-packages" in prefix
                    or "dist-packages" in prefix
                    or (self.virtual_env and self.virtual_env_src in prefix)
                ):
                    return sections.THIRDPARTY
                elif os.path.normcase(prefix) == self.stdlib_lib_prefix:
                    return sections.STDLIB
                elif self.conda_env and self.conda_env in prefix:
                    return sections.THIRDPARTY

                for src_path in self.config.src_paths:
                    if str(src_path.absolute) in os.path.abspath(package_path):
                        return sections.FIRSTPARTY

                if os.path.normcase(prefix).startswith(self.stdlib_lib_prefix):
                    return sections.STDLIB  # pragma: no cover - edge case for one OS. Hard to test.

                return self.config.default_section
        return None


class ReqsBaseFinder(BaseFinder):
{'fraction_non_alphanumeric': 0.09092707371752605, 'fraction_numerical': 0.0005274957140973229, 'mean_word_length': 2.2079103214890017, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3550723', 'n_tokens_mistral': 4469, 'n_tokens_neox': 4314, 'n_words': 1019}