<commit_before>"""Routines related to PyPI, indexes"""

import sys
import os
import re
import mimetypes
import posixpath
import pkg_resources

from pip.log import logger
from pip.util import Inf, normalize_name, splitext, is_prerelease
from pip.exceptions import DistributionNotFound, BestVersionAlreadyInstalled,\
    InstallationError
from pip.backwardcompat import urlparse, url2pathname
from pip.download import PipSession, path_to_url2, url_to_path
from pip.wheel import Wheel, wheel_ext, wheel_setuptools_support
from pip.pep425tags import supported_tags, supported_tags_noarch, get_platform
from pip._vendor import html5lib, requests
from pip._vendor.requests.exceptions import SSLError


__all__ = ['PackageFinder']


DEFAULT_MIRROR_HOSTNAME = "last.pypi.python.org"

INSECURE_SCHEMES = {
    "http": ["https"],
}


class PackageFinder(object):
    """This finds packages.

    This is meant to match easy_install's technique for looking for
    packages, by reading pages and looking for appropriate links
    """

    def __init__(self, find_links, index_urls,
            use_wheel=False, allow_external=[], allow_unverified=[],
            allow_all_external=False, allow_all_prereleases=False,
            process_dependency_links=False, session=None):
        self.find_links = find_links
        self.index_urls = index_urls
        self.dependency_links = []
        self.cache = PageCache()
        # These are boring links that have already been logged somehow:
        self.logged_links = set()

        self.use_wheel = use_wheel

        # Do we allow (safe and verifiable) externally hosted files?
        self.allow_external = set(normalize_name(n) for n in allow_external)

        # Which names are allowed to install insecure and unverifiable files?
        self.allow_unverified = set(
            normalize_name(n) for n in allow_unverified
        )

        # Do we allow all (safe and verifiable) externally hosted files?
        self.allow_all_external = allow_all_external

        # Stores if we ignored any external links so that we can instruct
        #   end users how to install them if no distributions are available
        self.need_warn_external = False

        # Stores if we ignored any unsafe links so that we can instruct
        #   end users how to install them if no distributions are available
        self.need_warn_unverified = False

        # Do we want to allow _all_ pre-releases?
        self.allow_all_prereleases = allow_all_prereleases

        # Do we process dependency links?
        self.process_dependency_links = process_dependency_links

        # The Session we'll use to make requests
        self.session = session or PipSession()

    @property
    def use_wheel(self):
        return self._use_wheel

    @use_wheel.setter
    def use_wheel(self, value):
        self._use_wheel = value
        if self._use_wheel and not wheel_setuptools_support():
            raise InstallationError("pip's wheel support requires setuptools >= 0.8 for dist-info support.")

    def add_dependency_links(self, links):
        ## FIXME: this shouldn't be global list this, it should only
        ## apply to requirements of the package that specifies the
        ## dependency_links value
        ## FIXME: also, we should track comes_from (i.e., use Link)
        if self.process_dependency_links:
            self.dependency_links.extend(links)

    def _sort_locations(self, locations):
        """
        Sort locations into "files" (archives) and "urls", and return
        a pair of lists (files,urls)
        """
        files = []
        urls = []

        # puts the url for the given file path into the appropriate list
        def sort_path(path):
            url = path_to_url2(path)
            if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
                urls.append(url)
            else:
                files.append(url)

        for url in locations:<commit_msg>Add a deprecation warning for dependency links<commit_after>"""Routines related to PyPI, indexes"""

import sys
import os
import re
import mimetypes
import posixpath
import pkg_resources

from pip.log import logger
from pip.util import Inf, normalize_name, splitext, is_prerelease
from pip.exceptions import DistributionNotFound, BestVersionAlreadyInstalled,\
    InstallationError
from pip.backwardcompat import urlparse, url2pathname
from pip.download import PipSession, path_to_url2, url_to_path
from pip.wheel import Wheel, wheel_ext, wheel_setuptools_support
from pip.pep425tags import supported_tags, supported_tags_noarch, get_platform
from pip._vendor import html5lib, requests
from pip._vendor.requests.exceptions import SSLError


__all__ = ['PackageFinder']


DEFAULT_MIRROR_HOSTNAME = "last.pypi.python.org"

INSECURE_SCHEMES = {
    "http": ["https"],
}


class PackageFinder(object):
    """This finds packages.

    This is meant to match easy_install's technique for looking for
    packages, by reading pages and looking for appropriate links
    """

    def __init__(self, find_links, index_urls,
            use_wheel=False, allow_external=[], allow_unverified=[],
            allow_all_external=False, allow_all_prereleases=False,
            process_dependency_links=False, session=None):
        self.find_links = find_links
        self.index_urls = index_urls
        self.dependency_links = []
        self.cache = PageCache()
        # These are boring links that have already been logged somehow:
        self.logged_links = set()

        self.use_wheel = use_wheel

        # Do we allow (safe and verifiable) externally hosted files?
        self.allow_external = set(normalize_name(n) for n in allow_external)

        # Which names are allowed to install insecure and unverifiable files?
        self.allow_unverified = set(
            normalize_name(n) for n in allow_unverified
        )

        # Do we allow all (safe and verifiable) externally hosted files?
        self.allow_all_external = allow_all_external

        # Stores if we ignored any external links so that we can instruct
        #   end users how to install them if no distributions are available
        self.need_warn_external = False

        # Stores if we ignored any unsafe links so that we can instruct
        #   end users how to install them if no distributions are available
        self.need_warn_unverified = False

        # Do we want to allow _all_ pre-releases?
        self.allow_all_prereleases = allow_all_prereleases

        # Do we process dependency links?
        self.process_dependency_links = process_dependency_links
        self._have_warned_dependency_links = False

        # The Session we'll use to make requests
        self.session = session or PipSession()

    @property
    def use_wheel(self):
        return self._use_wheel

    @use_wheel.setter
    def use_wheel(self, value):
        self._use_wheel = value
        if self._use_wheel and not wheel_setuptools_support():
            raise InstallationError("pip's wheel support requires setuptools >= 0.8 for dist-info support.")

    def add_dependency_links(self, links):
        ## FIXME: this shouldn't be global list this, it should only
        ## apply to requirements of the package that specifies the
        ## dependency_links value
        ## FIXME: also, we should track comes_from (i.e., use Link)
        if self.process_dependency_links:
            if not self._have_warned_dependency_links:
                logger.deprecated(
                    "1.6",
                    "Dependency Links processing has been deprecated with an "
                    "accelerated time schedule and will be removed in pip 1.6",
                )
                self._have_warned_dependency_links = True
            self.dependency_links.extend(links)

    def _sort_locations(self, locations):
        """
        Sort locations into "files" (archives) and "urls", and return
        a pair of lists (files,urls)
        """
        files = []
        urls = []

        # puts the url for the given file path into the appropriate list
        def sort_path(path):
            url = path_to_url2(path)
            if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
                urls.append(url)
            else:
                files.append(url)

        for url in locations:
{'fraction_non_alphanumeric': 0.06646706586826347, 'fraction_numerical': 0.002874251497005988, 'mean_word_length': 2.7821557971014492, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3653976', 'n_tokens_mistral': 2347, 'n_tokens_neox': 2229, 'n_words': 748}