<commit_before>"""Proxxy module. Defines a Proxxy element that fetches files using local
   proxxy instances (if available). The goal of Proxxy is to lower the traffic
   from the cloud to internal servers.
"""
import urlparse
import socket
from mozharness.base.log import INFO, ERROR, LogMixin
from mozharness.base.script import ScriptMixin


# Proxxy {{{1
class Proxxy(ScriptMixin, LogMixin):
    """
    Support downloading files from HTTP caching proxies

    Current supports 'proxxy' instances, in which the caching proxy at
    proxxy.domain.com will cache requests for ftp.mozilla.org when passed requests to
    http://ftp.mozilla.org.proxxy.domain.com/...

    self.config['proxxy']['urls'] defines the list of backend hosts we are currently caching, and
    the hostname prefix to use for proxxy

    self.config['proxxy']['instances'] lists current hostnames for proxxy instances. wildcard DNS
    is set up so that *.proxxy.domain.com is a CNAME to the proxxy instance
    """

    # Default configuration. Can be overridden via self.config
    PROXXY_CONFIG = {
        "urls": [
            ('http://ftp.mozilla.org', 'ftp.mozilla.org'),
            ('https://ftp.mozilla.org', 'ftp.mozilla.org'),
            ('https://ftp-ssl.mozilla.org', 'ftp.mozilla.org'),
            ('http://pvtbuilds.pvt.build.mozilla.org', 'pvtbuilds.mozilla.org'),
            # tooltool
            ('http://tooltool.pvt.build.mozilla.org', 'tooltool.pvt.build.mozilla.org'),
            # pypi
            ('http://pypi.pvt.build.mozilla.org', 'pypi.pvt.build.mozilla.org'),
            ('http://pypi.pub.build.mozilla.org', 'pypi.pub.build.mozilla.org'),
        ],
        "instances": [
            'proxxy1.srv.releng.use1.mozilla.com',
            'proxxy1.srv.releng.usw2.mozilla.com',
        ],
        "regions": [".use1.", ".usw2."],
    }

    def __init__(self, config, log_obj):
        # proxxy does not need the need the full configuration,
        # just the 'proxxy' element
        # if configuration has no 'proxxy' section use the default
        # configuration instead
        self.config = config.get('proxxy', self.PROXXY_CONFIG)
        self.log_obj = log_obj

    def get_proxies_for_url(self, url):
        """Maps url to its proxxy urls<commit_msg>Bug 1150196: Use proxxy instances in scl3 r=rail<commit_after>"""Proxxy module. Defines a Proxxy element that fetches files using local
   proxxy instances (if available). The goal of Proxxy is to lower the traffic
   from the cloud to internal servers.
"""
import urlparse
import socket
from mozharness.base.log import INFO, ERROR, LogMixin
from mozharness.base.script import ScriptMixin


# Proxxy {{{1
class Proxxy(ScriptMixin, LogMixin):
    """
    Support downloading files from HTTP caching proxies

    Current supports 'proxxy' instances, in which the caching proxy at
    proxxy.domain.com will cache requests for ftp.mozilla.org when passed requests to
    http://ftp.mozilla.org.proxxy.domain.com/...

    self.config['proxxy']['urls'] defines the list of backend hosts we are currently caching, and
    the hostname prefix to use for proxxy

    self.config['proxxy']['instances'] lists current hostnames for proxxy instances. wildcard DNS
    is set up so that *.proxxy.domain.com is a CNAME to the proxxy instance
    """

    # Default configuration. Can be overridden via self.config
    PROXXY_CONFIG = {
        "urls": [
            ('http://ftp.mozilla.org', 'ftp.mozilla.org'),
            ('https://ftp.mozilla.org', 'ftp.mozilla.org'),
            ('https://ftp-ssl.mozilla.org', 'ftp.mozilla.org'),
            ('http://pvtbuilds.pvt.build.mozilla.org', 'pvtbuilds.mozilla.org'),
            # tooltool
            ('http://tooltool.pvt.build.mozilla.org', 'tooltool.pvt.build.mozilla.org'),
            # pypi
            ('http://pypi.pvt.build.mozilla.org', 'pypi.pvt.build.mozilla.org'),
            ('http://pypi.pub.build.mozilla.org', 'pypi.pub.build.mozilla.org'),
            # taskcluster stuff
            ('https://queue.taskcluster.net', 'queue.taskcluster.net'),
        ],
        "instances": [
            'proxxy1.srv.releng.use1.mozilla.com',
            'proxxy1.srv.releng.usw2.mozilla.com',
            'proxxy1.srv.releng.scl3.mozilla.com',
        ],
        "regions": [".use1.", ".usw2.", ".scl3"],
    }

    def __init__(self, config, log_obj):
        # proxxy does not need the need the full configuration,
        # just the 'proxxy' element
        # if configuration has no 'proxxy' section use the default
        # configuration instead
        self.config = config.get('proxxy', self.PROXXY_CONFIG)
        self.log_obj = log_obj

    def get_proxies_for_url(self, url):
        """Maps url to its proxxy urls
{'fraction_non_alphanumeric': 0.12444631934191099, 'fraction_numerical': 0.005273149124657245, 'mean_word_length': 2.656129529683886, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5259038', 'n_tokens_mistral': 1553, 'n_tokens_neox': 1449, 'n_words': 394}