<commit_before>import datetime
import logging
import time
from kombu import Exchange, Queue
from kombu.mixins import ConsumerMixin
import os
import pystache
import re
from taskcluster import slugId, stringDate, fromNow
import yaml
import json

from funsize.utils import properties_to_dict, encrypt_env_var

log = logging.getLogger(__name__)

# TODO: move these to config
BRANCHES = ['mozilla-central', 'mozilla-aurora']
PLATFORMS = ['linux', 'linux64', 'win32', 'win64', 'macosx64']
BUILDERS = [
    r'WINNT \d+\.\d+ (x86-64 )?{branch} nightly',
    r'Linux (x86-64 )?{branch} nightly',
    r'OS X \d+\.\d+ {branch} nightly',
    r'Firefox {branch} (linux|linux64|win32|win64|macosx64) l10n nightly-\d+',
]


class FunsizeWorker(ConsumerMixin):

    def __init__(self, connection, queue_name, exchange, balrog_client,
                 scheduler):
        """Funsize consumer worker
        :type connection: kombu.Connection
        :param queue_name: Full queue name, including queue/<user> prefix
        :type exchange: kombu.Exchange
        :type balrog_client: funsize.balrog.BalrogClient
        :type scheduler: taskcluster.Scheduler
        """
        self.connection = connection
        # Using passive mode is important, otherwise pulse returns 403
        self.exchange = Exchange(exchange, type='topic', passive=True)
        self.queue_name = queue_name
        self.balrog_client = balrog_client
        self.scheduler = scheduler

    @property
    def routing_keys(self):
        """Returns an explicit list of routing patterns.

        Instead of using "build.*.*.finished", which generates a lot of noise,
        FunsizeWorker uses explicit list of routing keys to match builders we
        are interested in.
        """
        jobs = [
            # TODO: move to configs
            'build.{branch}-{platform}-nightly.*.finished',
            # TODO: find a better way to specify these
            'build.{branch}-{platform}-l10n-nightly-1.*.finished',
            'build.{branch}-{platform}-l10n-nightly-2.*.finished',
            'build.{branch}-{platform}-l10n-nightly-3.*.finished',
            'build.{branch}-{platform}-l10n-nightly-4.*.finished',
            'build.{branch}-{platform}-l10n-nightly-5.*.finished',
            'build.{branch}-{platform}-l10n-nightly-6.*.finished',
            'build.{branch}-{platform}-l10n-nightly-7.*.finished',
            'build.{branch}-{platform}-l10n-nightly-8.*.finished',
            'build.{branch}-{platform}-l10n-nightly-9.*.finished',
            'build.{branch}-{platform}-l10n-nightly-10.*.finished',
        ]
        return [job.format(branch=branch, platform=platform)
                for job in jobs
                for branch in BRANCHES
                for platform in PLATFORMS]

    @property
    def queues(self):
        """List of queues used by worker.
        Multiple queues are used to track multiple routing keys.
        """
        return [Queue(name=self.queue_name, exchange=self.exchange,
                      routing_key=routing_key, durable=True, exclusive=False,
                      auto_delete=False)
                for routing_key in self.routing_keys]

    def get_consumers(self, Consumer, channel):
        """Implement parent's method called to get the list of consumers"""
        return [Consumer(queues=self.queues, callbacks=[self.process_message])]

    def process_message(self, body, message):
        """Top level callback processing pulse messages.
        The callback tries to handle and log all exceptions

        :type body: kombu.Message.body
        :type message: kombu.Message
        """
        try:
            self.dispatch_message(body)
        except Exception:
            log.exception("Failed to process message")
        finally:
            # TODO: figure out what to do with failed tasks
            message.ack()

    def on_consume_ready(self, connection, channel, consumers, **kwargs):
        """Overrides parent's stub method. Called when ready to consume pulse
         messages.
        """
        log.info('Listening...')

    def dispatch_message(self, body):
        """Dispatches incoming pulse messages.
        If the method detects L10N repacks, it creates multiple Taskcluster
        tasks, otherwise a single en-US task is created.
        :type body: kombu.Message.body
        """
        payload = body["payload"]
        buildername = payload["build"]["builderName"]
        if not interesting_buildername(buildername):
            log.debug("Ignoring %s: not interested", buildername)
            return
        job_result = payload["results"]
        if job_result != 0:
            log.debug("Ignoring %s with result %s", buildername, job_result)
            return
        properties = properties_to_dict(payload["build"]["properties"])
        if "locales" in properties:
            log.debug("L10N repack detected")
            funsize_info = json.loads(properties["funsize_info"])
            locales = json.loads(properties["locales"])
            platform = funsize_info["platform"]
            branch = funsize_info["branch"]
            product = funsize_info["appName"]
            for locale, result in locales.iteritems():
                if result.lower() == "success":
                    self.create_partial(product, branch, platform,
                                        locale)
                else:
                    log.warn("Ignoring %s with result %s", locale, result)
        else:
            log.debug("en-US build detected")
            self.create_partial(properties["appName"], properties["branch"],
                                properties["platform"], 'en-US')

    def create_partial(self, product, branch, platform, locale):
        """Calculates "from" and "to" MAR URLs and calls  create_task_graph().
        Currently "from" MAR is 2 releases behind to avoid duplication of
        existing CI partials.

        :param product: capitalized product name, AKA appName, e.g. Firefox
        :param branch: branch name (mozilla-central)
        :param platform: buildbot platform (linux, macosx64)
        :param locale: en-US or locale
        """<commit_msg>Support old style L10N repacks<commit_after>import datetime
import logging
import time
from kombu import Exchange, Queue
from kombu.mixins import ConsumerMixin
import os
import pystache
import re
from taskcluster import slugId, stringDate, fromNow
import yaml
import json

from funsize.utils import properties_to_dict, encrypt_env_var

log = logging.getLogger(__name__)

# TODO: move these to config
BRANCHES = ['mozilla-central', 'mozilla-aurora']
PLATFORMS = ['linux', 'linux64', 'win32', 'win64', 'macosx64']
BUILDERS = [
    r'WINNT \d+\.\d+ (x86-64 )?{branch} nightly',
    r'Linux (x86-64 )?{branch} nightly',
    r'OS X \d+\.\d+ {branch} nightly',
    r'Firefox {branch} (linux|linux64|win32|win64|macosx64) l10n nightly-\d+',
]


class FunsizeWorker(ConsumerMixin):

    def __init__(self, connection, queue_name, exchange, balrog_client,
                 scheduler):
        """Funsize consumer worker
        :type connection: kombu.Connection
        :param queue_name: Full queue name, including queue/<user> prefix
        :type exchange: kombu.Exchange
        :type balrog_client: funsize.balrog.BalrogClient
        :type scheduler: taskcluster.Scheduler
        """
        self.connection = connection
        # Using passive mode is important, otherwise pulse returns 403
        self.exchange = Exchange(exchange, type='topic', passive=True)
        self.queue_name = queue_name
        self.balrog_client = balrog_client
        self.scheduler = scheduler

    @property
    def routing_keys(self):
        """Returns an explicit list of routing patterns.

        Instead of using "build.*.*.finished", which generates a lot of noise,
        FunsizeWorker uses explicit list of routing keys to match builders we
        are interested in.
        """
        jobs = [
            # TODO: move to configs
            'build.{branch}-{platform}-nightly.*.finished',
            # TODO: find a better way to specify these
            'build.{branch}-{platform}-l10n-nightly-1.*.finished',
            'build.{branch}-{platform}-l10n-nightly-2.*.finished',
            'build.{branch}-{platform}-l10n-nightly-3.*.finished',
            'build.{branch}-{platform}-l10n-nightly-4.*.finished',
            'build.{branch}-{platform}-l10n-nightly-5.*.finished',
            'build.{branch}-{platform}-l10n-nightly-6.*.finished',
            'build.{branch}-{platform}-l10n-nightly-7.*.finished',
            'build.{branch}-{platform}-l10n-nightly-8.*.finished',
            'build.{branch}-{platform}-l10n-nightly-9.*.finished',
            'build.{branch}-{platform}-l10n-nightly-10.*.finished',
        ]
        return [job.format(branch=branch, platform=platform)
                for job in jobs
                for branch in BRANCHES
                for platform in PLATFORMS]

    @property
    def queues(self):
        """List of queues used by worker.
        Multiple queues are used to track multiple routing keys.
        """
        return [Queue(name=self.queue_name, exchange=self.exchange,
                      routing_key=routing_key, durable=True, exclusive=False,
                      auto_delete=False)
                for routing_key in self.routing_keys]

    def get_consumers(self, Consumer, channel):
        """Implement parent's method called to get the list of consumers"""
        return [Consumer(queues=self.queues, callbacks=[self.process_message])]

    def process_message(self, body, message):
        """Top level callback processing pulse messages.
        The callback tries to handle and log all exceptions

        :type body: kombu.Message.body
        :type message: kombu.Message
        """
        try:
            self.dispatch_message(body)
        except Exception:
            log.exception("Failed to process message")
        finally:
            # TODO: figure out what to do with failed tasks
            message.ack()

    def on_consume_ready(self, connection, channel, consumers, **kwargs):
        """Overrides parent's stub method. Called when ready to consume pulse
         messages.
        """
        log.info('Listening...')

    def dispatch_message(self, body):
        """Dispatches incoming pulse messages.
        If the method detects L10N repacks, it creates multiple Taskcluster
        tasks, otherwise a single en-US task is created.
        :type body: kombu.Message.body
        """
        payload = body["payload"]
        buildername = payload["build"]["builderName"]
        if not interesting_buildername(buildername):
            log.debug("Ignoring %s: not interested", buildername)
            return
        job_result = payload["results"]
        if job_result != 0:
            log.debug("Ignoring %s with result %s", buildername, job_result)
            return
        properties = properties_to_dict(payload["build"]["properties"])
        if "locales" in properties:
            log.debug("L10N repack detected")
            funsize_info = json.loads(properties["funsize_info"])
            locales = json.loads(properties["locales"])
            platform = funsize_info["platform"]
            branch = funsize_info["branch"]
            product = funsize_info["appName"]
            for locale, result in locales.iteritems():
                if result.lower() == "success":
                    self.create_partial(product, branch, platform,
                                        locale)
                else:
                    log.warn("Ignoring %s with result %s", locale, result)
        elif "locale" in properties:
            # Old-style buildbotcustom l10n repacks
            # TODO: this branch can be removed when we switch to mozharness
            #  based l10n repacks for TB and ESRs
            log.debug("Single locale repack detected (%s)",
                      properties["locale"])
            self.create_partial(properties["appName"], properties["branch"],
                                properties["platform"], properties["locale"])
        else:
            log.debug("en-US build detected")
            self.create_partial(properties["appName"], properties["branch"],
                                properties["platform"], 'en-US')

    def create_partial(self, product, branch, platform, locale):
        """Calculates "from" and "to" MAR URLs and calls  create_task_graph().
        Currently "from" MAR is 2 releases behind to avoid duplication of
        existing CI partials.

        :param product: capitalized product name, AKA appName, e.g. Firefox
        :param branch: branch name (mozilla-central)
        :param platform: buildbot platform (linux, macosx64)
        :param locale: en-US or locale
        """
{'fraction_non_alphanumeric': 0.10823657757344346, 'fraction_numerical': 0.011065222473310996, 'mean_word_length': 2.345672575599583, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5854711', 'n_tokens_mistral': 3604, 'n_tokens_neox': 3439, 'n_words': 962}