<commit_before>import json
import logging
import os
import sys
import traceback

from argparse import ArgumentParser

from pulse_actions.authentication import (
    AuthenticationError,
    get_user_and_password,
)
from pulse_actions.handlers import config, route_functions

from mozci.mozci import disable_validations
from mozci.utils import transfer
from mozillapulse.config import PulseConfiguration
from mozillapulse.consumers import GenericConsumer
# This changes the behaviour of mozci in transfer.py
transfer.MEMORY_SAVING_MODE = True
transfer.SHOW_PROGRESS_BAR = False

LOG = None


class PulseConsumer(GenericConsumer):
    """
    Creates a consumer object for the given exchange.

    Documentation for the exchanges:
    https://wiki.mozilla.org/Auto-tools/Projects/Pulse/Exchanges
    """

    def __init__(self, exchange, **kwargs):
        super(PulseConsumer, self).__init__(
            PulseConfiguration(**kwargs), exchange, **kwargs)


def run_pulse(exchanges, topics, event_handler, topic_base, dry_run):
    """
    Listen to a pulse exchange in a infinite loop.

    Call event_handler on every message.
    """
    if len(topic_base) == 1:
        label = topic_base[0]
    else:
        label = str(topic_base)
    try:
        user, password = get_user_and_password()
    except AuthenticationError as e:
        print(e.message)
        sys.exit(1)

    pulse_args = {
        'applabel': label,
        'topic': topics,
        'durable': True,
        'user': user,
        'password': password
    }

    # Pulse consumer's callback passes only data and message arguments
    # to the function, we need to pass dry-run
    def handler_with_dry_run(data, message):
        return event_handler(data, message, dry_run)

    pulse = PulseConsumer(exchanges,
                          callback=handler_with_dry_run,
                          **pulse_args)
    LOG.info('Listening on %s, with topic %s', exchanges, topics)

    while True:
        try:
            pulse.listen()
        except KeyboardInterrupt:
            sys.exit(1)
        except:
            traceback.print_exc()


def setup_logging(logging_level):
    global LOG
    if LOG:
        return LOG
    # Let's use the root logger
    LOG = logging.getLogger()
    LOG.setLevel(logging.DEBUG)

    format = '%(asctime)s %(name)s\t %(levelname)s:\t %(message)s'
    formatter = logging.Formatter(format, datefmt='%H:%M:%S')
    # Handler 1 - Store all INFO messages in a specific file
    # This logging.txt file will *only* show messages for that same day as Heroku<commit_msg>Mesure request time<commit_after>import json
import logging
import os
import sys
import traceback

from argparse import ArgumentParser
from timeit import default_timer

from pulse_actions.authentication import (
    AuthenticationError,
    get_user_and_password,
)
from pulse_actions.handlers import config, route_functions

from mozci.mozci import disable_validations
from mozci.utils import transfer
from mozillapulse.config import PulseConfiguration
from mozillapulse.consumers import GenericConsumer
# This changes the behaviour of mozci in transfer.py
transfer.MEMORY_SAVING_MODE = True
transfer.SHOW_PROGRESS_BAR = False

LOG = None


class PulseConsumer(GenericConsumer):
    """
    Creates a consumer object for the given exchange.

    Documentation for the exchanges:
    https://wiki.mozilla.org/Auto-tools/Projects/Pulse/Exchanges
    """

    def __init__(self, exchange, **kwargs):
        super(PulseConsumer, self).__init__(
            PulseConfiguration(**kwargs), exchange, **kwargs)


def run_pulse(exchanges, topics, event_handler, topic_base, dry_run):
    """
    Listen to a pulse exchange in a infinite loop.

    Call event_handler on every message.
    """
    if len(topic_base) == 1:
        label = topic_base[0]
    else:
        label = str(topic_base)
    try:
        user, password = get_user_and_password()
    except AuthenticationError as e:
        print(e.message)
        sys.exit(1)

    pulse_args = {
        'applabel': label,
        'topic': topics,
        'durable': True,
        'user': user,
        'password': password
    }

    # Pulse consumer's callback passes only data and message arguments
    # to the function, we need to pass dry-run
    def handler_with_dry_run(data, message):
        start_time = default_timer.time()
        event_handler(data, message, dry_run)
        elapsed_time = default_timer() - start_time
        LOG.info('Message %s, took %s to execute', message, str(elapsed_time))

    pulse = PulseConsumer(exchanges,
                          callback=handler_with_dry_run,
                          **pulse_args)
    LOG.info('Listening on %s, with topic %s', exchanges, topics)

    while True:
        try:
            pulse.listen()
        except KeyboardInterrupt:
            sys.exit(1)
        except:
            traceback.print_exc()


def setup_logging(logging_level):
    global LOG
    if LOG:
        return LOG
    # Let's use the root logger
    LOG = logging.getLogger()
    LOG.setLevel(logging.DEBUG)

    format = '%(asctime)s %(name)s\t %(levelname)s:\t %(message)s'
    formatter = logging.Formatter(format, datefmt='%H:%M:%S')
    # Handler 1 - Store all INFO messages in a specific file
    # This logging.txt file will *only* show messages for that same day as Heroku
{'fraction_non_alphanumeric': 0.08559147822836853, 'fraction_numerical': 0.0018688095683049898, 'mean_word_length': 2.9094229364499635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4590551', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1533, 'n_words': 455}