<commit_before># This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""This module is used to send boot, build and test report emails."""

import bson
import copy
import datetime
import hashlib
import redis
import types

import handlers.base as hbase
import handlers.response as hresponse
import models
import taskqueue.tasks.report as taskq
import utils

# Max delay in sending email report set to 5hrs.
MAX_DELAY = 18000

TRIGGER_RECEIVED = "Email trigger received from '%s' for '%s-%s-%s' at %s (%s)"
TRIGGER_RECEIVED_ALREADY = "Email trigger for '%s-%s-%s' (%s) already received"
ERR_409_MESSAGE = "Email request already registered"

REPORT_TYPE_KEYS = {
    'build': [
        models.JOB_KEY,
        models.GIT_BRANCH_KEY,
        models.KERNEL_KEY,
    ],
    'boot': [
        models.JOB_KEY,
        models.GIT_BRANCH_KEY,
        models.KERNEL_KEY,
        models.LAB_NAME_KEY,
    ],
    'bisect': [
        models.JOB_KEY,
        models.GIT_BRANCH_KEY,
        models.KERNEL_KEY,
        models.TYPE_KEY,
        models.ARCHITECTURE_KEY,
        models.DEFCONFIG_FULL_KEY,
        models.LAB_NAME_KEY,
        models.BISECT_LOG_KEY,
        models.DEVICE_TYPE_KEY,
        models.BISECT_GOOD_COMMIT_KEY,
        models.BISECT_BAD_COMMIT_KEY,
    ],
    'test': [
        models.JOB_KEY,
        models.GIT_BRANCH_KEY,
        models.KERNEL_KEY,
        models.PLAN_KEY,
    ],
}


# pylint: disable=too-many-public-methods
class SendHandler(hbase.BaseHandler):
    """Handle the /send URLs."""

    def __init__(self, application, request, **kwargs):
        super(SendHandler, self).__init__(application, request, **kwargs)

    @staticmethod
    def _valid_keys(method):
        return models.SEND_VALID_KEYS.get(method, None)

    # pylint: disable=too-many-locals
    def _post(self, *args, **kwargs):
        response = hresponse.HandlerResponse(202)
        json_obj = kwargs["json_obj"]
        j_get = json_obj.get

        # Mandatory keys
        job = j_get(models.JOB_KEY)
        kernel = j_get(models.KERNEL_KEY)
        branch = utils.clean_branch_name(j_get(models.GIT_BRANCH_KEY))

        # Optional keys
        report_type = j_get(models.REPORT_TYPE_KEY)
        countdown = j_get(models.DELAY_KEY)
        if countdown is None:
            countdown = self.settings["senddelay"]

        # Deprecated - ToDo: use report_type only in client code
        if j_get(models.SEND_BOOT_REPORT_KEY):
            report_type = 'boot'
        elif j_get(models.SEND_BUILD_REPORT_KEY):
            report_type = 'build'

        report_keys = REPORT_TYPE_KEYS.get(report_type)
        if not report_keys:
            response.status_code = 400
            response.reason = (
                "Invalid report type: {}.  Valid values are: {}"
                .format(report_type, ", ".join(REPORT_TYPE_KEYS.keys()))
            )
            return response
        report_data = {k: j_get(k) for k in report_keys}

        email_format = j_get(models.EMAIL_FORMAT_KEY, None)
        email_format, email_errors = _check_email_format(email_format)
        response.errors = email_errors
        schedule_errors = None

        try:
            countdown = int(countdown)
            if countdown < 0:
                countdown = abs(countdown)
                response.errrors = (
                    "Negative value specified for the '%s' key, "
                    "its positive value will be used instead (%ds)" %
                    (models.DELAY_KEY, countdown)
                )

            if countdown > MAX_DELAY:
                response.errors = (
                    "Delay value specified out of range (%ds), "
                    "maximum delay permitted (%ds) will be used instead" %
                    (countdown, MAX_DELAY)
                )
                countdown = MAX_DELAY

            when = (
                datetime.datetime.now(tz=bson.tz_util.utc) +
                datetime.timedelta(seconds=countdown))

            def j_get_list(key):
                value = j_get(key)
                if value is None:
                    value = []
                elif not isinstance(value, list):
                    value = [value]
                return value

            email_opts = {
                "to": j_get_list(models.REPORT_SEND_TO_KEY),
                "cc": j_get_list(models.REPORT_CC_KEY),
                "bcc": j_get_list(models.REPORT_BCC_KEY),
                "in_reply_to": j_get(models.IN_REPLY_TO_KEY),
                "subject": j_get(models.SUBJECT_KEY),
                "format": email_format,
            }

            self.log.info(
                TRIGGER_RECEIVED,
                self.request.remote_ip,
                job,
                branch,
                kernel,
                datetime.datetime.utcnow(),
                report_type
            )

            hashable_str = ''.join(str(x) for x in [
                job,
                branch,
                kernel,
                email_opts["to"],
                email_opts["cc"],
                email_opts["bcc"],
                email_opts["in_reply_to"],
                email_opts["subject"],
                report_type,
                str(email_format),
            ])
            schedule_hash = hashlib.sha1(hashable_str).hexdigest()

            try:
                lock_key = '-'.join([
                    'email', report_type, job, branch, kernel])

                with redis.lock.Lock(self.redisdb, lock_key, timeout=2):
                    if not self.redisdb.exists(schedule_hash):
                        self.redisdb.set(
                            schedule_hash, "schedule", ex=86400)

                        schedule_method = getattr(
                            self, "_schedule_{}_report".format(report_type))

                        errors, response.errors = schedule_method(
                            report_data, email_opts, countdown)

                        response.reason, response.status_code = \
                            _check_status(report_type, errors, when)
                    else:
                        self.log.warn(
                            TRIGGER_RECEIVED_ALREADY,
                            job, branch, kernel, report_type
                        )
                        taskq.send_multiple_emails_error.apply_async(
                            [
                                job,
                                branch,
                                kernel,
                                datetime.datetime.utcnow(),
                                email_format,
                                report_type,
                                email_opts
                            ]
                        )
                        response.status_code = 409
                        response.reason = ERR_409_MESSAGE
            except redis.lock.LockError:
                # Probably only reached during the unit tests.<commit_msg>Include test plan name in test report email unique key<commit_after># This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""This module is used to send boot, build and test report emails."""

import bson
import copy
import datetime
import hashlib
import redis
import types

import handlers.base as hbase
import handlers.response as hresponse
import models
import taskqueue.tasks.report as taskq
import utils

# Max delay in sending email report set to 5hrs.
MAX_DELAY = 18000

TRIGGER_RECEIVED = "Email trigger received from '%s' for '%s-%s-%s' at %s (%s)"
TRIGGER_RECEIVED_ALREADY = "Email trigger for '%s-%s-%s' (%s) already received"
ERR_409_MESSAGE = "Email request already registered"

REPORT_TYPE_KEYS = {
    'build': [
        models.JOB_KEY,
        models.GIT_BRANCH_KEY,
        models.KERNEL_KEY,
    ],
    'boot': [
        models.JOB_KEY,
        models.GIT_BRANCH_KEY,
        models.KERNEL_KEY,
        models.LAB_NAME_KEY,
    ],
    'bisect': [
        models.JOB_KEY,
        models.GIT_BRANCH_KEY,
        models.KERNEL_KEY,
        models.TYPE_KEY,
        models.ARCHITECTURE_KEY,
        models.DEFCONFIG_FULL_KEY,
        models.LAB_NAME_KEY,
        models.BISECT_LOG_KEY,
        models.DEVICE_TYPE_KEY,
        models.BISECT_GOOD_COMMIT_KEY,
        models.BISECT_BAD_COMMIT_KEY,
    ],
    'test': [
        models.JOB_KEY,
        models.GIT_BRANCH_KEY,
        models.KERNEL_KEY,
        models.PLAN_KEY,
    ],
}


# pylint: disable=too-many-public-methods
class SendHandler(hbase.BaseHandler):
    """Handle the /send URLs."""

    def __init__(self, application, request, **kwargs):
        super(SendHandler, self).__init__(application, request, **kwargs)

    @staticmethod
    def _valid_keys(method):
        return models.SEND_VALID_KEYS.get(method, None)

    # pylint: disable=too-many-locals
    def _post(self, *args, **kwargs):
        response = hresponse.HandlerResponse(202)
        json_obj = kwargs["json_obj"]
        j_get = json_obj.get

        # Mandatory keys
        job = j_get(models.JOB_KEY)
        kernel = j_get(models.KERNEL_KEY)
        branch = utils.clean_branch_name(j_get(models.GIT_BRANCH_KEY))

        # Optional keys
        report_type = j_get(models.REPORT_TYPE_KEY)
        countdown = j_get(models.DELAY_KEY)
        if countdown is None:
            countdown = self.settings["senddelay"]

        # Deprecated - ToDo: use report_type only in client code
        if j_get(models.SEND_BOOT_REPORT_KEY):
            report_type = 'boot'
        elif j_get(models.SEND_BUILD_REPORT_KEY):
            report_type = 'build'

        report_keys = REPORT_TYPE_KEYS.get(report_type)
        if not report_keys:
            response.status_code = 400
            response.reason = (
                "Invalid report type: {}.  Valid values are: {}"
                .format(report_type, ", ".join(REPORT_TYPE_KEYS.keys()))
            )
            return response
        report_data = {k: j_get(k) for k in report_keys}

        email_format = j_get(models.EMAIL_FORMAT_KEY, None)
        email_format, email_errors = _check_email_format(email_format)
        response.errors = email_errors
        schedule_errors = None

        try:
            countdown = int(countdown)
            if countdown < 0:
                countdown = abs(countdown)
                response.errrors = (
                    "Negative value specified for the '%s' key, "
                    "its positive value will be used instead (%ds)" %
                    (models.DELAY_KEY, countdown)
                )

            if countdown > MAX_DELAY:
                response.errors = (
                    "Delay value specified out of range (%ds), "
                    "maximum delay permitted (%ds) will be used instead" %
                    (countdown, MAX_DELAY)
                )
                countdown = MAX_DELAY

            when = (
                datetime.datetime.now(tz=bson.tz_util.utc) +
                datetime.timedelta(seconds=countdown))

            def j_get_list(key):
                value = j_get(key)
                if value is None:
                    value = []
                elif not isinstance(value, list):
                    value = [value]
                return value

            email_opts = {
                "to": j_get_list(models.REPORT_SEND_TO_KEY),
                "cc": j_get_list(models.REPORT_CC_KEY),
                "bcc": j_get_list(models.REPORT_BCC_KEY),
                "in_reply_to": j_get(models.IN_REPLY_TO_KEY),
                "subject": j_get(models.SUBJECT_KEY),
                "format": email_format,
            }

            report_type_or_plan = j_get(models.PLAN_KEY, report_type)

            self.log.info(
                TRIGGER_RECEIVED,
                self.request.remote_ip,
                job,
                branch,
                kernel,
                datetime.datetime.utcnow(),
                report_type_or_plan
            )

            hashable_str = ''.join(str(x) for x in [
                job,
                branch,
                kernel,
                email_opts["to"],
                email_opts["cc"],
                email_opts["bcc"],
                email_opts["in_reply_to"],
                email_opts["subject"],
                report_type_or_plan,
                str(email_format),
            ])
            schedule_hash = hashlib.sha1(hashable_str).hexdigest()

            try:
                lock_key = '-'.join([
                    'email', report_type, job, branch, kernel])

                with redis.lock.Lock(self.redisdb, lock_key, timeout=2):
                    if not self.redisdb.exists(schedule_hash):
                        self.redisdb.set(
                            schedule_hash, "schedule", ex=86400)

                        schedule_method = getattr(
                            self, "_schedule_{}_report".format(report_type))

                        errors, response.errors = schedule_method(
                            report_data, email_opts, countdown)

                        response.reason, response.status_code = \
                            _check_status(report_type, errors, when)
                    else:
                        self.log.warn(
                            TRIGGER_RECEIVED_ALREADY,
                            job, branch, kernel, report_type_or_plan
                        )
                        taskq.send_multiple_emails_error.apply_async(
                            [
                                job,
                                branch,
                                kernel,
                                datetime.datetime.utcnow(),
                                email_format,
                                report_type,
                                email_opts
                            ]
                        )
                        response.status_code = 409
                        response.reason = ERR_409_MESSAGE
            except redis.lock.LockError:
                # Probably only reached during the unit tests.
{'fraction_non_alphanumeric': 0.08117863236645229, 'fraction_numerical': 0.00393752460952881, 'mean_word_length': 1.5654882154882155, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4291721', 'n_tokens_mistral': 4339, 'n_tokens_neox': 4105, 'n_words': 971}