<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import datetime

from configman import Namespace

from socorro.cron.base import BaseCronApp
from socorro.cron.mixins import (
    as_backfill_cron_app,
    using_postgres,
)
from socorro.external.postgresql.dbapi2_util import (
    execute_no_results,
    single_row_sql,
    SQLDidNotReturnSingleRow,
)
from socorro.external.postgresql.signature_first_date import SignatureFirstDate
from socorro.external.es.base import ElasticsearchConfig
from socorro.external.es.supersearch import SuperSearch
from socorro.external.es.super_search_fields import SuperSearchFields


# Maximum number of results returned for a super search query
MAX_PAGE = 1000


@using_postgres()
@as_backfill_cron_app
class UpdateSignaturesCronApp(BaseCronApp):
    """Updates the signatures table using crash data from Elasticsearch"""
    app_name = 'update-signatures'
    app_description = 'Updates signatures table'
    app_version = '0.1'

    required_config = Namespace()
    # NOTE(willkg): period defaults to 90 minutes because this job runs hourly
    # and that creates some overlap with the previous period even if this run
    # and the last run didn't run on the hour.
    required_config.add_option(
        'period',
        default=90,
        doc='Length of the window to look at in minutes'
    )
    required_config.add_option(
        'elasticsearch_class',
        default=ElasticsearchConfig,
    )
    required_config.add_option(
        'dry_run',
        default=False,
        doc='Print to stdout instead of updating/inserting data'
    )

    def update_crashstats_signature(self, connection, signature, report_date, report_build):
        # Pull the data in the db. If it's there, then do an update. If it's
        # not there, then do an insert.
        try:
            sig = single_row_sql(
                connection,
                """
                SELECT signature, first_build, first_date
                FROM crashstats_signature
                WHERE signature=%s
                """,
                (signature,)
            )
            sql = """
            UPDATE crashstats_signature
            SET first_build=%s, first_date=%s
            WHERE signature=%s
            """
            params = (
                min(sig[1], report_build),
                min(str(sig[2]), str(report_date)),
                sig[0]
            )
        except SQLDidNotReturnSingleRow:
            sql = """
            INSERT INTO crashstats_signature (signature, first_build, first_date)
            VALUES (%s, %s, %s)
            """
            params = (signature, report_build, report_date)

        execute_no_results(connection, sql, params)

    def run(self, end_datetime):
        # Truncate to the hour
        # FIXME end_datetime = end_datetime.replace(minute=0, second=0, microsecond=0)<commit_msg>Undo commented out line<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import datetime

from configman import Namespace

from socorro.cron.base import BaseCronApp
from socorro.cron.mixins import (
    as_backfill_cron_app,
    using_postgres,
)
from socorro.external.postgresql.dbapi2_util import (
    execute_no_results,
    single_row_sql,
    SQLDidNotReturnSingleRow,
)
from socorro.external.postgresql.signature_first_date import SignatureFirstDate
from socorro.external.es.base import ElasticsearchConfig
from socorro.external.es.supersearch import SuperSearch
from socorro.external.es.super_search_fields import SuperSearchFields


# Maximum number of results returned for a super search query
MAX_PAGE = 1000


@using_postgres()
@as_backfill_cron_app
class UpdateSignaturesCronApp(BaseCronApp):
    """Updates the signatures table using crash data from Elasticsearch"""
    app_name = 'update-signatures'
    app_description = 'Updates signatures table'
    app_version = '0.1'

    required_config = Namespace()
    # NOTE(willkg): period defaults to 90 minutes because this job runs hourly
    # and that creates some overlap with the previous period even if this run
    # and the last run didn't run on the hour.
    required_config.add_option(
        'period',
        default=90,
        doc='Length of the window to look at in minutes'
    )
    required_config.add_option(
        'elasticsearch_class',
        default=ElasticsearchConfig,
    )
    required_config.add_option(
        'dry_run',
        default=False,
        doc='Print to stdout instead of updating/inserting data'
    )

    def update_crashstats_signature(self, connection, signature, report_date, report_build):
        # Pull the data in the db. If it's there, then do an update. If it's
        # not there, then do an insert.
        try:
            sig = single_row_sql(
                connection,
                """
                SELECT signature, first_build, first_date
                FROM crashstats_signature
                WHERE signature=%s
                """,
                (signature,)
            )
            sql = """
            UPDATE crashstats_signature
            SET first_build=%s, first_date=%s
            WHERE signature=%s
            """
            params = (
                min(sig[1], report_build),
                min(str(sig[2]), str(report_date)),
                sig[0]
            )
        except SQLDidNotReturnSingleRow:
            sql = """
            INSERT INTO crashstats_signature (signature, first_build, first_date)
            VALUES (%s, %s, %s)
            """
            params = (signature, report_build, report_date)

        execute_no_results(connection, sql, params)

    def run(self, end_datetime):
        # Truncate to the hour
        end_datetime = end_datetime.replace(minute=0, second=0, microsecond=0)
{'fraction_non_alphanumeric': 0.07518920697597894, 'fraction_numerical': 0.006910167818361303, 'mean_word_length': 2.4856651376146788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1198745', 'n_tokens_mistral': 1749, 'n_tokens_neox': 1656, 'n_words': 503}