<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import codecs
import datetime
import urllib2
import os
import tempfile
import unicodedata

import pyhs2

from configman import Namespace, class_converter
from crontabber.base import BaseCronApp
from crontabber.mixins import as_backfill_cron_app
from socorro.external.postgresql.connection_context import ConnectionContext
from socorro.external.postgresql.dbapi2_util import execute_no_results


class NoRowsWritten(Exception):
    pass


"""
 Detailed documentation on columns avaiable from our Hive system at:
 https://intranet.mozilla.org/Metrics/Blocklist

 Columns being queried are:
    report_date
    product_name
    product_os_platform
    product_os_version
    product_version
    build
    build_channel
    product_guid
    count
"""

_QUERY = """
    select
        ds,
        split(request_url,'/')[5],
        split(split(request_url,'/')[10], '%%20')[0],
        split(split(request_url,'/')[10], '%%20')[1],
        split(request_url,'/')[4],
        split(request_url,'/')[6],
        split(request_url,'/')[9],
        split(request_url,'/')[3],
        count(*)
    FROM v2_raw_logs
    WHERE
        (domain='addons.mozilla.org' OR domain='blocklist.addons.mozilla.org')
        and http_status_code = '200'
        and request_url like '/blocklist/3/%%'
        and ds='%s'
    GROUP BY
        ds,
        split(request_url,'/')[5],
        split(split(request_url,'/')[10], '%%20')[0],
        split(split(request_url,'/')[10], '%%20')[1],
        split(request_url,'/')[4],
        split(request_url,'/')[6],
        split(request_url,'/')[9],
        split(request_url,'/')[3]
"""

_RAW_ADI_QUERY = """
    INSERT INTO raw_adi (
        adi_count,
        date,
        product_name,
        product_os_platform,
        product_os_version,
        product_version,
        build,
        product_guid,
        update_channel
    )
    SELECT
        sum(count),
        report_date,
        CASE WHEN (raw_adi_logs.product_name = 'Fennec'
            AND product_guid = '{aa3c5121-dab2-40e2-81ca-7ea25febc110}')
        THEN 'FennecAndroid'
        WHEN (raw_adi_logs.product_name = 'Webapp Runtime')
        THEN 'WebappRuntime'
        ELSE raw_adi_logs.product_name
        END,
        product_os_platform,
        product_os_version,<commit_msg>bug 1307980 - new Hive URL schema (#3538)<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import codecs
import datetime
import urllib2
import os
import tempfile
import unicodedata

import pyhs2

from configman import Namespace, class_converter
from crontabber.base import BaseCronApp
from crontabber.mixins import as_backfill_cron_app
from socorro.external.postgresql.connection_context import ConnectionContext
from socorro.external.postgresql.dbapi2_util import execute_no_results


class NoRowsWritten(Exception):
    pass


"""
 Detailed documentation on columns avaiable from our Hive system at:
 https://intranet.mozilla.org/Metrics/Blocklist

 Columns being queried are:
    report_date
    product_name
    product_os_platform
    product_os_version
    product_version
    build
    build_channel
    product_guid
    count
"""

_QUERY = """
    select
        ds,
        split(request_url,'/')[6],
        split(split(request_url,'/')[11], '%%20')[0],
        split(split(request_url,'/')[11], '%%20')[1],
        split(request_url,'/')[5],
        split(request_url,'/')[7],
        split(request_url,'/')[10],
        split(request_url,'/')[4],
        count(*)
    FROM v2_raw_logs
    WHERE
        (domain='addons.mozilla.org' OR domain='blocklist.addons.mozilla.org')
        and http_status_code = '200'
        and request_url like '/v1/blocklist/3/%%'
        and ds='%s'
    GROUP BY
        ds,
        split(request_url,'/')[6],
        split(split(request_url,'/')[11], '%%20')[0],
        split(split(request_url,'/')[11], '%%20')[1],
        split(request_url,'/')[5],
        split(request_url,'/')[7],
        split(request_url,'/')[10],
        split(request_url,'/')[4]
"""

_RAW_ADI_QUERY = """
    INSERT INTO raw_adi (
        adi_count,
        date,
        product_name,
        product_os_platform,
        product_os_version,
        product_version,
        build,
        product_guid,
        update_channel
    )
    SELECT
        sum(count),
        report_date,
        CASE WHEN (raw_adi_logs.product_name = 'Fennec'
            AND product_guid = '{aa3c5121-dab2-40e2-81ca-7ea25febc110}')
        THEN 'FennecAndroid'
        WHEN (raw_adi_logs.product_name = 'Webapp Runtime')
        THEN 'WebappRuntime'
        ELSE raw_adi_logs.product_name
        END,
        product_os_platform,
        product_os_version,
{'fraction_non_alphanumeric': 0.11810396335391356, 'fraction_numerical': 0.02628958374825732, 'mean_word_length': 2.810318664643399, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5312115', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1649, 'n_words': 351}