<commit_before># -*- coding: utf-8 -*-
#
# awsdbrparser/parser.py
#
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function

import collections
import csv
import json
import threading
import time

import boto3
import click
from elasticsearch import Elasticsearch, RequestsHttpConnection, helpers
from requests_aws4auth import AWS4Auth

from . import utils
from .config import PROCESS_BY_BULK, PROCESS_BY_LINE, PROCESS_BI_ONLY

Summary = collections.namedtuple('Summary', 'added skipped updated control_messages')
"""
Holds the summary of documents processed by the parser.
"""


class ParserError(Exception):
    pass


def analytics(config, echo):
    """
    This function generate extra information in elasticsearch analyzing the line items of the file
    :param echo:
    :param config:
    :return:
    """
    index_name = '{}-{:d}-{:02d}'.format(
        config.es_index,
        config.es_year,
        config.es_month)

    # Opening Input filename again to run in parallel
    file_in = open(config.input_filename, 'r')
    awsauth = None
    if config.awsauth:
        session = boto3.Session()
        credentials = session.get_credentials()
        if credentials:
            region = session.region_name
            awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, 'es',
                               session_token=credentials.token)

    es = Elasticsearch([{'host': config.es_host, 'port': config.es_port}], timeout=config.es_timeout, http_auth=awsauth,
                       connection_class=RequestsHttpConnection)
    es.indices.create(index_name, ignore=400)

    csv_file = csv.DictReader(file_in, delimiter=config.csv_delimiter)
    analytics_daytime = dict()
    analytics_day_only = dict()
    for recno, json_row in enumerate(csv_file):
        # Pre-Process the row to append extra information
        json_row = utils.pre_process(json_row)
        if is_control_message(json_row, config):
            # Skip this line
            continue
        elif json_row.get('ProductName') == 'Amazon Elastic Compute Cloud' and 'RunInstances' in json_row.get(
                'Operation') and json_row.get('UsageItem'):
            # Get the day time ('2016-03-01 01:00:00')
            daytime = json_row.get('UsageStartDate')
            # the day only '2016-03-01'
            day = json_row.get('UsageStartDate').split(' ')[0]
            # Add the day time to the dict
            analytics_daytime.setdefault(daytime, {"Count": 0, "Cost": 0.00, "RI": 0, "Spot": 0, "Unblended": 0.00})
            # Increment the count of total instances
            analytics_daytime[daytime]["Count"] += 1
            analytics_daytime[daytime]["Unblended"] += float(json_row.get('UnBlendedCost', 0.00))
            analytics_daytime[daytime]["Cost"] += float(json_row.get('Cost', 0.00))

            # Add the day only to the dict
            analytics_day_only.setdefault(day, {"Count": 0, "RI": 0, "Spot": 0, "Min": None, "Max": None})
            analytics_day_only[day]["Count"] += 1<commit_msg>add missing index<commit_after># -*- coding: utf-8 -*-
#
# awsdbrparser/parser.py
#
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function

import collections
import csv
import json
import threading
import time

import boto3
import click
from elasticsearch import Elasticsearch, RequestsHttpConnection, helpers
from requests_aws4auth import AWS4Auth

from . import utils
from .config import PROCESS_BY_BULK, PROCESS_BY_LINE, PROCESS_BI_ONLY

Summary = collections.namedtuple('Summary', 'added skipped updated control_messages')
"""
Holds the summary of documents processed by the parser.
"""


class ParserError(Exception):
    pass


def analytics(config, echo):
    """
    This function generate extra information in elasticsearch analyzing the line items of the file
    :param echo:
    :param config:
    :return:
    """
    index_name = '{}-{:d}-{:02d}'.format(
        config.es_index,
        config.es_year,
        config.es_month)

    # Opening Input filename again to run in parallel
    file_in = open(config.input_filename, 'r')
    awsauth = None
    if config.awsauth:
        session = boto3.Session()
        credentials = session.get_credentials()
        if credentials:
            region = session.region_name
            awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, 'es',
                               session_token=credentials.token)

    es = Elasticsearch([{'host': config.es_host, 'port': config.es_port}], timeout=config.es_timeout, http_auth=awsauth,
                       connection_class=RequestsHttpConnection)
    es.indices.create(index_name, ignore=400)
    es.indices.create(config.es_doctype, ignore=400)

    csv_file = csv.DictReader(file_in, delimiter=config.csv_delimiter)
    analytics_daytime = dict()
    analytics_day_only = dict()
    for recno, json_row in enumerate(csv_file):
        # Pre-Process the row to append extra information
        json_row = utils.pre_process(json_row)
        if is_control_message(json_row, config):
            # Skip this line
            continue
        elif json_row.get('ProductName') == 'Amazon Elastic Compute Cloud' and 'RunInstances' in json_row.get(
                'Operation') and json_row.get('UsageItem'):
            # Get the day time ('2016-03-01 01:00:00')
            daytime = json_row.get('UsageStartDate')
            # the day only '2016-03-01'
            day = json_row.get('UsageStartDate').split(' ')[0]
            # Add the day time to the dict
            analytics_daytime.setdefault(daytime, {"Count": 0, "Cost": 0.00, "RI": 0, "Spot": 0, "Unblended": 0.00})
            # Increment the count of total instances
            analytics_daytime[daytime]["Count"] += 1
            analytics_daytime[daytime]["Unblended"] += float(json_row.get('UnBlendedCost', 0.00))
            analytics_daytime[daytime]["Cost"] += float(json_row.get('Cost', 0.00))

            # Add the day only to the dict
            analytics_day_only.setdefault(day, {"Count": 0, "RI": 0, "Spot": 0, "Min": None, "Max": None})
            analytics_day_only[day]["Count"] += 1
{'fraction_non_alphanumeric': 0.10284702356627162, 'fraction_numerical': 0.017300095354856288, 'mean_word_length': 3.134009009009009, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1268532', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2105, 'n_words': 645}