<commit_before># Copyright 2019 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Timesketch data importer."""
from __future__ import unicode_literals

import codecs
import io
import json
import logging
import math
import os
import time
import uuid

import pandas

from timesketch_api_client import timeline
from timesketch_api_client import definitions
from timesketch_import_client import utils

logger = logging.getLogger('timesketch_importer.importer')


class ImportStreamer(object):
    """Upload object used to stream results to Timesketch."""

    # The number of entries before automatically flushing
    # the streamer.
    DEFAULT_ENTRY_THRESHOLD = 50000

    # Number of bytes in a binary file before automatically
    # chunking it up into smaller pieces.
    DEFAULT_FILESIZE_THRESHOLD = 104857600  # 100 Mb.

    # Define default values.
    DEFAULT_TEXT_ENCODING = 'utf-8'
    DEFAULT_TIMESTAMP_DESC = 'Time Logged'
    DEFAULT_DATA_TYPE = 'data:entry'

    def __init__(self):
        """Initialize the upload streamer."""
        self._count = 0
        self._config_helper = None
        self._dict_config_loaded = False
        self._csv_delimiter = None
        self._data_lines = []
        self._datetime_field = None
        self._format_string = None
        self._index = uuid.uuid4().hex
        self._last_response = None
        self._resource_url = ''
        self._sketch = None
        self._timeline_id = None
        self._timeline_name = None

        self._chunk = 1

        self._data_type = self.DEFAULT_DATA_TYPE
        self._text_encoding = self.DEFAULT_TEXT_ENCODING
        self._timestamp_desc = self.DEFAULT_TIMESTAMP_DESC
        self._threshold_entry = self.DEFAULT_ENTRY_THRESHOLD
        self._threshold_filesize = self.DEFAULT_FILESIZE_THRESHOLD

    def _fix_dict(self, my_dict):
        """Adjusts a dict with so that it can be uploaded to Timesketch.

        This function will take a dictionary and modify it. Summary of the
        changes are:
          * If "message" is not a key and a format message string has been
              defined, a message field is constructed.
          * If "datetime" is not a key, an attempt to generate it is made.
          * If "timestamp_desc" is not set but defined by the importer it
              is added.
          * All keys that start with an underscore ("_") are removed.

        Args:
            my_dict: a dictionary that may be missing few fields needed
                    for Timesketch.
        """
        if 'message' not in my_dict:
            format_string = (
                self._format_string or utils.get_combined_message_string(
                    mydict=my_dict))
            my_dict['message'] = format_string.format(**my_dict)

        _ = my_dict.setdefault('timestamp_desc', self._timestamp_desc)
        _ = my_dict.setdefault('data_type', self._data_type)

        if 'datetime' not in my_dict:
            date = ''
            if self._datetime_field:
                value = my_dict.get(self._datetime_field)
                if value:
                    date = utils.get_datestring_from_value(value)
            if not date:
                for key in my_dict:
                    key_string = key.lower()
                    if 'time' not in key_string:
                        continue

                    if key_string == 'timestamp_desc':
                        continue

                    value = my_dict[key]
                    date = utils.get_datestring_from_value(value)
                    if date:
                        break

            if date:
                my_dict['datetime'] = date

        # We don't want to include any columns that start with an underscore.
        underscore_columns = [x for x in my_dict if x.startswith('_')]
        if underscore_columns:
            for column in underscore_columns:
                del my_dict[column]

    def _fix_data_frame(self, data_frame):
        """Returns a data frame with added columns for Timesketch upload.

        Args:
            data_frame: a pandas data frame.

        Returns:
            A pandas data frame with added columns needed for Timesketch.
        """
        if 'message' not in data_frame:
            format_string = (
                self._format_string or utils.get_combined_message_string(
                    dataframe=data_frame))
            utils.format_data_frame(data_frame, format_string)

        if 'timestamp_desc' not in data_frame:
            data_frame['timestamp_desc'] = self._timestamp_desc

        if 'data_type' not in data_frame:
            data_frame['data_type'] = self._data_type

        if 'datetime' not in data_frame:
            if self._datetime_field and self._datetime_field in data_frame:
                try:
                    data_frame['timestamp'] = pandas.to_datetime(
                        data_frame[self._datetime_field], utc=True)
                except ValueError as e:
                    logger.info(
                        'Unable to convert timestamp in column: %s, error %s',
                        self._datetime_field, e)
            else:
                for column in data_frame.columns[
                        data_frame.columns.str.contains('time', case=False)]:
                    if column.lower() == 'timestamp_desc':
                        continue
                    try:
                        data_frame['timestamp'] = pandas.to_datetime(<commit_msg>Making the data_type more explicit.<commit_after># Copyright 2019 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Timesketch data importer."""
from __future__ import unicode_literals

import codecs
import io
import json
import logging
import math
import os
import time
import uuid

import pandas

from timesketch_api_client import timeline
from timesketch_api_client import definitions
from timesketch_import_client import utils

logger = logging.getLogger('timesketch_importer.importer')


class ImportStreamer(object):
    """Upload object used to stream results to Timesketch."""

    # The number of entries before automatically flushing
    # the streamer.
    DEFAULT_ENTRY_THRESHOLD = 50000

    # Number of bytes in a binary file before automatically
    # chunking it up into smaller pieces.
    DEFAULT_FILESIZE_THRESHOLD = 104857600  # 100 Mb.

    # Define default values.
    DEFAULT_TEXT_ENCODING = 'utf-8'
    DEFAULT_TIMESTAMP_DESC = 'Time Logged'

    def __init__(self):
        """Initialize the upload streamer."""
        self._count = 0
        self._config_helper = None
        self._dict_config_loaded = False
        self._csv_delimiter = None
        self._data_lines = []
        self._data_type = None
        self._datetime_field = None
        self._format_string = None
        self._index = uuid.uuid4().hex
        self._last_response = None
        self._resource_url = ''
        self._sketch = None
        self._timeline_id = None
        self._timeline_name = None

        self._chunk = 1

        self._text_encoding = self.DEFAULT_TEXT_ENCODING
        self._timestamp_desc = self.DEFAULT_TIMESTAMP_DESC
        self._threshold_entry = self.DEFAULT_ENTRY_THRESHOLD
        self._threshold_filesize = self.DEFAULT_FILESIZE_THRESHOLD

    def _fix_dict(self, my_dict):
        """Adjusts a dict with so that it can be uploaded to Timesketch.

        This function will take a dictionary and modify it. Summary of the
        changes are:
          * If "message" is not a key and a format message string has been
              defined, a message field is constructed.
          * If "datetime" is not a key, an attempt to generate it is made.
          * If "timestamp_desc" is not set but defined by the importer it
              is added.
          * All keys that start with an underscore ("_") are removed.

        Args:
            my_dict: a dictionary that may be missing few fields needed
                    for Timesketch.
        """
        if 'message' not in my_dict:
            format_string = (
                self._format_string or utils.get_combined_message_string(
                    mydict=my_dict))
            my_dict['message'] = format_string.format(**my_dict)

        _ = my_dict.setdefault('timestamp_desc', self._timestamp_desc)
        if self._data_type:
            _ = my_dict.setdefault('data_type', self._data_type)

        if 'datetime' not in my_dict:
            date = ''
            if self._datetime_field:
                value = my_dict.get(self._datetime_field)
                if value:
                    date = utils.get_datestring_from_value(value)
            if not date:
                for key in my_dict:
                    key_string = key.lower()
                    if 'time' not in key_string:
                        continue

                    if key_string == 'timestamp_desc':
                        continue

                    value = my_dict[key]
                    date = utils.get_datestring_from_value(value)
                    if date:
                        break

            if date:
                my_dict['datetime'] = date

        # We don't want to include any columns that start with an underscore.
        underscore_columns = [x for x in my_dict if x.startswith('_')]
        if underscore_columns:
            for column in underscore_columns:
                del my_dict[column]

    def _fix_data_frame(self, data_frame):
        """Returns a data frame with added columns for Timesketch upload.

        Args:
            data_frame: a pandas data frame.

        Returns:
            A pandas data frame with added columns needed for Timesketch.
        """
        if 'message' not in data_frame:
            format_string = (
                self._format_string or utils.get_combined_message_string(
                    dataframe=data_frame))
            utils.format_data_frame(data_frame, format_string)

        if 'timestamp_desc' not in data_frame:
            data_frame['timestamp_desc'] = self._timestamp_desc

        if self._data_type and 'data_type' not in data_frame:
            data_frame['data_type'] = self._data_type

        if 'datetime' not in data_frame:
            if self._datetime_field and self._datetime_field in data_frame:
                try:
                    data_frame['timestamp'] = pandas.to_datetime(
                        data_frame[self._datetime_field], utc=True)
                except ValueError as e:
                    logger.info(
                        'Unable to convert timestamp in column: %s, error %s',
                        self._datetime_field, e)
            else:
                for column in data_frame.columns[
                        data_frame.columns.str.contains('time', case=False)]:
                    if column.lower() == 'timestamp_desc':
                        continue
                    try:
                        data_frame['timestamp'] = pandas.to_datetime(
{'fraction_non_alphanumeric': 0.06584533422126031, 'fraction_numerical': 0.004828102888537418, 'mean_word_length': 1.9990014977533699, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5829420', 'n_tokens_mistral': 3224, 'n_tokens_neox': 3039, 'n_words': 1011}