<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

""" The sending process is run according to a schedule in order to send reading data
to the historian, e.g. the PI system.
It’s role is to implement the rules as to what needs to be sent and when,
extract the data from the storage subsystem and stream it to the north
for sending to the external system.
The sending process does not implement the protocol used to send the data,
that is devolved to the translation plugin in order to allow for flexibility
in the translation process.
"""

import resource
import asyncio
import sys
import time
import importlib
import logging
import datetime
import signal

import foglamp.plugins.north.common.common as plugin_common

from foglamp.common.parser import Parser
from foglamp.common.storage_client.storage_client import StorageClient, ReadingsStorageClient
from foglamp.common import logger
from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.common.storage_client import payload_builder
from foglamp.common.statistics import Statistics
from foglamp.common.jqfilter import JQFilter


__author__ = "<NAME>, <NAME>"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"


""" Module information """
_MODULE_NAME = "sending_process"
_MESSAGES_LIST = {
    # Information messages
    "i000001": "Started.",
    "i000002": "Execution completed.",
    "i000003": _MODULE_NAME + " disabled.",
    "i000004": "no data will be sent, the stream id is disabled - stream id |{0}|",
    # Warning / Error messages
    "e000000": "general error",
    "e000001": "cannot start the logger - error details |{0}|",
    "e000002": "cannot complete the operation - error details |{0}|",
    "e000003": "cannot complete the retrieval of the configuration",
    "e000004": "cannot complete the initialization - error details |{0}|",
    "e000005": "cannot load the plugin |{0}|",
    "e000006": "cannot complete the sending operation of a block of data.",
    "e000007": "cannot complete the termination of the sending process.",
    "e000008": "unknown data source, it could be only: readings, statistics or audit.",
    "e000009": "cannot load data into memory - error details |{0}|",
    "e000010": "cannot update statistics.",
    "e000011": "invalid input parameters, the stream id is required and it should be a number "
               "- parameters |{0}|",
    "e000012": "cannot connect to the DB Layer - error details |{0}|",
    "e000013": "cannot validate the stream id - error details |{0}|",
    "e000014": "multiple streams having same id are defined - stream id |{0}|",
    "e000015": "the selected plugin is not a valid north plug in type/name |{0} / {1}|",
    "e000016": "invalid stream id, it is not defined - stream id |{0}|",
    "e000017": "cannot handle command line parameters - error details |{0}|",
    "e000018": "cannot initialize the plugin |{0}|",
    "e000019": "cannot retrieve the starting point for sending operation.",
    "e000020": "cannot update the reached position - error details |{0}|",
    "e000021": "cannot complete the sending operation - error details |{0}|",
    "e000022": "unable to convert in memory data structure related to the statistics data "
               "- error details |{0}|",
    "e000023": "cannot complete the initialization - error details |{0}|",
    "e000024": "unable to log the operation in the Storage Layer - error details |{0}|",
    "e000025": "Required argument '--name' is missing - command line |{0}|",
    "e000026": "Required argument '--port' is missing - command line |{0}|",
    "e000027": "Required argument '--address' is missing - command line |{0}|",

}
""" Messages used for Information, Warning and Error notice """

# LOG configuration
_LOG_LEVEL_DEBUG = 10
_LOG_LEVEL_INFO = 20
_LOG_LEVEL_WARNING = 30

_LOGGER_LEVEL = _LOG_LEVEL_WARNING
_LOGGER_DESTINATION = logger.SYSLOG

_LOGGER = logger.setup(__name__, destination=_LOGGER_DESTINATION, level=_LOGGER_LEVEL)

_event_loop = ""
_log_performance = False
""" Enable/Disable performance logging, enabled using a command line parameter"""


class PluginInitialiseFailed(RuntimeError):
    """ PluginInitializeFailed """
    pass


class UnknownDataSource(RuntimeError):
    """ the data source could be only one among: readings, statistics or audit """
    pass


class InvalidCommandLineParameters(RuntimeError):
    """ Invalid command line parameters, the stream id is the only required """
    pass

def apply_date_format(in_data):
    """ This routine adds the default UTC zone format to the input date time string
    If a timezone (strting with + or -) is found, all the following chars
    are replaced by +00, otherwise +00 is added.

    Note: if the input zone is +02:00 no date conversion is done,
          at the time being this routine expects UTC date time values.

    Examples:
        2018-03-22 17:17:17.166347       ==> 2018-03-22 17:17:17.166347+00
        2018-03-22 17:17:17.166347+00:00 ==> 2018-03-22 17:17:17.166347+00
        2018-03-22 17:17:17.166347+00    ==> 2018-03-22 17:17:17.166347+00
        2018-03-22 17:17:17.166347+02:00 ==> 2018-03-22 17:17:17.166347+00

    Args:
        the date time string to format
    Returns:
        the newly formatted datetime string
    """

    # Look for timezone start with '-' a the end of the date (-XY:WZ)
    zoneIndex = in_data.rfind("-")
    # If index is less than 10 we don't have the trailing zone with -
    if (zoneIndex < 10):
        #  Look for timezone start with '+' (+XY:ZW)
        zoneIndex = in_data.rfind("+")

    if zoneIndex == -1:
        # Just add +00
        timestamp = in_data + "+00"
    else:
        # Remove everything after - or + and add +00
        timestamp = in_data[:zoneIndex] + "+00"

    return timestamp

def _performance_log(func):
    """ Logs information for performance measurement """
    def wrapper(*arg):
        """ wrapper """
        start = datetime.datetime.now()
        # Code execution
        res = func(*arg)

        if _log_performance:
            usage = resource.getrusage(resource.RUSAGE_SELF)
            process_memory = usage.ru_maxrss / 1000
            delta = datetime.datetime.now() - start
            delta_milliseconds = int(delta.total_seconds() * 1000)
            _LOGGER.info("PERFORMANCE - {0} - milliseconds |{1:>8,}| - memory MB |{2:>8,}|"
                         .format(func.__name__,
                                 delta_milliseconds,
                                 process_memory))
        return res
    return wrapper

<commit_msg>FOGL-1154: fixed Python code style<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

""" The sending process is run according to a schedule in order to send reading data
to the historian, e.g. the PI system.
It’s role is to implement the rules as to what needs to be sent and when,
extract the data from the storage subsystem and stream it to the north
for sending to the external system.
The sending process does not implement the protocol used to send the data,
that is devolved to the translation plugin in order to allow for flexibility
in the translation process.
"""

import resource
import asyncio
import sys
import time
import importlib
import logging
import datetime
import signal

import foglamp.plugins.north.common.common as plugin_common

from foglamp.common.parser import Parser
from foglamp.common.storage_client.storage_client import StorageClient, ReadingsStorageClient
from foglamp.common import logger
from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.common.storage_client import payload_builder
from foglamp.common.statistics import Statistics
from foglamp.common.jqfilter import JQFilter


__author__ = "<NAME>, <NAME>"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"


""" Module information """
_MODULE_NAME = "sending_process"
_MESSAGES_LIST = {
    # Information messages
    "i000001": "Started.",
    "i000002": "Execution completed.",
    "i000003": _MODULE_NAME + " disabled.",
    "i000004": "no data will be sent, the stream id is disabled - stream id |{0}|",
    # Warning / Error messages
    "e000000": "general error",
    "e000001": "cannot start the logger - error details |{0}|",
    "e000002": "cannot complete the operation - error details |{0}|",
    "e000003": "cannot complete the retrieval of the configuration",
    "e000004": "cannot complete the initialization - error details |{0}|",
    "e000005": "cannot load the plugin |{0}|",
    "e000006": "cannot complete the sending operation of a block of data.",
    "e000007": "cannot complete the termination of the sending process.",
    "e000008": "unknown data source, it could be only: readings, statistics or audit.",
    "e000009": "cannot load data into memory - error details |{0}|",
    "e000010": "cannot update statistics.",
    "e000011": "invalid input parameters, the stream id is required and it should be a number "
               "- parameters |{0}|",
    "e000012": "cannot connect to the DB Layer - error details |{0}|",
    "e000013": "cannot validate the stream id - error details |{0}|",
    "e000014": "multiple streams having same id are defined - stream id |{0}|",
    "e000015": "the selected plugin is not a valid north plug in type/name |{0} / {1}|",
    "e000016": "invalid stream id, it is not defined - stream id |{0}|",
    "e000017": "cannot handle command line parameters - error details |{0}|",
    "e000018": "cannot initialize the plugin |{0}|",
    "e000019": "cannot retrieve the starting point for sending operation.",
    "e000020": "cannot update the reached position - error details |{0}|",
    "e000021": "cannot complete the sending operation - error details |{0}|",
    "e000022": "unable to convert in memory data structure related to the statistics data "
               "- error details |{0}|",
    "e000023": "cannot complete the initialization - error details |{0}|",
    "e000024": "unable to log the operation in the Storage Layer - error details |{0}|",
    "e000025": "Required argument '--name' is missing - command line |{0}|",
    "e000026": "Required argument '--port' is missing - command line |{0}|",
    "e000027": "Required argument '--address' is missing - command line |{0}|",

}
""" Messages used for Information, Warning and Error notice """

# LOG configuration
_LOG_LEVEL_DEBUG = 10
_LOG_LEVEL_INFO = 20
_LOG_LEVEL_WARNING = 30

_LOGGER_LEVEL = _LOG_LEVEL_WARNING
_LOGGER_DESTINATION = logger.SYSLOG

_LOGGER = logger.setup(__name__, destination=_LOGGER_DESTINATION, level=_LOGGER_LEVEL)

_event_loop = ""
_log_performance = False
""" Enable/Disable performance logging, enabled using a command line parameter"""


class PluginInitialiseFailed(RuntimeError):
    """ PluginInitializeFailed """
    pass


class UnknownDataSource(RuntimeError):
    """ the data source could be only one among: readings, statistics or audit """
    pass


class InvalidCommandLineParameters(RuntimeError):
    """ Invalid command line parameters, the stream id is the only required """
    pass

def apply_date_format(in_data):
    """ This routine adds the default UTC zone format to the input date time string
    If a timezone (strting with + or -) is found, all the following chars
    are replaced by +00, otherwise +00 is added.

    Note: if the input zone is +02:00 no date conversion is done,
          at the time being this routine expects UTC date time values.

    Examples:
        2018-03-22 17:17:17.166347       ==> 2018-03-22 17:17:17.166347+00
        2018-03-22 17:17:17.166347+00:00 ==> 2018-03-22 17:17:17.166347+00
        2018-03-22 17:17:17.166347+00    ==> 2018-03-22 17:17:17.166347+00
        2018-03-22 17:17:17.166347+02:00 ==> 2018-03-22 17:17:17.166347+00

    Args:
        the date time string to format
    Returns:
        the newly formatted datetime string
    """

    # Look for timezone start with '-' a the end of the date (-XY:WZ)
    zone_index = in_data.rfind("-")
    # If index is less than 10 we don't have the trailing zone with -
    if (zone_index < 10):
        #  Look for timezone start with '+' (+XY:ZW)
        zone_index = in_data.rfind("+")

    if zone_index == -1:
        # Just add +00
        timestamp = in_data + "+00"
    else:
        # Remove everything after - or + and add +00
        timestamp = in_data[:zone_index] + "+00"

    return timestamp

def _performance_log(func):
    """ Logs information for performance measurement """
    def wrapper(*arg):
        """ wrapper """
        start = datetime.datetime.now()
        # Code execution
        res = func(*arg)

        if _log_performance:
            usage = resource.getrusage(resource.RUSAGE_SELF)
            process_memory = usage.ru_maxrss / 1000
            delta = datetime.datetime.now() - start
            delta_milliseconds = int(delta.total_seconds() * 1000)
            _LOGGER.info("PERFORMANCE - {0} - milliseconds |{1:>8,}| - memory MB |{2:>8,}|"
                         .format(func.__name__,
                                 delta_milliseconds,
                                 process_memory))
        return res
    return wrapper


{'fraction_non_alphanumeric': 0.1088679527441304, 'fraction_numerical': 0.06624794377149694, 'mean_word_length': 3.289608723540731, 'pattern_counts': {'":': 64, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2970581', 'n_tokens_mistral': 4473, 'n_tokens_neox': 3829, 'n_words': 1468}