<commit_before>#!/usr/bin/pcp python
#
# Copyright (C) 2015 <NAME> <<EMAIL>>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# pylint: disable=fixme, line-too-long, bad-whitespace, invalid-name
# pylint: disable=superfluous-parens
""" Performance Metrics Reporter """

from collections import OrderedDict
from datetime import datetime
try:
    import ConfigParser
except ImportError:
    import configparser as ConfigParser
import time
import copy
import sys
import os
import re

from pcp import pmapi, pmgui, pmi
from cpmapi import PM_CONTEXT_ARCHIVE, PM_CONTEXT_HOST, PM_CONTEXT_LOCAL, PM_MODE_FORW, PM_MODE_INTERP, PM_ERR_TYPE, PM_IN_NULL, PM_SEM_COUNTER, PM_TIME_MSEC, PM_TIME_SEC, PM_XTB_SET, PM_DEBUG_APPL1
from cpmapi import PM_TYPE_32, PM_TYPE_U32, PM_TYPE_64, PM_TYPE_U64, PM_TYPE_FLOAT, PM_TYPE_DOUBLE, PM_TYPE_STRING
from cpmgui import PM_REC_ON, PM_REC_OFF, PM_REC_SETARG

if sys.version_info[0] >= 3:
    long = int

# Default config
DEFAULT_CONFIG = "./pmrep.conf"

# Default field separators, time formats, missing/truncated values
CSVSEP  = ","
CSVTIME = "%Y-%m-%d %H:%M:%S"
OUTSEP  = "  "
OUTTIME = "%H:%M:%S"
ZBXPORT = 10051
ZBXPRFX = "pcp."
NO_VAL  = "N/A"
TRUNC   = "xxx"

# Output targets
OUTPUT_ARCHIVE = "archive"
OUTPUT_CSV     = "csv"
OUTPUT_STDOUT  = "stdout"
OUTPUT_ZABBIX  = "zabbix"

class PMReporter(object):
    """ Report PCP metrics """

    def __init__(self):
        """ Construct object, prepare for command line handling """
        self.context = None
        self.version = 1
        self.check = 0
        self.format = None # output format
        self.opts = self.options()

        # Configuration directives
        self.keys = ('source', 'output', 'derived', 'header', 'unitinfo',
                     'globals', 'timestamp', 'samples', 'interval', 'runtime',
                     'delay', 'raw', 'width', 'decimals', 'delimiter',
                     'extheader', 'repeat_header', 'timefmt', 'interpol',
                     'count_scale', 'space_scale', 'time_scale', 'version',
                     'zabbix_server', 'zabbix_port', 'zabbix_host', 'zabbix_interval')

        # Command line switches without arguments
        self.argless = ('-C', '-L', '-H', '-U', '-G', '-p', '-d', '--delay', '-r', '--raw', '-x', '--extended-header', '-u')
        self.arghelp = ('-?', '--help', '-V', '--version')

        # The order of preference for parameters (as present):
        # 1 - command line parameters
        # 2 - parameters from configuration file(s)
        # 3 - built-in defaults defined below
        self.config = self.set_config_file()
        self.source = "local:"
        self.output = OUTPUT_STDOUT
        self.archive = None # output archive
        self.log = None # pmi handle
        self.derived = None
        self.header = 1
        self.unitinfo = 1
        self.globals = 1
        self.timestamp = 0
        self.samples = None # forever
        self.interval = pmapi.timeval(1) # 1 sec
        self.opts.pmSetOptionInterval(str(1))
        self.runtime = -1
        self.delay = 0
        self.raw = 0
        self.width = 0
        self.decimals = 3 # .3f
        self.delimiter = None
        self.extheader = 0
        self.repeat_header = 0
        self.timefmt = None
        self.interpol = 1
        self.count_scale = None
        self.space_scale = None
        self.time_scale = None

        # Performance metrics store
        # key - metric name
        # values - 0:label, 1:instance(s), 2:unit/scale, 3:rawness, 4:width
        self.metrics = OrderedDict()

        # Corresponding config file metric specifiers
        self.metricspec = ('label', 'instance', 'unit', 'raw', 'width', 'formula')

        self.prevvals = None
        self.currvals = None
        self.ptstamp = 0
        self.ctstamp = 0
        self.pmids = []
        self.descs = []
        self.insts = []

        # Zabbix integration
        self.zabbix_server = None
        self.zabbix_port = ZBXPORT
        self.zabbix_host = None
        self.zabbix_interval = None
        self.zabbix_prevsend = None
        self.zabbix_metrics = []

    def set_config_file(self):
        """ Set configuration file """
        config = DEFAULT_CONFIG

        # Possibly override the built-in default config file before
        # parsing the rest of the command line parameters
        args = iter(sys.argv[1:])
        for arg in args:
            if arg in self.arghelp:
                return None
            if arg == '-c' or arg == '--config':
                try:
                    config = args.next()
                    if not os.path.isfile(config) or not os.access(config, os.R_OK):
                        raise IOError("Failed to read configuration file '%s'." % config)
                except StopIteration:<commit_msg>pmrep: fix use of list next operator in python3<commit_after>#!/usr/bin/pcp python
#
# Copyright (C) 2015 <NAME> <<EMAIL>>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# pylint: disable=fixme, line-too-long, bad-whitespace, invalid-name
# pylint: disable=superfluous-parens
""" Performance Metrics Reporter """

from collections import OrderedDict
from datetime import datetime
try:
    import ConfigParser
except ImportError:
    import configparser as ConfigParser
import time
import copy
import sys
import os
import re

from pcp import pmapi, pmgui, pmi
from cpmapi import PM_CONTEXT_ARCHIVE, PM_CONTEXT_HOST, PM_CONTEXT_LOCAL, PM_MODE_FORW, PM_MODE_INTERP, PM_ERR_TYPE, PM_IN_NULL, PM_SEM_COUNTER, PM_TIME_MSEC, PM_TIME_SEC, PM_XTB_SET, PM_DEBUG_APPL1
from cpmapi import PM_TYPE_32, PM_TYPE_U32, PM_TYPE_64, PM_TYPE_U64, PM_TYPE_FLOAT, PM_TYPE_DOUBLE, PM_TYPE_STRING
from cpmgui import PM_REC_ON, PM_REC_OFF, PM_REC_SETARG

if sys.version_info[0] >= 3:
    long = int

# Default config
DEFAULT_CONFIG = "./pmrep.conf"

# Default field separators, time formats, missing/truncated values
CSVSEP  = ","
CSVTIME = "%Y-%m-%d %H:%M:%S"
OUTSEP  = "  "
OUTTIME = "%H:%M:%S"
ZBXPORT = 10051
ZBXPRFX = "pcp."
NO_VAL  = "N/A"
TRUNC   = "xxx"

# Output targets
OUTPUT_ARCHIVE = "archive"
OUTPUT_CSV     = "csv"
OUTPUT_STDOUT  = "stdout"
OUTPUT_ZABBIX  = "zabbix"

class PMReporter(object):
    """ Report PCP metrics """

    def __init__(self):
        """ Construct object, prepare for command line handling """
        self.context = None
        self.version = 1
        self.check = 0
        self.format = None # output format
        self.opts = self.options()

        # Configuration directives
        self.keys = ('source', 'output', 'derived', 'header', 'unitinfo',
                     'globals', 'timestamp', 'samples', 'interval', 'runtime',
                     'delay', 'raw', 'width', 'decimals', 'delimiter',
                     'extheader', 'repeat_header', 'timefmt', 'interpol',
                     'count_scale', 'space_scale', 'time_scale', 'version',
                     'zabbix_server', 'zabbix_port', 'zabbix_host', 'zabbix_interval')

        # Command line switches without arguments
        self.argless = ('-C', '-L', '-H', '-U', '-G', '-p', '-d', '--delay', '-r', '--raw', '-x', '--extended-header', '-u')
        self.arghelp = ('-?', '--help', '-V', '--version')

        # The order of preference for parameters (as present):
        # 1 - command line parameters
        # 2 - parameters from configuration file(s)
        # 3 - built-in defaults defined below
        self.config = self.set_config_file()
        self.source = "local:"
        self.output = OUTPUT_STDOUT
        self.archive = None # output archive
        self.log = None # pmi handle
        self.derived = None
        self.header = 1
        self.unitinfo = 1
        self.globals = 1
        self.timestamp = 0
        self.samples = None # forever
        self.interval = pmapi.timeval(1) # 1 sec
        self.opts.pmSetOptionInterval(str(1))
        self.runtime = -1
        self.delay = 0
        self.raw = 0
        self.width = 0
        self.decimals = 3 # .3f
        self.delimiter = None
        self.extheader = 0
        self.repeat_header = 0
        self.timefmt = None
        self.interpol = 1
        self.count_scale = None
        self.space_scale = None
        self.time_scale = None

        # Performance metrics store
        # key - metric name
        # values - 0:label, 1:instance(s), 2:unit/scale, 3:rawness, 4:width
        self.metrics = OrderedDict()

        # Corresponding config file metric specifiers
        self.metricspec = ('label', 'instance', 'unit', 'raw', 'width', 'formula')

        self.prevvals = None
        self.currvals = None
        self.ptstamp = 0
        self.ctstamp = 0
        self.pmids = []
        self.descs = []
        self.insts = []

        # Zabbix integration
        self.zabbix_server = None
        self.zabbix_port = ZBXPORT
        self.zabbix_host = None
        self.zabbix_interval = None
        self.zabbix_prevsend = None
        self.zabbix_metrics = []

    def set_config_file(self):
        """ Set configuration file """
        config = DEFAULT_CONFIG

        # Possibly override the built-in default config file before
        # parsing the rest of the command line parameters
        args = iter(sys.argv[1:])
        for arg in args:
            if arg in self.arghelp:
                return None
            if arg == '-c' or arg == '--config':
                try:
                    config = next(args)
                    if not os.path.isfile(config) or not os.access(config, os.R_OK):
                        raise IOError("Failed to read configuration file '%s'." % config)
                except StopIteration:
{'fraction_non_alphanumeric': 0.10983154087751024, 'fraction_numerical': 0.009612639192919006, 'mean_word_length': 2.38749194068343, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4849117', 'n_tokens_mistral': 3296, 'n_tokens_neox': 3153, 'n_words': 951}