<commit_before>#!/usr/bin/pcp python
#
# Copyright (C) 2014-2015 Red Hat.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
""" Relay PCP metrics to a graphite server """

import re
import sys
import time

from pcp import pmapi
import cpmapi as c_api

class GraphiteRelay(object):
    """ Sends a periodic report to graphite about all instances of named
        metrics.  Knows about some of the default PCP arguments.
    """

    def describe_source(self):
        """ Return a string describing our context; apprx. inverse of
            pmapi.fromOptions
        """

        ctxtype = self.context.type
        if ctxtype == c_api.PM_CONTEXT_ARCHIVE:
            return "archive " + ", ".join(self.opts.pmGetOptionArchives())
        elif ctxtype == c_api.PM_CONTEXT_HOST:
            hosts = self.opts.pmGetOptionHosts()
            if hosts is None: # pmapi.py idiosyncracy; it has already defaulted to this
                hosts = ["local:"]
            return "host " + ", ".join(hosts)
        elif ctxtype == c_api.PM_CONTEXT_LOCAL:
            hosts = ["local:"]
            return "host " + ", ".join(hosts)
        else:
            raise pmapi.pmUsageErr

    def __init__(self):
        """ Construct object, parse command line """
        self.context = None
        self.socket = None
        self.sample_count = 0
        self.opts = pmapi.pmOptions()
        self.opts.pmSetShortOptions("a:O:s:T:g:p:P:r:u:m:t:h:t:D:LV?")
        self.opts.pmSetShortUsage("[options] metricname ...")
        self.opts.pmSetOptionCallback(self.option)
        self.opts.pmSetOverrideCallback(self.option_override)
        self.opts.pmSetLongOptionText("""
Description: Periodically, relay raw values of all instances of a
given hierarchies of PCP metrics to a graphite/carbon server on the
network.""")
        self.opts.pmSetLongOptionHeader("Options")
        self.opts.pmSetLongOptionVersion() # -V
        self.opts.pmSetLongOptionArchive() # -a FILE
        self.opts.pmSetLongOptionOrigin() # -O TIME
        self.opts.pmSetLongOptionSamples() # -s NUMBER
        self.opts.pmSetLongOptionFinish() # -T NUMBER
        self.opts.pmSetLongOptionDebug() # -D stuff
        self.opts.pmSetLongOptionHost() # -h HOST
        self.opts.pmSetLongOptionLocalPMDA() # -L
        self.opts.pmSetLongOptionInterval() # -t NUMBER
        self.opts.pmSetLongOption("graphite-host", 1, 'g', '',
                                  "graphite server host " +
                                  "(default \"localhost\")")
        self.opts.pmSetLongOption("pickled-port", 1, 'p', '',
                                  "graphite pickled port (default 2004)")
        self.opts.pmSetLongOption("text-port", 1, 'P', '',
                                  "graphite plaintext port (usually 2003)")
        self.opts.pmSetLongOption("units", 1, 'u', '',
                                  "rescale units " +
                                  "(e.g. \"MB\", will omit incompatible units)")
        self.opts.pmSetLongOption("prefix", 1, 'm', '',
                                  "prefix for metric names (default \"pcp.\")")
        self.opts.pmSetLongOption("pickle-protocol", 1, 'r', 'PROTOCOL',
                                  "graphite pickle protocol (default 0)")
        self.opts.pmSetLongOptionHelp()
        self.graphite_host = "localhost"
        self.graphite_port = 2004
        self.pickle = True
        self.pickle_protocol = 0
        self.prefix = "pcp."
        self.unitsstr = None
        self.units = None # pass verbatim by default
        self.units_mult = None # pass verbatim by default

        # now actually parse
        self.context = pmapi.pmContext.fromOptions(self.opts, sys.argv)
        self.interval = self.opts.pmGetOptionInterval() or pmapi.timeval(60, 0)
        if self.unitsstr is not None:
            units = self.context.pmParseUnitsStr(self.unitsstr)
            (self.units, self.units_mult) = units
        self.metrics = []
        self.pmids = []
        self.descs = []
        metrics = self.opts.pmGetNonOptionsFromList(sys.argv)
        if metrics:
            for m in metrics:
                try:
                    self.context.pmTraversePMNS(m, self.handle_candidate_metric)
                except pmapi.pmErr as error:
                    sys.stderr.write("Excluding metric %s (%s)\n" %
                                     (m, str(error)))
            sys.stderr.flush()

        if len(self.metrics) == 0:
            sys.stderr.write("No acceptable metrics specified.\n")
            raise pmapi.pmUsageErr()

        # Report what we're about to do
        print("Relaying %d %smetric(s) with prefix %s from %s "
              "in %s mode to %s:%d every %f s" %
              (len(self.metrics),
               "rescaled " if self.units else "",
               self.prefix,
               self.describe_source(),
               "pickled" if self.pickle else "text",
               self.graphite_host, self.graphite_port,
               self.interval))
        sys.stdout.flush()

    def option_override(self, opt):
        if (opt == 'p') or (opt == 'g'):
            return 1
        return 0

    def option(self, opt, optarg, index):
        # need only handle the non-common options
        if opt == 'g':
            self.graphite_host = optarg
        elif opt == 'p':
            self.graphite_port = int(optarg if optarg else "2004")
            self.pickle = True
        elif opt == 'r':
            self.pickle_protocol = int(optarg if optarg else "0")
        elif opt == 'P':
            self.graphite_port = int(optarg if optarg else "2003")
            self.pickle = False
        elif opt == 'u':
            self.unitsstr = optarg
        elif opt == 'm':
            self.prefix = optarg
        else:
            raise pmapi.pmUsageErr()


    # Check the given metric name (a leaf in the PMNS) for
    # acceptability for graphite: it needs to be numeric, and
    # convertable to the given unit (if specified).
    #
    # Print an error message here if needed; can't throw an exception
    # through the pmapi pmTraversePMNS wrapper.
    def handle_candidate_metric(self, name):
        try:
            pmid = self.context.pmLookupName(name)[0]
            desc = self.context.pmLookupDescs(pmid)[0]
        except pmapi.pmErr as err:
            sys.stderr.write("Excluding metric %s (%s)\n" % (name, str(err)))
            return

        # reject non-numeric types (future pmExtractValue failure)
        types = desc.contents.type
        if not ((types == c_api.PM_TYPE_32) or
                (types == c_api.PM_TYPE_U32) or
                (types == c_api.PM_TYPE_64) or
                (types == c_api.PM_TYPE_U64) or
                (types == c_api.PM_TYPE_FLOAT) or
                (types == c_api.PM_TYPE_DOUBLE)):
            sys.stderr.write("Excluding metric %s (%s)\n" %
                             (name, "need numeric type"))
            return

        # reject dimensionally incompatible (future pmConvScale failure)
        if self.units is not None:
            units = desc.contents.units
            if (units.dimSpace != self.units.dimSpace or
                    units.dimTime != self.units.dimTime or
                    units.dimCount != self.units.dimCount):
                sys.stderr.write("Excluding metric %s (%s)\n" %
                                 (name, "incompatible dimensions"))
                return

        self.metrics.append(name)
        self.pmids.append(pmid)
        self.descs.append(desc)


    # Convert a python list of pmids (numbers) to a ctypes LP_c_uint
    # (a C array of uints).
    def convert_pmids_to_ctypes(self, pmids):
        from ctypes import c_uint
        pmidA = (c_uint * len(pmids))()
        for i, p in enumerate(pmids):
            pmidA[i] = c_uint(p)
        return pmidA

    def send(self, timestamp, miv_tuples):
        import socket
        try:
            # reuse socket
            if self.socket is None:
                self.socket = socket.create_connection((self.graphite_host,
                                                        self.graphite_port))
            if self.pickle:
                try:
                    import cPickle as pickle
                except:
                    import pickle
                import struct
                pickled_input = []
                for (metric, value) in miv_tuples:
                    pickled_input.append((metric, (timestamp, value)))
                pickled_output = pickle.dumps(pickled_input, protocol=self.pickle_protocol)
                header = struct.pack("!L", len(pickled_output))
                msg = header + pickled_output<commit_msg>Revert "pcp2graphite: workaround for cPickle and Python 3"<commit_after>#!/usr/bin/pcp python
#
# Copyright (C) 2014-2015 Red Hat.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
""" Relay PCP metrics to a graphite server """

import re
import sys
import time

from pcp import pmapi
import cpmapi as c_api

class GraphiteRelay(object):
    """ Sends a periodic report to graphite about all instances of named
        metrics.  Knows about some of the default PCP arguments.
    """

    def describe_source(self):
        """ Return a string describing our context; apprx. inverse of
            pmapi.fromOptions
        """

        ctxtype = self.context.type
        if ctxtype == c_api.PM_CONTEXT_ARCHIVE:
            return "archive " + ", ".join(self.opts.pmGetOptionArchives())
        elif ctxtype == c_api.PM_CONTEXT_HOST:
            hosts = self.opts.pmGetOptionHosts()
            if hosts is None: # pmapi.py idiosyncracy; it has already defaulted to this
                hosts = ["local:"]
            return "host " + ", ".join(hosts)
        elif ctxtype == c_api.PM_CONTEXT_LOCAL:
            hosts = ["local:"]
            return "host " + ", ".join(hosts)
        else:
            raise pmapi.pmUsageErr

    def __init__(self):
        """ Construct object, parse command line """
        self.context = None
        self.socket = None
        self.sample_count = 0
        self.opts = pmapi.pmOptions()
        self.opts.pmSetShortOptions("a:O:s:T:g:p:P:r:u:m:t:h:t:D:LV?")
        self.opts.pmSetShortUsage("[options] metricname ...")
        self.opts.pmSetOptionCallback(self.option)
        self.opts.pmSetOverrideCallback(self.option_override)
        self.opts.pmSetLongOptionText("""
Description: Periodically, relay raw values of all instances of a
given hierarchies of PCP metrics to a graphite/carbon server on the
network.""")
        self.opts.pmSetLongOptionHeader("Options")
        self.opts.pmSetLongOptionVersion() # -V
        self.opts.pmSetLongOptionArchive() # -a FILE
        self.opts.pmSetLongOptionOrigin() # -O TIME
        self.opts.pmSetLongOptionSamples() # -s NUMBER
        self.opts.pmSetLongOptionFinish() # -T NUMBER
        self.opts.pmSetLongOptionDebug() # -D stuff
        self.opts.pmSetLongOptionHost() # -h HOST
        self.opts.pmSetLongOptionLocalPMDA() # -L
        self.opts.pmSetLongOptionInterval() # -t NUMBER
        self.opts.pmSetLongOption("graphite-host", 1, 'g', '',
                                  "graphite server host " +
                                  "(default \"localhost\")")
        self.opts.pmSetLongOption("pickled-port", 1, 'p', '',
                                  "graphite pickled port (default 2004)")
        self.opts.pmSetLongOption("text-port", 1, 'P', '',
                                  "graphite plaintext port (usually 2003)")
        self.opts.pmSetLongOption("units", 1, 'u', '',
                                  "rescale units " +
                                  "(e.g. \"MB\", will omit incompatible units)")
        self.opts.pmSetLongOption("prefix", 1, 'm', '',
                                  "prefix for metric names (default \"pcp.\")")
        self.opts.pmSetLongOption("pickle-protocol", 1, 'r', 'PROTOCOL',
                                  "graphite pickle protocol (default 0)")
        self.opts.pmSetLongOptionHelp()
        self.graphite_host = "localhost"
        self.graphite_port = 2004
        self.pickle = True
        self.pickle_protocol = 0
        self.prefix = "pcp."
        self.unitsstr = None
        self.units = None # pass verbatim by default
        self.units_mult = None # pass verbatim by default

        # now actually parse
        self.context = pmapi.pmContext.fromOptions(self.opts, sys.argv)
        self.interval = self.opts.pmGetOptionInterval() or pmapi.timeval(60, 0)
        if self.unitsstr is not None:
            units = self.context.pmParseUnitsStr(self.unitsstr)
            (self.units, self.units_mult) = units
        self.metrics = []
        self.pmids = []
        self.descs = []
        metrics = self.opts.pmGetNonOptionsFromList(sys.argv)
        if metrics:
            for m in metrics:
                try:
                    self.context.pmTraversePMNS(m, self.handle_candidate_metric)
                except pmapi.pmErr as error:
                    sys.stderr.write("Excluding metric %s (%s)\n" %
                                     (m, str(error)))
            sys.stderr.flush()

        if len(self.metrics) == 0:
            sys.stderr.write("No acceptable metrics specified.\n")
            raise pmapi.pmUsageErr()

        # Report what we're about to do
        print("Relaying %d %smetric(s) with prefix %s from %s "
              "in %s mode to %s:%d every %f s" %
              (len(self.metrics),
               "rescaled " if self.units else "",
               self.prefix,
               self.describe_source(),
               "pickled" if self.pickle else "text",
               self.graphite_host, self.graphite_port,
               self.interval))
        sys.stdout.flush()

    def option_override(self, opt):
        if (opt == 'p') or (opt == 'g'):
            return 1
        return 0

    def option(self, opt, optarg, index):
        # need only handle the non-common options
        if opt == 'g':
            self.graphite_host = optarg
        elif opt == 'p':
            self.graphite_port = int(optarg if optarg else "2004")
            self.pickle = True
        elif opt == 'r':
            self.pickle_protocol = int(optarg if optarg else "0")
        elif opt == 'P':
            self.graphite_port = int(optarg if optarg else "2003")
            self.pickle = False
        elif opt == 'u':
            self.unitsstr = optarg
        elif opt == 'm':
            self.prefix = optarg
        else:
            raise pmapi.pmUsageErr()


    # Check the given metric name (a leaf in the PMNS) for
    # acceptability for graphite: it needs to be numeric, and
    # convertable to the given unit (if specified).
    #
    # Print an error message here if needed; can't throw an exception
    # through the pmapi pmTraversePMNS wrapper.
    def handle_candidate_metric(self, name):
        try:
            pmid = self.context.pmLookupName(name)[0]
            desc = self.context.pmLookupDescs(pmid)[0]
        except pmapi.pmErr as err:
            sys.stderr.write("Excluding metric %s (%s)\n" % (name, str(err)))
            return

        # reject non-numeric types (future pmExtractValue failure)
        types = desc.contents.type
        if not ((types == c_api.PM_TYPE_32) or
                (types == c_api.PM_TYPE_U32) or
                (types == c_api.PM_TYPE_64) or
                (types == c_api.PM_TYPE_U64) or
                (types == c_api.PM_TYPE_FLOAT) or
                (types == c_api.PM_TYPE_DOUBLE)):
            sys.stderr.write("Excluding metric %s (%s)\n" %
                             (name, "need numeric type"))
            return

        # reject dimensionally incompatible (future pmConvScale failure)
        if self.units is not None:
            units = desc.contents.units
            if (units.dimSpace != self.units.dimSpace or
                    units.dimTime != self.units.dimTime or
                    units.dimCount != self.units.dimCount):
                sys.stderr.write("Excluding metric %s (%s)\n" %
                                 (name, "incompatible dimensions"))
                return

        self.metrics.append(name)
        self.pmids.append(pmid)
        self.descs.append(desc)


    # Convert a python list of pmids (numbers) to a ctypes LP_c_uint
    # (a C array of uints).
    def convert_pmids_to_ctypes(self, pmids):
        from ctypes import c_uint
        pmidA = (c_uint * len(pmids))()
        for i, p in enumerate(pmids):
            pmidA[i] = c_uint(p)
        return pmidA

    def send(self, timestamp, miv_tuples):
        import socket
        try:
            # reuse socket
            if self.socket is None:
                self.socket = socket.create_connection((self.graphite_host,
                                                        self.graphite_port))
            if self.pickle:
                import cPickle
                import struct
                pickled_input = []
                for (metric, value) in miv_tuples:
                    pickled_input.append((metric, (timestamp, value)))
                pickled_output = cPickle.dumps(pickled_input, protocol=self.pickle_protocol)
                header = struct.pack("!L", len(pickled_output))
                msg = header + pickled_output
{'fraction_non_alphanumeric': 0.10175361442471552, 'fraction_numerical': 0.0061568907701610685, 'mean_word_length': 1.8309990662931839, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '500747', 'n_tokens_mistral': 5333, 'n_tokens_neox': 5069, 'n_words': 1414}