<commit_before>/*
 * Copyright (c) 2013-2014 Red Hat.
 * 
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 */


#include "pmmgr.h"
#include "impl.h"

#include <sys/stat.h>
#include <cassert>
#include <cstdlib>
#include <fstream>
#include <iostream>

extern "C" {
#include <fcntl.h>
#include <unistd.h>
#include <glob.h>
#include <sys/wait.h>
#ifdef HAVE_PTHREAD_H
#include <pthread.h>
#endif
#ifdef IS_LINUX
#include <sys/syscall.h>
#endif
}


using namespace std;

// ------------------------------------------------------------------------


int quit;


// ------------------------------------------------------------------------


// Create a string that is safe to pass to system(3), i.e., sh -c,
// by quoting metacharacters.  This transform generally should be
// applied only once.
string
sh_quote(const string& input)
{
  string output;
  for (unsigned i=0; i<input.length(); i++)
    {
      char c = input[i];
      if ((ispunct(c) || isspace(c)) && // quite aggressive
          (c != ':' && c != '.' && c != '_' && c != '/' && c != '-')) // safe & popular punctuation
        output += '\\';
      output += c;
    }

  return output;
}


// Print a string to cout/cerr progress reports, similar to the
// stuff produced by __pmNotifyErr
ostream&
timestamp(ostream &o)
{
  time_t now;
  time (&now);
  char *now2 = ctime (&now);
  if (now2)
    now2[19] = '\0'; // overwrite \n

  return o << "[" << (now2 ? now2 : "") << "] " << pmProgname << "(" 
           << getpid()
#ifdef HAVE_PTHREAD_H
#ifdef IS_LINUX
           << "/" << syscall(SYS_gettid)
#else
           << "/" << pthread_self()
#endif
#endif
           << "): ";
}


extern "C" int
pmValue_compare (const void* a, const void* b)
{
  return ((pmValue *)a)->inst - ((pmValue *)b)->inst;
}


extern "C" void *
pmmgr_daemon_poll_thread (void* a)
{
  pmmgr_daemon* d = (pmmgr_daemon*) a;
  d->poll();
  return 0;
}



// ------------------------------------------------------------------------


pmmgr_configurable::pmmgr_configurable(const string& dir):
  config_directory(dir)
{
}


vector<string>
pmmgr_configurable::get_config_multi(const string& file) const
{
  vector<string> lines;

  string complete_filename = config_directory + (char)__pmPathSeparator() + file;
  ifstream f (complete_filename.c_str());
  while (f.good()) {
    string line;
    getline(f, line);
    if (! f.good())
      break;
    if (line != "")
      lines.push_back(line);
  }

  return lines;
}


bool
pmmgr_configurable::get_config_exists(const string& file) const
{
  string complete_filename = config_directory + (char)__pmPathSeparator() + file;
  ifstream f (complete_filename.c_str());
  return (f.good());
}


string
pmmgr_configurable::get_config_single(const string& file) const
{
  vector<string> lines = get_config_multi (file);
  if (lines.size() == 1)
    return lines[0];
  else
    return "";
}

ostream&
pmmgr_configurable::timestamp(ostream& o)
{
  return ::timestamp(o) << config_directory << ": ";
}



// ------------------------------------------------------------------------


pmMetricSpec*
pmmgr_job_spec::parse_metric_spec (const string& spec)
{
  if (parsed_metric_cache.find(spec) != parsed_metric_cache.end())
    return parsed_metric_cache[spec];

  const char* specstr =  spec.c_str();
  pmMetricSpec* pms = 0;
  char *errmsg;
  char dummy_host[] = "";
  int rc = pmParseMetricSpec (specstr,
                              0, dummy_host, /* both ignored */
                              & pms, & errmsg);
  if (rc < 0) {
    timestamp(cerr) << "hostid-metrics '" << specstr << "' parse error: " << errmsg << endl;
    free (errmsg);
  }

  parsed_metric_cache[spec] = pms;
  return pms;
}


pmmgr_hostid
pmmgr_job_spec::compute_hostid (const pcp_context_spec& ctx)
{
  int pmc = pmNewContext (PM_CONTEXT_HOST, ctx.c_str());
  if (pmc < 0)
    return "";

  // parse all the hostid metric specifications
  vector<string> hostid_specs = get_config_multi("hostid-metrics");
  if (hostid_specs.size() == 0)
    hostid_specs.push_back(string("pmcd.hostname"));

  // fetch all hostid metrics in sequence
  vector<string> hostid_fields;
  for (unsigned i=0; i<hostid_specs.size(); i++)
    {
      pmMetricSpec* pms = parse_metric_spec (hostid_specs[i]);

      pmID pmid;
      int rc = pmLookupName (1, & pms->metric, &pmid);
      if (rc < 0)
        continue;

      pmDesc desc;
      rc = pmLookupDesc (pmid, & desc);
      if (rc < 0)
        continue;

      if (desc.type != PM_TYPE_STRING)
        continue;

      if ((desc.indom != PM_INDOM_NULL) && pms->ninst > 0)
        {
          // reset the indom to include all elements
          rc = pmDelProfile(desc.indom, 0, (int *)0);
          if (rc < 0)
            continue;

          int *inums = (int *) malloc (pms->ninst * sizeof(int));
          if (inums == NULL)
            continue;
          // NB: after this point, 'continue' must also free(inums);

          // map the instance names to instance numbers
          unsigned numinums_used = 0;
          for (int j=0; j<pms->ninst; j++)
            {
              int inum = pmLookupInDom (desc.indom, pms->inst[j]);
              if (inum < 0)
                continue;
              inums[numinums_used++] = inum;
            }

          // add the selected instances to the profile
          rc = pmAddProfile (desc.indom, numinums_used, inums);
          free (inums);
          if (rc < 0)
            continue;
        }

      // fetch the values
      pmResult *r;
      rc = pmFetch (1, &pmid, &r);
      if (rc < 0)
        continue;
      // NB: after this point, 'continue' must also pmFreeResult(r)

      // only vset[0] will be set, for csb->pmid
      if (r->vset[0]->numval > 0)
        {
          // in-place sort value list by indom number
          qsort (r->vset[0]->vlist,
                 (size_t) r->vset[0]->numval,
                 sizeof(pmValue),
                 pmValue_compare);

          for (int j=0; j<r->vset[0]->numval; j++) // iterate over instances
            {
              // fetch the string value
              pmAtomValue av;
              rc = pmExtractValue(r->vset[0]->valfmt,
                                  & r->vset[0]->vlist[j],
                                  PM_TYPE_STRING, & av, PM_TYPE_STRING);
              if (rc < 0)
                continue;
              
              // at last!  we have a string we can accumulate
              hostid_fields.push_back (av.cp);
              free (av.cp);
            }
        }

      (void) pmFreeResult (r);
    }

  (void) pmDestroyContext (pmc);

  // Sanitize the host-id metric values into a single string that is
  // suitable for posix-portable-filenames, and not too ugly for
  // someone to look at or type in.
  //
  // http://www.opengroup.org/onlinepubs/007904975/basedefs/xbd_chap03.html
  string sanitized;
  for (unsigned i=0; i<hostid_fields.size(); i++)
    {<commit_msg>pmmgr: use pmSortInstances() instead of same thing open-coded locally<commit_after>/*
 * Copyright (c) 2013-2014 Red Hat.
 * 
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 */


#include "pmmgr.h"
#include "impl.h"

#include <sys/stat.h>
#include <cassert>
#include <cstdlib>
#include <fstream>
#include <iostream>

extern "C" {
#include <fcntl.h>
#include <unistd.h>
#include <glob.h>
#include <sys/wait.h>
#ifdef HAVE_PTHREAD_H
#include <pthread.h>
#endif
#ifdef IS_LINUX
#include <sys/syscall.h>
#endif
}


using namespace std;

// ------------------------------------------------------------------------


int quit;


// ------------------------------------------------------------------------


// Create a string that is safe to pass to system(3), i.e., sh -c,
// by quoting metacharacters.  This transform generally should be
// applied only once.
string
sh_quote(const string& input)
{
  string output;
  for (unsigned i=0; i<input.length(); i++)
    {
      char c = input[i];
      if ((ispunct(c) || isspace(c)) && // quite aggressive
          (c != ':' && c != '.' && c != '_' && c != '/' && c != '-')) // safe & popular punctuation
        output += '\\';
      output += c;
    }

  return output;
}


// Print a string to cout/cerr progress reports, similar to the
// stuff produced by __pmNotifyErr
ostream&
timestamp(ostream &o)
{
  time_t now;
  time (&now);
  char *now2 = ctime (&now);
  if (now2)
    now2[19] = '\0'; // overwrite \n

  return o << "[" << (now2 ? now2 : "") << "] " << pmProgname << "(" 
           << getpid()
#ifdef HAVE_PTHREAD_H
#ifdef IS_LINUX
           << "/" << syscall(SYS_gettid)
#else
           << "/" << pthread_self()
#endif
#endif
           << "): ";
}


extern "C" void *
pmmgr_daemon_poll_thread (void* a)
{
  pmmgr_daemon* d = (pmmgr_daemon*) a;
  d->poll();
  return 0;
}



// ------------------------------------------------------------------------


pmmgr_configurable::pmmgr_configurable(const string& dir):
  config_directory(dir)
{
}


vector<string>
pmmgr_configurable::get_config_multi(const string& file) const
{
  vector<string> lines;

  string complete_filename = config_directory + (char)__pmPathSeparator() + file;
  ifstream f (complete_filename.c_str());
  while (f.good()) {
    string line;
    getline(f, line);
    if (! f.good())
      break;
    if (line != "")
      lines.push_back(line);
  }

  return lines;
}


bool
pmmgr_configurable::get_config_exists(const string& file) const
{
  string complete_filename = config_directory + (char)__pmPathSeparator() + file;
  ifstream f (complete_filename.c_str());
  return (f.good());
}


string
pmmgr_configurable::get_config_single(const string& file) const
{
  vector<string> lines = get_config_multi (file);
  if (lines.size() == 1)
    return lines[0];
  else
    return "";
}

ostream&
pmmgr_configurable::timestamp(ostream& o)
{
  return ::timestamp(o) << config_directory << ": ";
}



// ------------------------------------------------------------------------


pmMetricSpec*
pmmgr_job_spec::parse_metric_spec (const string& spec)
{
  if (parsed_metric_cache.find(spec) != parsed_metric_cache.end())
    return parsed_metric_cache[spec];

  const char* specstr =  spec.c_str();
  pmMetricSpec* pms = 0;
  char *errmsg;
  char dummy_host[] = "";
  int rc = pmParseMetricSpec (specstr,
                              0, dummy_host, /* both ignored */
                              & pms, & errmsg);
  if (rc < 0) {
    timestamp(cerr) << "hostid-metrics '" << specstr << "' parse error: " << errmsg << endl;
    free (errmsg);
  }

  parsed_metric_cache[spec] = pms;
  return pms;
}


pmmgr_hostid
pmmgr_job_spec::compute_hostid (const pcp_context_spec& ctx)
{
  int pmc = pmNewContext (PM_CONTEXT_HOST, ctx.c_str());
  if (pmc < 0)
    return "";

  // parse all the hostid metric specifications
  vector<string> hostid_specs = get_config_multi("hostid-metrics");
  if (hostid_specs.size() == 0)
    hostid_specs.push_back(string("pmcd.hostname"));

  // fetch all hostid metrics in sequence
  vector<string> hostid_fields;
  for (unsigned i=0; i<hostid_specs.size(); i++)
    {
      pmMetricSpec* pms = parse_metric_spec (hostid_specs[i]);

      pmID pmid;
      int rc = pmLookupName (1, & pms->metric, &pmid);
      if (rc < 0)
        continue;

      pmDesc desc;
      rc = pmLookupDesc (pmid, & desc);
      if (rc < 0)
        continue;

      if (desc.type != PM_TYPE_STRING)
        continue;

      if ((desc.indom != PM_INDOM_NULL) && pms->ninst > 0)
        {
          // reset the indom to include all elements
          rc = pmDelProfile(desc.indom, 0, (int *)0);
          if (rc < 0)
            continue;

          int *inums = (int *) malloc (pms->ninst * sizeof(int));
          if (inums == NULL)
            continue;
          // NB: after this point, 'continue' must also free(inums);

          // map the instance names to instance numbers
          unsigned numinums_used = 0;
          for (int j=0; j<pms->ninst; j++)
            {
              int inum = pmLookupInDom (desc.indom, pms->inst[j]);
              if (inum < 0)
                continue;
              inums[numinums_used++] = inum;
            }

          // add the selected instances to the profile
          rc = pmAddProfile (desc.indom, numinums_used, inums);
          free (inums);
          if (rc < 0)
            continue;
        }

      // fetch the values
      pmResult *r;
      rc = pmFetch (1, &pmid, &r);
      if (rc < 0)
        continue;
      // NB: after this point, 'continue' must also pmFreeResult(r)

      // in-place sort value list by indom number
      pmSortInstances(r);

      // only vset[0] will be set, for csb->pmid
      if (r->vset[0]->numval > 0)
        {
          for (int j=0; j<r->vset[0]->numval; j++) // iterate over instances
            {
              // fetch the string value
              pmAtomValue av;
              rc = pmExtractValue(r->vset[0]->valfmt,
                                  & r->vset[0]->vlist[j],
                                  PM_TYPE_STRING, & av, PM_TYPE_STRING);
              if (rc < 0)
                continue;
              
              // at last!  we have a string we can accumulate
              hostid_fields.push_back (av.cp);
              free (av.cp);
            }
        }

      (void) pmFreeResult (r);
    }

  (void) pmDestroyContext (pmc);

  // Sanitize the host-id metric values into a single string that is
  // suitable for posix-portable-filenames, and not too ugly for
  // someone to look at or type in.
  //
  // http://www.opengroup.org/onlinepubs/007904975/basedefs/xbd_chap03.html
  string sanitized;
  for (unsigned i=0; i<hostid_fields.size(); i++)
    {
{'fraction_non_alphanumeric': 0.16856021483164635, 'fraction_numerical': 0.008538180816635682, 'mean_word_length': 2.058328069067172, 'pattern_counts': {'":': 2, '<': 135, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6136998', 'n_tokens_mistral': 4790, 'n_tokens_neox': 4612, 'n_words': 1280}