<commit_before>/** @file

    Include file for the traffic_top stats.

    @section license License

    Licensed to the Apache Software Foundation (ASF) under one
    or more contributor license agreements.  See the NOTICE file
    distributed with this work for additional information
    regarding copyright ownership.  The ASF licenses this file
    to you under the Apache License, Version 2.0 (the
    "License"); you may not use this file except in compliance
    with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/
#if HAS_CURL
#include <curl/curl.h>
#endif
#include <map>
#include <string>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <cinttypes>
#include <sys/time.h>
#include "mgmtapi.h"

using namespace std;

struct LookupItem {
  LookupItem(const char *s, const char *n, const int t) : pretty(s), name(n), numerator(""), denominator(""), type(t) {}
  LookupItem(const char *s, const char *n, const char *d, const int t) : pretty(s), name(n), numerator(n), denominator(d), type(t)
  {
  }
  const char *pretty;
  const char *name;
  const char *numerator;
  const char *denominator;
  int type;
};
extern size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream);
#if HAS_CURL
extern char curl_error[CURL_ERROR_SIZE];
#endif
extern string response;

namespace constant
{
const char global[]    = "\"global\": {\n";
const char start[]     = "\"proxy.process.";
const char separator[] = "\": \"";
const char end[]       = "\",\n";
}; // namespace constant

//----------------------------------------------------------------------------
class Stats
{
public:
  Stats(const string &url) : _url(url)
  {
    if (url != "") {
      if (_url.substr(0, 4) != "http") {
        // looks like it is a host using it the old way
        _url = "http://" + _url + "/_stats";
      }

      // set the host
      size_t start = _url.find(":");
      size_t end   = _url.find("/", start + 3);
      _host        = _url.substr(start + 3, end - start - 3);
      end          = _host.find(":");
      if (end != string::npos) {
        _host = _host.substr(0, end);
      }
    } else {
      char hostname[25];
      hostname[sizeof(hostname) - 1] = '\0';
      gethostname(hostname, sizeof(hostname) - 1);
      _host = hostname;
    }

    _time_diff = 0;
    _old_time  = 0;
    _now       = 0;
    _time      = (struct timeval){0, 0};
    _stats     = nullptr;
    _old_stats = nullptr;
    _absolute  = false;
    lookup_table.insert(make_pair("version", LookupItem("Version", "proxy.process.version.server.short", 1)));
    lookup_table.insert(make_pair("disk_used", LookupItem("Disk Used", "proxy.process.cache.bytes_used", 1)));
    lookup_table.insert(make_pair("disk_total", LookupItem("Disk Total", "proxy.process.cache.bytes_total", 1)));
    lookup_table.insert(make_pair("ram_used", LookupItem("Ram Used", "proxy.process.cache.ram_cache.bytes_used", 1)));
    lookup_table.insert(make_pair("ram_total", LookupItem("Ram Total", "proxy.process.cache.ram_cache.total_bytes", 1)));
    lookup_table.insert(make_pair("lookups", LookupItem("Lookups", "proxy.process.http.cache_lookups", 2)));
    lookup_table.insert(make_pair("cache_writes", LookupItem("Writes", "proxy.process.http.cache_writes", 2)));
    lookup_table.insert(make_pair("cache_updates", LookupItem("Updates", "proxy.process.http.cache_updates", 2)));
    lookup_table.insert(make_pair("cache_deletes", LookupItem("Deletes", "proxy.process.http.cache_deletes", 2)));
    lookup_table.insert(make_pair("read_active", LookupItem("Read Active", "proxy.process.cache.read.active", 1)));
    lookup_table.insert(make_pair("write_active", LookupItem("Writes Active", "proxy.process.cache.write.active", 1)));
    lookup_table.insert(make_pair("update_active", LookupItem("Update Active", "proxy.process.cache.update.active", 1)));
    lookup_table.insert(make_pair("entries", LookupItem("Entries", "proxy.process.cache.direntries.used", 1)));
    lookup_table.insert(make_pair("avg_size", LookupItem("Avg Size", "disk_used", "entries", 3)));

    lookup_table.insert(make_pair("dns_entry", LookupItem("DNS Entry", "proxy.process.hostdb.cache.current_items", 1)));
    lookup_table.insert(make_pair("dns_hits", LookupItem("DNS Hits", "proxy.process.hostdb.total_hits", 2)));
    lookup_table.insert(make_pair("dns_lookups", LookupItem("DNS Lookups", "proxy.process.hostdb.total_lookups", 2)));
    lookup_table.insert(make_pair("client_req", LookupItem("Requests", "proxy.process.http.incoming_requests", 2)));
    lookup_table.insert(make_pair("client_conn", LookupItem("New Conn", "proxy.process.http.total_client_connections", 2)));
    lookup_table.insert(make_pair("client_req_conn", LookupItem("Req/Conn", "client_req", "client_conn", 3)));
    lookup_table.insert(make_pair("client_curr_conn", LookupItem("Curr Conn", "proxy.process.http.current_client_connections", 1)));
    lookup_table.insert(
      make_pair("client_actv_conn", LookupItem("Active Con", "proxy.process.http.current_active_client_connections", 1)));

    lookup_table.insert(make_pair("server_req", LookupItem("Requests", "proxy.process.http.outgoing_requests", 2)));
    lookup_table.insert(make_pair("server_conn", LookupItem("New Conn", "proxy.process.http.total_server_connections", 2)));
    lookup_table.insert(make_pair("server_req_conn", LookupItem("Req/Conn", "server_req", "server_conn", 3)));
    lookup_table.insert(make_pair("server_curr_conn", LookupItem("Curr Conn", "proxy.process.http.current_server_connections", 1)));

    lookup_table.insert(
      make_pair("client_head", LookupItem("Head Bytes", "proxy.process.http.user_agent_response_header_total_size", 2)));
    lookup_table.insert(
      make_pair("client_body", LookupItem("Body Bytes", "proxy.process.http.user_agent_response_document_total_size", 2)));
    lookup_table.insert(
      make_pair("server_head", LookupItem("Head Bytes", "proxy.process.http.origin_server_response_header_total_size", 2)));
    lookup_table.insert(
      make_pair("server_body", LookupItem("Body Bytes", "proxy.process.http.origin_server_response_document_total_size", 2)));

    // not used directly<commit_msg>Fix client connections of traffic_top<commit_after>/** @file

    Include file for the traffic_top stats.

    @section license License

    Licensed to the Apache Software Foundation (ASF) under one
    or more contributor license agreements.  See the NOTICE file
    distributed with this work for additional information
    regarding copyright ownership.  The ASF licenses this file
    to you under the Apache License, Version 2.0 (the
    "License"); you may not use this file except in compliance
    with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/
#if HAS_CURL
#include <curl/curl.h>
#endif
#include <map>
#include <string>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <cinttypes>
#include <sys/time.h>
#include "mgmtapi.h"

using namespace std;

struct LookupItem {
  LookupItem(const char *s, const char *n, const int t) : pretty(s), name(n), numerator(""), denominator(""), type(t) {}
  LookupItem(const char *s, const char *n, const char *d, const int t) : pretty(s), name(n), numerator(n), denominator(d), type(t)
  {
  }
  const char *pretty;
  const char *name;
  const char *numerator;
  const char *denominator;
  int type;
};
extern size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream);
#if HAS_CURL
extern char curl_error[CURL_ERROR_SIZE];
#endif
extern string response;

namespace constant
{
const char global[]    = "\"global\": {\n";
const char start[]     = "\"proxy.process.";
const char separator[] = "\": \"";
const char end[]       = "\",\n";
}; // namespace constant

//----------------------------------------------------------------------------
class Stats
{
public:
  Stats(const string &url) : _url(url)
  {
    if (url != "") {
      if (_url.substr(0, 4) != "http") {
        // looks like it is a host using it the old way
        _url = "http://" + _url + "/_stats";
      }

      // set the host
      size_t start = _url.find(":");
      size_t end   = _url.find("/", start + 3);
      _host        = _url.substr(start + 3, end - start - 3);
      end          = _host.find(":");
      if (end != string::npos) {
        _host = _host.substr(0, end);
      }
    } else {
      char hostname[25];
      hostname[sizeof(hostname) - 1] = '\0';
      gethostname(hostname, sizeof(hostname) - 1);
      _host = hostname;
    }

    _time_diff = 0;
    _old_time  = 0;
    _now       = 0;
    _time      = (struct timeval){0, 0};
    _stats     = nullptr;
    _old_stats = nullptr;
    _absolute  = false;
    lookup_table.insert(make_pair("version", LookupItem("Version", "proxy.process.version.server.short", 1)));
    lookup_table.insert(make_pair("disk_used", LookupItem("Disk Used", "proxy.process.cache.bytes_used", 1)));
    lookup_table.insert(make_pair("disk_total", LookupItem("Disk Total", "proxy.process.cache.bytes_total", 1)));
    lookup_table.insert(make_pair("ram_used", LookupItem("Ram Used", "proxy.process.cache.ram_cache.bytes_used", 1)));
    lookup_table.insert(make_pair("ram_total", LookupItem("Ram Total", "proxy.process.cache.ram_cache.total_bytes", 1)));
    lookup_table.insert(make_pair("lookups", LookupItem("Lookups", "proxy.process.http.cache_lookups", 2)));
    lookup_table.insert(make_pair("cache_writes", LookupItem("Writes", "proxy.process.http.cache_writes", 2)));
    lookup_table.insert(make_pair("cache_updates", LookupItem("Updates", "proxy.process.http.cache_updates", 2)));
    lookup_table.insert(make_pair("cache_deletes", LookupItem("Deletes", "proxy.process.http.cache_deletes", 2)));
    lookup_table.insert(make_pair("read_active", LookupItem("Read Active", "proxy.process.cache.read.active", 1)));
    lookup_table.insert(make_pair("write_active", LookupItem("Writes Active", "proxy.process.cache.write.active", 1)));
    lookup_table.insert(make_pair("update_active", LookupItem("Update Active", "proxy.process.cache.update.active", 1)));
    lookup_table.insert(make_pair("entries", LookupItem("Entries", "proxy.process.cache.direntries.used", 1)));
    lookup_table.insert(make_pair("avg_size", LookupItem("Avg Size", "disk_used", "entries", 3)));

    lookup_table.insert(make_pair("dns_entry", LookupItem("DNS Entry", "proxy.process.hostdb.cache.current_items", 1)));
    lookup_table.insert(make_pair("dns_hits", LookupItem("DNS Hits", "proxy.process.hostdb.total_hits", 2)));
    lookup_table.insert(make_pair("dns_lookups", LookupItem("DNS Lookups", "proxy.process.hostdb.total_lookups", 2)));

    // Incoming HTTP/1.1 and HTTP/2 connections - some metrics are HTTP version specific
    lookup_table.insert(make_pair("client_req", LookupItem("Requests", "proxy.process.http.incoming_requests", 2)));

    // total_client_connections
    lookup_table.insert(
      make_pair("client_conn_h1", LookupItem("New Conn HTTP/1.x", "proxy.process.http.total_client_connections", 2)));
    lookup_table.insert(
      make_pair("client_conn_h2", LookupItem("New Conn HTTP/2", "proxy.process.http2.total_client_connections", 2)));
    lookup_table.insert(make_pair("client_conn", LookupItem("New Conn", "client_conn_h1", "client_conn_h2", 6)));

    // requests / connections
    lookup_table.insert(make_pair("client_req_conn", LookupItem("Req/Conn", "client_req", "client_conn", 3)));

    // current_client_connections
    lookup_table.insert(
      make_pair("client_curr_conn_h1", LookupItem("Curr Conn HTTP/1.x", "proxy.process.http.current_client_connections", 1)));
    lookup_table.insert(
      make_pair("client_curr_conn_h2", LookupItem("Curr Conn HTTP/2", "proxy.process.http2.current_client_connections", 1)));
    lookup_table.insert(make_pair("client_curr_conn", LookupItem("Curr Conn", "client_curr_conn_h1", "client_curr_conn_h2", 6)));

    // current_active_client_connections
    lookup_table.insert(make_pair("client_actv_conn_h1",
                                  LookupItem("Active Con HTTP/1.x", "proxy.process.http.current_active_client_connections", 1)));
    lookup_table.insert(make_pair("client_actv_conn_h2",
                                  LookupItem("Active Con HTTP/2", "proxy.process.http2.current_active_client_connections", 1)));
    lookup_table.insert(make_pair("client_actv_conn", LookupItem("Active Con", "client_actv_conn_h1", "client_actv_conn_h2", 6)));

    lookup_table.insert(make_pair("server_req", LookupItem("Requests", "proxy.process.http.outgoing_requests", 2)));
    lookup_table.insert(make_pair("server_conn", LookupItem("New Conn", "proxy.process.http.total_server_connections", 2)));
    lookup_table.insert(make_pair("server_req_conn", LookupItem("Req/Conn", "server_req", "server_conn", 3)));
    lookup_table.insert(make_pair("server_curr_conn", LookupItem("Curr Conn", "proxy.process.http.current_server_connections", 1)));

    lookup_table.insert(
      make_pair("client_head", LookupItem("Head Bytes", "proxy.process.http.user_agent_response_header_total_size", 2)));
    lookup_table.insert(
      make_pair("client_body", LookupItem("Body Bytes", "proxy.process.http.user_agent_response_document_total_size", 2)));
    lookup_table.insert(
      make_pair("server_head", LookupItem("Head Bytes", "proxy.process.http.origin_server_response_header_total_size", 2)));
    lookup_table.insert(
      make_pair("server_body", LookupItem("Body Bytes", "proxy.process.http.origin_server_response_document_total_size", 2)));

    // not used directly
{'fraction_non_alphanumeric': 0.15485950529044917, 'fraction_numerical': 0.009109382664144069, 'mean_word_length': 4.297698589458055, 'pattern_counts': {'":': 8, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5967968', 'n_tokens_mistral': 4666, 'n_tokens_neox': 4496, 'n_words': 989}