<commit_before>#include <errno.h>         // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <ev.h>            // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <math.h>          // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <netinet/in.h>    // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <string.h>        // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <sys/socket.h>    // NOLINT(llvmlibc-restrict-system-libc-headers)

#include "https_client.h"
#include "logging.h"
#include "options.h"

#define ASSERT_CURL_MULTI_SETOPT(curlm, option, param) \
  do { \
    CURLMcode code = curl_multi_setopt(curlm, option, param); \
    if (code != CURLM_OK) { \
      FLOG(#option " error %d: %s", code, curl_multi_strerror(code)); \
    } \
  } while(0);

#define ASSERT_CURL_EASY_SETOPT(curl, option, param) \
  do { \
    CURLcode code = curl_easy_setopt(curl, option, param); \
    if (code != CURLE_OK) { \
      FLOG(#option " error %d: %s", code, curl_easy_strerror(code)); \
    } \
  } while(0);

static size_t write_buffer(void *buf, size_t size, size_t nmemb, void *userp) {
  struct https_fetch_ctx *ctx = (struct https_fetch_ctx *)userp;
  char *new_buf = (char *)realloc(
      ctx->buf, ctx->buflen + size * nmemb + 1);
  if (new_buf == NULL) {
    ELOG("Out of memory!");
    return 0;
  }
  ctx->buf = new_buf;
  // NOLINTNEXTLINE(clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling)
  memcpy(&(ctx->buf[ctx->buflen]), buf, size * nmemb);
  ctx->buflen += size * nmemb;
  // We always expect to receive valid non-null ASCII but just to be safe...
  ctx->buf[ctx->buflen] = '\0';
  return size * nmemb;
}

static curl_socket_t opensocket_callback(void *clientp, curlsocktype purpose,
                                         struct curl_sockaddr *addr) {
  curl_socket_t sock = socket(addr->family, addr->socktype, addr->protocol);

  DLOG("curl opened socket: %d", sock);

#if defined(IP_TOS)
  if (purpose != CURLSOCKTYPE_IPCXN) {
    return sock;
  }

  if (sock != -1) {
    if (addr->family == AF_INET) {
        (void)setsockopt(sock, IPPROTO_IP, IP_TOS, (int *)clientp, sizeof(int));
    }
#if defined(IPV6_TCLASS)
    else if (addr->family == AF_INET6) {
        (void)setsockopt(sock, IPPROTO_IPV6, IPV6_TCLASS, (int *)clientp, sizeof(int));
    }
#endif
  }
#endif

  return sock;
}

static int closesocket_callback(void __attribute__((unused)) *clientp, curl_socket_t item)
{
  DLOG("curl closed socket: %d", item);
  return 0;
}

static void https_fetch_ctx_init(https_client_t *client,
                                 struct https_fetch_ctx *ctx, const char *url,
                                 const char* data, size_t datalen,
                                 struct curl_slist *resolv,
                                 https_response_cb cb, void *cb_data) {
  ctx->curl = curl_easy_init(); // if fails, first setopt will fail
  ctx->cb = cb;
  ctx->cb_data = cb_data;
  ctx->buf = NULL;
  ctx->buflen = 0;
  ctx->next = client->fetches;
  client->fetches = ctx;

  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_RESOLVE, resolv);

  DLOG("Requesting HTTP/1.1: %d\n", client->opt->use_http_1_1);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_HTTP_VERSION,
                          client->opt->use_http_1_1 ?
                          CURL_HTTP_VERSION_1_1 :
                          CURL_HTTP_VERSION_2_0);
  if (logging_debug_enabled()) {
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_VERBOSE, 1L);
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_OPENSOCKETFUNCTION, opensocket_callback);
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_CLOSESOCKETFUNCTION, closesocket_callback);
  }
#if defined(IP_TOS)
  if (client->opt->dscp) {
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_OPENSOCKETDATA, &client->opt->dscp);
    if (!logging_debug_enabled()) {
        ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_OPENSOCKETFUNCTION, opensocket_callback);
      }
  }
#endif
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_URL, url);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_HTTPHEADER, client->header_list);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_POSTFIELDSIZE, datalen);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_POSTFIELDS, data);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_WRITEFUNCTION, &write_buffer);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_WRITEDATA, ctx);
#ifdef CURLOPT_MAXAGE_CONN
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_TCP_KEEPALIVE, 1L);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_TCP_KEEPIDLE, 50L);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_TCP_KEEPINTVL, 50L);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_MAXAGE_CONN, 300L);
#endif
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_USERAGENT, "dns-to-https-proxy/0.2");
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_FOLLOWLOCATION, 0);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_NOSIGNAL, 0);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_TIMEOUT, 10 /* seconds */);
  // We know Google supports this, so force it.
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
  if (client->opt->curl_proxy) {
    DLOG("Using curl proxy: %s", client->opt->curl_proxy);
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_PROXY, client->opt->curl_proxy);
  }
  CURLMcode code = curl_multi_add_handle(client->curlm, ctx->curl);
  if (code != CURLM_OK) {
    FLOG("curl_multi_add_handle error %d: %s", code, curl_multi_strerror(code));
  }
}

static void https_fetch_ctx_process_response(https_client_t *client,
                                             struct https_fetch_ctx *ctx)
{
  CURLcode res = 0;
  long long_resp = 0;
  char *str_resp = NULL;
  int faulty_response = 1;

  if ((res = curl_easy_getinfo(
        ctx->curl, CURLINFO_RESPONSE_CODE, &long_resp)) != CURLE_OK) {
    ELOG("CURLINFO_RESPONSE_CODE: %s", curl_easy_strerror(res));
  } else {
    if (long_resp == 200) {
      faulty_response = 0;
    } else {
      ELOG("curl response code: %d", long_resp);
    }
  }

  if (logging_debug_enabled() || faulty_response || ctx->buflen == 0) {
    if ((res = curl_easy_getinfo(
            ctx->curl, CURLINFO_REDIRECT_URL, &str_resp)) != CURLE_OK) {
      ELOG("CURLINFO_REDIRECT_URL: %s", curl_easy_strerror(res));
    } else if (str_resp != NULL) {
      ELOG("Request would be redirected to: %s", str_resp);
      if (strcmp(str_resp, client->opt->resolver_url)) {
        ELOG("Please update Resolver URL to avoid redirection!");
      }
    }
    if ((res = curl_easy_getinfo(
            ctx->curl, CURLINFO_SSL_VERIFYRESULT, &long_resp)) != CURLE_OK) {
      ELOG("CURLINFO_SSL_VERIFYRESULT: %s", curl_easy_strerror(res));
    } else if (long_resp != CURLE_OK) {
      ELOG("CURLINFO_SSL_VERIFYRESULT: %s", curl_easy_strerror(long_resp));
    }
    if ((res = curl_easy_getinfo(
            ctx->curl, CURLINFO_OS_ERRNO, &long_resp)) != CURLE_OK) {
      ELOG("CURLINFO_OS_ERRNO: %s", curl_easy_strerror(res));
    } else if (long_resp != 0) {
      ELOG("CURLINFO_OS_ERRNO: %d %s", long_resp, strerror(long_resp));
      if (long_resp == ENETUNREACH && !client->opt->ipv4) {
        ELOG("Try to run application with -4 argument!");<commit_msg>Dump body of failed requests Test with: https://doh.opendns.com<commit_after>#include <errno.h>         // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <ev.h>            // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <math.h>          // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <netinet/in.h>    // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <string.h>        // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <sys/socket.h>    // NOLINT(llvmlibc-restrict-system-libc-headers)
#include <ctype.h>

#include "https_client.h"
#include "logging.h"
#include "options.h"

#define ASSERT_CURL_MULTI_SETOPT(curlm, option, param) \
  do { \
    CURLMcode code = curl_multi_setopt(curlm, option, param); \
    if (code != CURLM_OK) { \
      FLOG(#option " error %d: %s", code, curl_multi_strerror(code)); \
    } \
  } while(0);

#define ASSERT_CURL_EASY_SETOPT(curl, option, param) \
  do { \
    CURLcode code = curl_easy_setopt(curl, option, param); \
    if (code != CURLE_OK) { \
      FLOG(#option " error %d: %s", code, curl_easy_strerror(code)); \
    } \
  } while(0);

static size_t write_buffer(void *buf, size_t size, size_t nmemb, void *userp) {
  struct https_fetch_ctx *ctx = (struct https_fetch_ctx *)userp;
  char *new_buf = (char *)realloc(
      ctx->buf, ctx->buflen + size * nmemb + 1);
  if (new_buf == NULL) {
    ELOG("Out of memory!");
    return 0;
  }
  ctx->buf = new_buf;
  // NOLINTNEXTLINE(clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling)
  memcpy(&(ctx->buf[ctx->buflen]), buf, size * nmemb);
  ctx->buflen += size * nmemb;
  // We always expect to receive valid non-null ASCII but just to be safe...
  ctx->buf[ctx->buflen] = '\0';
  return size * nmemb;
}

static curl_socket_t opensocket_callback(void *clientp, curlsocktype purpose,
                                         struct curl_sockaddr *addr) {
  curl_socket_t sock = socket(addr->family, addr->socktype, addr->protocol);

  DLOG("curl opened socket: %d", sock);

#if defined(IP_TOS)
  if (purpose != CURLSOCKTYPE_IPCXN) {
    return sock;
  }

  if (sock != -1) {
    if (addr->family == AF_INET) {
        (void)setsockopt(sock, IPPROTO_IP, IP_TOS, (int *)clientp, sizeof(int));
    }
#if defined(IPV6_TCLASS)
    else if (addr->family == AF_INET6) {
        (void)setsockopt(sock, IPPROTO_IPV6, IPV6_TCLASS, (int *)clientp, sizeof(int));
    }
#endif
  }
#endif

  return sock;
}

static int closesocket_callback(void __attribute__((unused)) *clientp, curl_socket_t item)
{
  DLOG("curl closed socket: %d", item);
  return 0;
}

static void https_fetch_ctx_init(https_client_t *client,
                                 struct https_fetch_ctx *ctx, const char *url,
                                 const char* data, size_t datalen,
                                 struct curl_slist *resolv,
                                 https_response_cb cb, void *cb_data) {
  ctx->curl = curl_easy_init(); // if fails, first setopt will fail
  ctx->cb = cb;
  ctx->cb_data = cb_data;
  ctx->buf = NULL;
  ctx->buflen = 0;
  ctx->next = client->fetches;
  client->fetches = ctx;

  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_RESOLVE, resolv);

  DLOG("Requesting HTTP/1.1: %d\n", client->opt->use_http_1_1);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_HTTP_VERSION,
                          client->opt->use_http_1_1 ?
                          CURL_HTTP_VERSION_1_1 :
                          CURL_HTTP_VERSION_2_0);
  if (logging_debug_enabled()) {
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_VERBOSE, 1L);
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_OPENSOCKETFUNCTION, opensocket_callback);
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_CLOSESOCKETFUNCTION, closesocket_callback);
  }
#if defined(IP_TOS)
  if (client->opt->dscp) {
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_OPENSOCKETDATA, &client->opt->dscp);
    if (!logging_debug_enabled()) {
        ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_OPENSOCKETFUNCTION, opensocket_callback);
      }
  }
#endif
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_URL, url);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_HTTPHEADER, client->header_list);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_POSTFIELDSIZE, datalen);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_POSTFIELDS, data);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_WRITEFUNCTION, &write_buffer);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_WRITEDATA, ctx);
#ifdef CURLOPT_MAXAGE_CONN
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_TCP_KEEPALIVE, 1L);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_TCP_KEEPIDLE, 50L);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_TCP_KEEPINTVL, 50L);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_MAXAGE_CONN, 300L);
#endif
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_USERAGENT, "dns-to-https-proxy/0.2");
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_FOLLOWLOCATION, 0);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_NOSIGNAL, 0);
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_TIMEOUT, 10 /* seconds */);
  // We know Google supports this, so force it.
  ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
  if (client->opt->curl_proxy) {
    DLOG("Using curl proxy: %s", client->opt->curl_proxy);
    ASSERT_CURL_EASY_SETOPT(ctx->curl, CURLOPT_PROXY, client->opt->curl_proxy);
  }
  CURLMcode code = curl_multi_add_handle(client->curlm, ctx->curl);
  if (code != CURLM_OK) {
    FLOG("curl_multi_add_handle error %d: %s", code, curl_multi_strerror(code));
  }
}

static void https_log_response_content(char *ptr, size_t size)
{
  const size_t width = 0x10;

  for (size_t i = 0; i < size; i += width) {
    char hex[3 * width + 1];
    char str[width + 1];
    size_t hex_off = 0;
    size_t str_off = 0;
    memset(hex, 0, sizeof(hex));
    memset(str, 0, sizeof(str));

    for (size_t c = 0; c < width; c++) {
      if (i+c < size) {
        hex_off += snprintf(hex + hex_off, sizeof(hex) - hex_off,
                            "%02x ", (unsigned char)ptr[i+c]);
        str_off += snprintf(str + str_off, sizeof(str) - str_off,
                            "%c", isprint(ptr[i+c]) ? ptr[i+c] : '.');
      } else {
        hex_off += snprintf(hex + hex_off, sizeof(hex) - hex_off, "   ");
      }
    }

    ELOG("%4.4lx: %s%s", (long)i, hex, str);
  }
}

static void https_fetch_ctx_process_response(https_client_t *client,
                                             struct https_fetch_ctx *ctx)
{
  CURLcode res = 0;
  long long_resp = 0;
  char *str_resp = NULL;
  int faulty_response = 1;

  if ((res = curl_easy_getinfo(
        ctx->curl, CURLINFO_RESPONSE_CODE, &long_resp)) != CURLE_OK) {
    ELOG("CURLINFO_RESPONSE_CODE: %s", curl_easy_strerror(res));
  } else {
    if (long_resp == 200) {
      faulty_response = 0;
    } else {
      ELOG("curl response code: %d, content length: %zu", long_resp, ctx->buflen);
      if (ctx->buflen >= 0) {
        https_log_response_content(ctx->buf, ctx->buflen);
      }
    }
  }

  if (logging_debug_enabled() || faulty_response || ctx->buflen == 0) {
    if ((res = curl_easy_getinfo(
            ctx->curl, CURLINFO_REDIRECT_URL, &str_resp)) != CURLE_OK) {
      ELOG("CURLINFO_REDIRECT_URL: %s", curl_easy_strerror(res));
    } else if (str_resp != NULL) {
      ELOG("Request would be redirected to: %s", str_resp);
      if (strcmp(str_resp, client->opt->resolver_url)) {
        ELOG("Please update Resolver URL to avoid redirection!");
      }
    }
    if ((res = curl_easy_getinfo(
            ctx->curl, CURLINFO_SSL_VERIFYRESULT, &long_resp)) != CURLE_OK) {
      ELOG("CURLINFO_SSL_VERIFYRESULT: %s", curl_easy_strerror(res));
    } else if (long_resp != CURLE_OK) {
      ELOG("CURLINFO_SSL_VERIFYRESULT: %s", curl_easy_strerror(long_resp));
    }
    if ((res = curl_easy_getinfo(
            ctx->curl, CURLINFO_OS_ERRNO, &long_resp)) != CURLE_OK) {
      ELOG("CURLINFO_OS_ERRNO: %s", curl_easy_strerror(res));
    } else if (long_resp != 0) {
      ELOG("CURLINFO_OS_ERRNO: %d %s", long_resp, strerror(long_resp));
      if (long_resp == ENETUNREACH && !client->opt->ipv4) {
        ELOG("Try to run application with -4 argument!");
{'fraction_non_alphanumeric': 0.1355079095306576, 'fraction_numerical': 0.00764805856974768, 'mean_word_length': 3.440928882438316, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 161, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '612901', 'n_tokens_mistral': 6085, 'n_tokens_neox': 5756, 'n_words': 1041}