<commit_before>/*
 * This File is part of Davix, The IO library for HTTP based protocols
 * Copyright (C) CERN 2019
 * Author: <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 *
*/

#include "StandaloneCurlRequest.hpp"
#include "CurlSessionFactory.hpp"
#include "CurlSession.hpp"
#include "HeaderlineParser.hpp"
#include <utils/davix_logger_internal.hpp>
#include <core/ContentProvider.hpp>
#include <curl/curl.h>

#define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str()
#define DBG(message) std::cerr << __FILE__ << ":" << __LINE__ << " -- " << #message << " = " << message << std::endl;

namespace Davix {

//------------------------------------------------------------------------------
// Header callback
//------------------------------------------------------------------------------
static size_t header_callback(char *buffer, size_t size, size_t nitems, void *userdata) {
  size_t bytes = size * nitems;

  StandaloneCurlRequest* req = (StandaloneCurlRequest*) userdata;
  req->feedResponseHeader(std::string(buffer, bytes));
  return bytes;
}

//------------------------------------------------------------------------------
// Write callback
//------------------------------------------------------------------------------
static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) {
  size_t bytes = size * nmemb;

  ResponseBuffer* buff = (ResponseBuffer*) userdata;
  buff->feed(ptr, bytes);
  return bytes;
}

//------------------------------------------------------------------------------
// Read callback
//------------------------------------------------------------------------------
size_t read_callback(char *buffer, size_t size, size_t nitems, void *userdata) {
  size_t bytes = size * nitems;

  ContentProvider* provider = (ContentProvider*) userdata;
  ssize_t retval = provider->pullBytes(buffer, bytes);

  if(retval < 0) {
    DAVIX_SLOG(DAVIX_LOG_WARNING, DAVIX_LOG_HTTP, "Content provider reported an errc={}", retval);
    return 0;
  }

  return retval;
}

//------------------------------------------------------------------------------
// Constructor
//------------------------------------------------------------------------------
StandaloneCurlRequest::StandaloneCurlRequest(CurlSessionFactory &sessionFactory, bool reuseSession,
  const BoundHooks &boundHooks, const Uri &uri, const std::string &verb, const RequestParams &params,
  const std::vector<HeaderLine> &headers, int reqFlag, ContentProvider *contentProvider,
  Chrono::TimePoint deadline)
: _session_factory(sessionFactory), _reuse_session(reuseSession), _bound_hooks(boundHooks),
  _uri(uri), _verb(verb), _params(params), _headers(headers), _req_flag(reqFlag),
  _content_provider(contentProvider), _deadline(deadline), _state(RequestState::kNotStarted),
  _chunklist(NULL), _received_headers(false) {}

//------------------------------------------------------------------------------
// Destructor
//------------------------------------------------------------------------------
StandaloneCurlRequest::~StandaloneCurlRequest() {
    curl_slist_free_all(_chunklist);
}

//------------------------------------------------------------------------------
// Get a specific response header
//------------------------------------------------------------------------------
bool StandaloneCurlRequest::getAnswerHeader(const std::string &header_name, std::string &value) const {
  for(auto it = _response_headers.begin(); it != _response_headers.end(); it++) {
    if(it->first == header_name) {
      value = it->second;
      return true;
    }
  }

  return false;
}

//------------------------------------------------------------------------------
// Get all response headers
//------------------------------------------------------------------------------
size_t StandaloneCurlRequest::getAnswerHeaders(std::vector<std::pair<std::string, std::string > > & vec_headers) const {
  vec_headers = _response_headers;
  return vec_headers.size();
}

//------------------------------------------------------------------------------
// Start request - calling this multiple times will do nothing.
//------------------------------------------------------------------------------
Status StandaloneCurlRequest::startRequest() {
  if(_state != RequestState::kNotStarted) {
    return Status(); ;
  }

  //----------------------------------------------------------------------------
  // Have we timed out already?
  //----------------------------------------------------------------------------
  Status st = checkTimeout();
  if(!st.ok()) {
    // markCompleted();
    return st;
  }

  //----------------------------------------------------------------------------
  // Retrieve a session, create request
  //----------------------------------------------------------------------------
  _session = _session_factory.provideCurlSession(_uri, _params, st);
  if(!st.ok()) {
    // markCompleted();
    return st;
  }

  //----------------------------------------------------------------------------
  // Set request verb, target URL
  //----------------------------------------------------------------------------
  CURL* handle = _session->getHandle()->handle;
  CURLM* mhandle = _session->getHandle()->mhandle;

  curl_easy_setopt(handle, CURLOPT_CUSTOMREQUEST, _verb.c_str());
  curl_easy_setopt(handle, CURLOPT_URL, _uri.getString().c_str());

  //----------------------------------------------------------------------------
  // Set up callback to consume response headers
  //----------------------------------------------------------------------------
  curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, header_callback);
  curl_easy_setopt(handle, CURLOPT_HEADERDATA, this);

  //----------------------------------------------------------------------------
  // Set up callback to consume response body
  //----------------------------------------------------------------------------
  curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, write_callback);
  curl_easy_setopt(handle, CURLOPT_WRITEDATA, &_response_buffer);

  //----------------------------------------------------------------------------
  // Set up callback to provide request body
  //----------------------------------------------------------------------------
  if(_content_provider) {
    _content_provider->rewind();
    curl_easy_setopt(handle, CURLOPT_READFUNCTION, read_callback);
    curl_easy_setopt(handle, CURLOPT_READDATA, _content_provider);
  }

  //----------------------------------------------------------------------------
  // Set-up headers
  //----------------------------------------------------------------------------
  for(size_t i = 0; i < _headers.size(); i++) {
    _chunklist = curl_slist_append(_chunklist, SSTR(_headers[i].first << ": " << _headers[i].second).c_str());
  }

  curl_easy_setopt(handle, CURLOPT_HTTPHEADER, _chunklist);

  //----------------------------------------------------------------------------
  // Start request
  //----------------------------------------------------------------------------
  int still_running = 1;

  while(still_running != 0) {
    if(curl_multi_perform(mhandle, &still_running) != CURLM_OK) {
      break;
    }
  }

  _state = RequestState::kStarted;
  return Status();
}

//------------------------------------------------------------------------------
// Major read function - read a block of max_size bytes (at max) into buffer.
//------------------------------------------------------------------------------
dav_ssize_t StandaloneCurlRequest::readBlock(char* buffer, dav_size_t max_size, Status& st) {
  if(!_session) {
    st = Status(davix_scope_http_request(), StatusCode::AlreadyRunning, "Request has not been started yet");
    return -1;
  }

  if(max_size == 0) {
    return 0;
  }

  st = checkTimeout();
  if(!st.ok()) {
    return -1;
  }

  st = Status();
  return _response_buffer.consume(buffer, max_size);
}

//------------------------------------------------------------------------------<commit_msg>Implement CA path registration, allow skipping server certificate verification<commit_after>/*
 * This File is part of Davix, The IO library for HTTP based protocols
 * Copyright (C) CERN 2019
 * Author: <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 *
*/

#include "StandaloneCurlRequest.hpp"
#include "CurlSessionFactory.hpp"
#include "CurlSession.hpp"
#include "HeaderlineParser.hpp"
#include <utils/davix_logger_internal.hpp>
#include <core/ContentProvider.hpp>
#include <curl/curl.h>

#define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str()
#define DBG(message) std::cerr << __FILE__ << ":" << __LINE__ << " -- " << #message << " = " << message << std::endl;

namespace Davix {

//------------------------------------------------------------------------------
// Header callback
//------------------------------------------------------------------------------
static size_t header_callback(char *buffer, size_t size, size_t nitems, void *userdata) {
  size_t bytes = size * nitems;

  StandaloneCurlRequest* req = (StandaloneCurlRequest*) userdata;
  req->feedResponseHeader(std::string(buffer, bytes));
  return bytes;
}

//------------------------------------------------------------------------------
// Write callback
//------------------------------------------------------------------------------
static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) {
  size_t bytes = size * nmemb;

  ResponseBuffer* buff = (ResponseBuffer*) userdata;
  buff->feed(ptr, bytes);
  return bytes;
}

//------------------------------------------------------------------------------
// Read callback
//------------------------------------------------------------------------------
size_t read_callback(char *buffer, size_t size, size_t nitems, void *userdata) {
  size_t bytes = size * nitems;

  ContentProvider* provider = (ContentProvider*) userdata;
  ssize_t retval = provider->pullBytes(buffer, bytes);

  if(retval < 0) {
    DAVIX_SLOG(DAVIX_LOG_WARNING, DAVIX_LOG_HTTP, "Content provider reported an errc={}", retval);
    return 0;
  }

  return retval;
}

//------------------------------------------------------------------------------
// Constructor
//------------------------------------------------------------------------------
StandaloneCurlRequest::StandaloneCurlRequest(CurlSessionFactory &sessionFactory, bool reuseSession,
  const BoundHooks &boundHooks, const Uri &uri, const std::string &verb, const RequestParams &params,
  const std::vector<HeaderLine> &headers, int reqFlag, ContentProvider *contentProvider,
  Chrono::TimePoint deadline)
: _session_factory(sessionFactory), _reuse_session(reuseSession), _bound_hooks(boundHooks),
  _uri(uri), _verb(verb), _params(params), _headers(headers), _req_flag(reqFlag),
  _content_provider(contentProvider), _deadline(deadline), _state(RequestState::kNotStarted),
  _chunklist(NULL), _received_headers(false) {}

//------------------------------------------------------------------------------
// Destructor
//------------------------------------------------------------------------------
StandaloneCurlRequest::~StandaloneCurlRequest() {
    curl_slist_free_all(_chunklist);
}

//------------------------------------------------------------------------------
// Get a specific response header
//------------------------------------------------------------------------------
bool StandaloneCurlRequest::getAnswerHeader(const std::string &header_name, std::string &value) const {
  for(auto it = _response_headers.begin(); it != _response_headers.end(); it++) {
    if(it->first == header_name) {
      value = it->second;
      return true;
    }
  }

  return false;
}

//------------------------------------------------------------------------------
// Get all response headers
//------------------------------------------------------------------------------
size_t StandaloneCurlRequest::getAnswerHeaders(std::vector<std::pair<std::string, std::string > > & vec_headers) const {
  vec_headers = _response_headers;
  return vec_headers.size();
}

//------------------------------------------------------------------------------
// Traslate curl code into status
//------------------------------------------------------------------------------
static Status curlCodeToStatus(CURLcode code) {
  switch(code) {
    case CURLE_OK: {
      return Status();
    }
    default: {
      std::ostringstream ss;
      ss << "curl error (" << code << "): " << curl_easy_strerror(code);
      return Status(davix_scope_http_request(), StatusCode::UnknowError, ss.str());
    }
  }
}

//------------------------------------------------------------------------------
// Start request - calling this multiple times will do nothing.
//------------------------------------------------------------------------------
Status StandaloneCurlRequest::startRequest() {
  if(_state != RequestState::kNotStarted) {
    return Status(); ;
  }

  //----------------------------------------------------------------------------
  // Have we timed out already?
  //----------------------------------------------------------------------------
  Status st = checkTimeout();
  if(!st.ok()) {
    // markCompleted();
    return st;
  }

  //----------------------------------------------------------------------------
  // Retrieve a session, create request
  //----------------------------------------------------------------------------
  _session = _session_factory.provideCurlSession(_uri, _params, st);
  if(!st.ok()) {
    // markCompleted();
    return st;
  }

  //----------------------------------------------------------------------------
  // Set request verb, target URL
  //----------------------------------------------------------------------------
  CURL* handle = _session->getHandle()->handle;
  CURLM* mhandle = _session->getHandle()->mhandle;

  curl_easy_setopt(handle, CURLOPT_CUSTOMREQUEST, _verb.c_str());
  curl_easy_setopt(handle, CURLOPT_URL, _uri.getString().c_str());

  //----------------------------------------------------------------------------
  // Set up callback to consume response headers
  //----------------------------------------------------------------------------
  curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, header_callback);
  curl_easy_setopt(handle, CURLOPT_HEADERDATA, this);

  //----------------------------------------------------------------------------
  // Set up callback to consume response body
  //----------------------------------------------------------------------------
  curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, write_callback);
  curl_easy_setopt(handle, CURLOPT_WRITEDATA, &_response_buffer);

  //----------------------------------------------------------------------------
  // Set up callback to provide request body
  //----------------------------------------------------------------------------
  if(_content_provider) {
    _content_provider->rewind();
    curl_easy_setopt(handle, CURLOPT_READFUNCTION, read_callback);
    curl_easy_setopt(handle, CURLOPT_READDATA, _content_provider);
  }

  //----------------------------------------------------------------------------
  // Set up CA store
  //----------------------------------------------------------------------------
  if(_params.getSSLCACheck()) {
    for(auto it = _params.listCertificateAuthorityPath().begin(); it != _params.listCertificateAuthorityPath().end(); it++) {
      struct stat st;
      if (stat(it->c_str(), &st) < 0 || S_ISDIR(st.st_mode) == false) {
        DAVIX_SLOG(DAVIX_LOG_WARNING, DAVIX_LOG_HTTP, "CA Path invalid : {}, {} ", *it, ((errno != 0)?strerror(errno):strerror(ENOTDIR)));
        errno = 0;
      } else {
        DAVIX_SLOG(DAVIX_LOG_TRACE, DAVIX_LOG_HTTP, "add CA PATH {}", *it);
        curl_easy_setopt(handle, CURLOPT_CAPATH, it->c_str());
      }
    }
  }
  else {
    curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0);
  }

  //----------------------------------------------------------------------------
  // Set-up headers
  //----------------------------------------------------------------------------
  for(size_t i = 0; i < _headers.size(); i++) {
    _chunklist = curl_slist_append(_chunklist, SSTR(_headers[i].first << ": " << _headers[i].second).c_str());
  }

  curl_easy_setopt(handle, CURLOPT_HTTPHEADER, _chunklist);

  //----------------------------------------------------------------------------
  // Start request
  //----------------------------------------------------------------------------
  int still_running = 1;

  while(still_running != 0) {
    if(curl_multi_perform(mhandle, &still_running) != CURLM_OK) {
      break;
    }
  }

  _state = RequestState::kStarted;

  CURLMsg *msg;
  int msgs_left = 0;
  while(msg = curl_multi_info_read(mhandle, &msgs_left)) {
    if(msg->msg == CURLMSG_DONE && msg->data.result != CURLE_OK) {
      return curlCodeToStatus(msg->data.result);
    }
  }

  return Status();
}

//------------------------------------------------------------------------------
// Major read function - read a block of max_size bytes (at max) into buffer.
//------------------------------------------------------------------------------
dav_ssize_t StandaloneCurlRequest::readBlock(char* buffer, dav_size_t max_size, Status& st) {
  if(!_session) {
    st = Status(davix_scope_http_request(), StatusCode::AlreadyRunning, "Request has not been started yet");
    return -1;
  }

  if(max_size == 0) {
    return 0;
  }

  st = checkTimeout();
  if(!st.ok()) {
    return -1;
  }

  st = Status();
  return _response_buffer.consume(buffer, max_size);
}

//------------------------------------------------------------------------------
{'fraction_non_alphanumeric': 0.38938284518828453, 'fraction_numerical': 0.002981171548117155, 'mean_word_length': 1.2885697187312986, 'pattern_counts': {'":': 4, '<': 80, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4650224', 'n_tokens_mistral': 4618, 'n_tokens_neox': 4443, 'n_words': 1102}