<commit_before>//
// sync_client.cpp
// ~~~~~~~~~~~~~~~
//
// Copyright (c) 2003-2012 <NAME> (chris at kohlhoff dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//

#include <iostream>
#include <istream>
#include <ostream>
#include <string>
#include <regex>
#include <boost/algorithm/string.hpp>
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include <fc/variant.hpp>
#include <fc/io/json.hpp>
#include <eosio/chain/exceptions.hpp>
#include <eosio/http_plugin/http_plugin.hpp>
#include <eosio/chain_plugin/chain_plugin.hpp>
#include "httpc.hpp"

using boost::asio::ip::tcp;
namespace eosio { namespace client { namespace http {
   void do_connect(tcp::socket& sock, const std::string& server, const std::string& port) {
      // Get a list of endpoints corresponding to the server name.
      tcp::resolver resolver(sock.get_io_service());
      tcp::resolver::query query(server, port);
      boost::asio::connect(sock, resolver.resolve(query));
   }

   template<class T>
   std::string do_txrx(T& socket, boost::asio::streambuf& request_buff, unsigned int& status_code) {
      // Send the request.
      boost::asio::write(socket, request_buff);

      // Read the response status line. The response streambuf will automatically
      // grow to accommodate the entire line. The growth may be limited by passing
      // a maximum size to the streambuf constructor.
      boost::asio::streambuf response;
      boost::asio::read_until(socket, response, "\r\n");

      // Check that response is OK.
      std::istream response_stream(&response);
      std::string http_version;
      response_stream >> http_version;
      response_stream >> status_code;
      std::string status_message;
      std::getline(response_stream, status_message);
      FC_ASSERT( !(!response_stream || http_version.substr(0, 5) != "HTTP/"), "Invalid Response" );

      // Read the response headers, which are terminated by a blank line.
      boost::asio::read_until(socket, response, "\r\n\r\n");

      // Process the response headers.
      std::string header;
      int response_content_length = -1;
      std::regex clregex(R"xx(^content-length:\s+(\d+))xx", std::regex_constants::icase);
      while (std::getline(response_stream, header) && header != "\r") {
         std::smatch match;
         if(std::regex_search(header, match, clregex))
            response_content_length = std::stoi(match[1]);
      }
      FC_ASSERT(response_content_length >= 0, "Invalid content-length response");

      std::stringstream re;
      // Write whatever content we already have to output.
      response_content_length -= response.size();
      if (response.size() > 0)
         re << &response;

      boost::asio::read(socket, response, boost::asio::transfer_exactly(response_content_length));
      re << &response;

      return re.str();
   }

   parsed_url parse_url( const string& server_url ) {
      parsed_url res;

      //via rfc3986 and modified a bit to suck out the port number
      //Sadly this doesn't work for ipv6 addresses
      std::regex rgx(R"xx(^(([^:/?#]+):)?(//([^:/?#]*)(:(\d+))?)?([^?#]*)(\?([^#]*))?(#(.*))?)xx");
      std::smatch match;
      if(std::regex_search(server_url.begin(), server_url.end(), match, rgx)) {
         res.scheme = match[2];
         res.server = match[4];
         res.port = match[6];
         res.path_prefix = match[7];
      }
      if(res.scheme != "http" && res.scheme != "https")
         FC_THROW("Unrecognized URL scheme (${s}) in URL \"${u}\"", ("s", res.scheme)("u", server_url));
      if(res.server.empty())
         FC_THROW("No server parsed from URL \"${u}\"", ("u", server_url));
      if(res.port.empty())
         res.port = res.scheme == "http" ? "80" : "443";
      boost::trim_right_if(res.path_prefix, boost::is_any_of("/"));
      return res;
   }

   fc::variant do_http_call( const connection_param& cp,
                             const fc::variant& postdata,
                             bool print_request,
                             bool print_response ) {
   std::string postjson;
   if( !postdata.is_null() ) {
      postjson = print_request ? fc::json::to_pretty_string( postdata ) : fc::json::to_string( postdata );
   }

   boost::asio::io_service io_service;

   const string& server_url = cp.url;
   const string& path = cp.path;

   auto url = parse_url( server_url );

   boost::asio::streambuf request;
   std::ostream request_stream(&request);
   request_stream << "POST " << url.path_prefix + path << " HTTP/1.0\r\n";
   request_stream << "Host: " << url.server << "\r\n";
   request_stream << "content-length: " << postjson.size() << "\r\n";
   request_stream << "Accept: */*\r\n";
   request_stream << "Connection: close\r\n";
   request_stream << "\r\n";
   // append more customized headers
   std::vector<string>::iterator itr;
   for (itr = cp.headers.begin(); itr != cp.headers.end(); itr++) {
      request_stream << *itr << "\r\n";
   }
   request_stream << postjson;

   if ( print_request ) {
      string s(request.size(), '\0');
      buffer_copy(boost::asio::buffer(s), request.data());
      std::cerr << "REQUEST:" << std::endl
                << "---------------------" << std::endl
                << s << std::endl
                << "---------------------" << std::endl;
   }

   unsigned int status_code;
   std::string re;

   if(url.scheme == "http") {
      tcp::socket socket(io_service);
      do_connect(socket, url.server, url.port);
      re = do_txrx(socket, request, status_code);
   }
   else { //https
      boost::asio::ssl::context ssl_context(boost::asio::ssl::context::sslv23_client);
#if defined( __APPLE__ )
      //TODO: this is undocumented/not supported; fix with keychain based approach
      ssl_context.load_verify_file("/private/etc/ssl/cert.pem");
#elif defined( _WIN32 )
      FC_THROW("HTTPS on Windows not supported");
#else
      ssl_context.set_default_verify_paths();
#endif

      boost::asio::ssl::stream<boost::asio::ip::tcp::socket> socket(io_service, ssl_context);
      if(cp.verify_cert)
         socket.set_verify_mode(boost::asio::ssl::verify_peer);

      do_connect(socket.next_layer(), url.server, url.port);
      socket.handshake(boost::asio::ssl::stream_base::client);
      re = do_txrx(socket, request, status_code);
      //try and do a clean shutdown; but swallow if this fails (other side could have already gave TCP the ax)
      try {socket.shutdown();} catch(...) {}
   }

   const auto response_result = fc::json::from_string(re);
   if( print_response ) {
      std::cerr << "RESPONSE:" << std::endl
                << "---------------------" << std::endl
                << fc::json::to_pretty_string( response_result ) << std::endl
                << "---------------------" << std::endl;
   }
   if( status_code == 200 || status_code == 201 || status_code == 202 ) {
      return response_result;
   } else if( status_code == 404 ) {
      // Unknown endpoint
      if (path.compare(0, chain_func_base.size(), chain_func_base) == 0) {
         throw chain::missing_chain_api_plugin_exception(FC_LOG_MESSAGE(error, "Chain API plugin is not enabled"));
      } else if (path.compare(0, wallet_func_base.size(), wallet_func_base) == 0) {
         throw chain::missing_wallet_api_plugin_exception(FC_LOG_MESSAGE(error, "Wallet is not available"));<commit_msg>Add SNI to HTTPS requests in cleos<commit_after>//
// sync_client.cpp
// ~~~~~~~~~~~~~~~
//
// Copyright (c) 2003-2012 <NAME> (chris at kohlhoff dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//

#include <iostream>
#include <istream>
#include <ostream>
#include <string>
#include <regex>
#include <boost/algorithm/string.hpp>
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include <fc/variant.hpp>
#include <fc/io/json.hpp>
#include <eosio/chain/exceptions.hpp>
#include <eosio/http_plugin/http_plugin.hpp>
#include <eosio/chain_plugin/chain_plugin.hpp>
#include "httpc.hpp"

using boost::asio::ip::tcp;
namespace eosio { namespace client { namespace http {
   void do_connect(tcp::socket& sock, const std::string& server, const std::string& port) {
      // Get a list of endpoints corresponding to the server name.
      tcp::resolver resolver(sock.get_io_service());
      tcp::resolver::query query(server, port);
      boost::asio::connect(sock, resolver.resolve(query));
   }

   template<class T>
   std::string do_txrx(T& socket, boost::asio::streambuf& request_buff, unsigned int& status_code) {
      // Send the request.
      boost::asio::write(socket, request_buff);

      // Read the response status line. The response streambuf will automatically
      // grow to accommodate the entire line. The growth may be limited by passing
      // a maximum size to the streambuf constructor.
      boost::asio::streambuf response;
      boost::asio::read_until(socket, response, "\r\n");

      // Check that response is OK.
      std::istream response_stream(&response);
      std::string http_version;
      response_stream >> http_version;
      response_stream >> status_code;
      std::string status_message;
      std::getline(response_stream, status_message);
      FC_ASSERT( !(!response_stream || http_version.substr(0, 5) != "HTTP/"), "Invalid Response" );

      // Read the response headers, which are terminated by a blank line.
      boost::asio::read_until(socket, response, "\r\n\r\n");

      // Process the response headers.
      std::string header;
      int response_content_length = -1;
      std::regex clregex(R"xx(^content-length:\s+(\d+))xx", std::regex_constants::icase);
      while (std::getline(response_stream, header) && header != "\r") {
         std::smatch match;
         if(std::regex_search(header, match, clregex))
            response_content_length = std::stoi(match[1]);
      }
      FC_ASSERT(response_content_length >= 0, "Invalid content-length response");

      std::stringstream re;
      // Write whatever content we already have to output.
      response_content_length -= response.size();
      if (response.size() > 0)
         re << &response;

      boost::asio::read(socket, response, boost::asio::transfer_exactly(response_content_length));
      re << &response;

      return re.str();
   }

   parsed_url parse_url( const string& server_url ) {
      parsed_url res;

      //via rfc3986 and modified a bit to suck out the port number
      //Sadly this doesn't work for ipv6 addresses
      std::regex rgx(R"xx(^(([^:/?#]+):)?(//([^:/?#]*)(:(\d+))?)?([^?#]*)(\?([^#]*))?(#(.*))?)xx");
      std::smatch match;
      if(std::regex_search(server_url.begin(), server_url.end(), match, rgx)) {
         res.scheme = match[2];
         res.server = match[4];
         res.port = match[6];
         res.path_prefix = match[7];
      }
      if(res.scheme != "http" && res.scheme != "https")
         FC_THROW("Unrecognized URL scheme (${s}) in URL \"${u}\"", ("s", res.scheme)("u", server_url));
      if(res.server.empty())
         FC_THROW("No server parsed from URL \"${u}\"", ("u", server_url));
      if(res.port.empty())
         res.port = res.scheme == "http" ? "80" : "443";
      boost::trim_right_if(res.path_prefix, boost::is_any_of("/"));
      return res;
   }

   fc::variant do_http_call( const connection_param& cp,
                             const fc::variant& postdata,
                             bool print_request,
                             bool print_response ) {
   std::string postjson;
   if( !postdata.is_null() ) {
      postjson = print_request ? fc::json::to_pretty_string( postdata ) : fc::json::to_string( postdata );
   }

   boost::asio::io_service io_service;

   const string& server_url = cp.url;
   const string& path = cp.path;

   auto url = parse_url( server_url );

   boost::asio::streambuf request;
   std::ostream request_stream(&request);
   request_stream << "POST " << url.path_prefix + path << " HTTP/1.0\r\n";
   request_stream << "Host: " << url.server << "\r\n";
   request_stream << "content-length: " << postjson.size() << "\r\n";
   request_stream << "Accept: */*\r\n";
   request_stream << "Connection: close\r\n";
   request_stream << "\r\n";
   // append more customized headers
   std::vector<string>::iterator itr;
   for (itr = cp.headers.begin(); itr != cp.headers.end(); itr++) {
      request_stream << *itr << "\r\n";
   }
   request_stream << postjson;

   if ( print_request ) {
      string s(request.size(), '\0');
      buffer_copy(boost::asio::buffer(s), request.data());
      std::cerr << "REQUEST:" << std::endl
                << "---------------------" << std::endl
                << s << std::endl
                << "---------------------" << std::endl;
   }

   unsigned int status_code;
   std::string re;

   if(url.scheme == "http") {
      tcp::socket socket(io_service);
      do_connect(socket, url.server, url.port);
      re = do_txrx(socket, request, status_code);
   }
   else { //https
      boost::asio::ssl::context ssl_context(boost::asio::ssl::context::sslv23_client);
#if defined( __APPLE__ )
      //TODO: this is undocumented/not supported; fix with keychain based approach
      ssl_context.load_verify_file("/private/etc/ssl/cert.pem");
#elif defined( _WIN32 )
      FC_THROW("HTTPS on Windows not supported");
#else
      ssl_context.set_default_verify_paths();
#endif

      boost::asio::ssl::stream<boost::asio::ip::tcp::socket> socket(io_service, ssl_context);
      SSL_set_tlsext_host_name(socket.native_handle(), url.server.c_str());
      if(cp.verify_cert)
         socket.set_verify_mode(boost::asio::ssl::verify_peer);

      do_connect(socket.next_layer(), url.server, url.port);
      socket.handshake(boost::asio::ssl::stream_base::client);
      re = do_txrx(socket, request, status_code);
      //try and do a clean shutdown; but swallow if this fails (other side could have already gave TCP the ax)
      try {socket.shutdown();} catch(...) {}
   }

   const auto response_result = fc::json::from_string(re);
   if( print_response ) {
      std::cerr << "RESPONSE:" << std::endl
                << "---------------------" << std::endl
                << fc::json::to_pretty_string( response_result ) << std::endl
                << "---------------------" << std::endl;
   }
   if( status_code == 200 || status_code == 201 || status_code == 202 ) {
      return response_result;
   } else if( status_code == 404 ) {
      // Unknown endpoint
      if (path.compare(0, chain_func_base.size(), chain_func_base) == 0) {
         throw chain::missing_chain_api_plugin_exception(FC_LOG_MESSAGE(error, "Chain API plugin is not enabled"));
      } else if (path.compare(0, wallet_func_base.size(), wallet_func_base) == 0) {
         throw chain::missing_wallet_api_plugin_exception(FC_LOG_MESSAGE(error, "Wallet is not available"));
{'fraction_non_alphanumeric': 0.16914659371455262, 'fraction_numerical': 0.0076065923800627214, 'mean_word_length': 3.0225442834138487, 'pattern_counts': {'":': 0, '<': 169, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2066112', 'n_tokens_mistral': 4838, 'n_tokens_neox': 4651, 'n_words': 1071}