<commit_before>/*
 * Copyright (c) 2017 Cryptonomex, Inc., and contributors.
 *
 * The MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <graphene/es_objects/es_objects.hpp>

//#include <graphene/app/impacted.hpp>

//#include <graphene/chain/account_evaluator.hpp>
//#include <graphene/chain/account_object.hpp>
#include <graphene/chain/config.hpp>
#include <graphene/chain/database.hpp>
//#include <graphene/chain/evaluator.hpp>
//#include <graphene/chain/operation_history_object.hpp>
//#include <graphene/chain/transaction_evaluation_state.hpp>

#include <fc/smart_ref_impl.hpp>
#include <fc/thread/thread.hpp>

#include <curl/curl.h>
#include <boost/algorithm/string/replace.hpp>
#include <graphene/chain/proposal_object.hpp>
#include <boost/algorithm/string/join.hpp>
#include <graphene/chain/balance_object.hpp>


namespace graphene { namespace es_objects {

namespace detail
{

class es_objects_plugin_impl
{
   public:
      es_objects_plugin_impl(es_objects_plugin& _plugin)
         : _self( _plugin )
      {  curl = curl_easy_init(); }
      virtual ~es_objects_plugin_impl();

      void updateDatabase( const vector<object_id_type>& ids , bool isNew);

      graphene::chain::database& database()
      {
         return _self.database();
      }

      es_objects_plugin& _self;
      std::string _es_objects_elasticsearch_url = "http://localhost:9200/";
      uint32_t _es_objects_bulk_replay = 1000;
      uint32_t _es_objects_bulk_sync = 2;
      bool _es_objects_proposals = true;
      bool _es_objects_accounts = true;
      bool _es_objects_assets = true;
      bool _es_objects_balances = true;
      bool _es_objects_logs = true;
      CURL *curl; // curl handler
      vector <string> bulk;
   private:
      void PrepareProposal(const proposal_object* proposal_object);
      void PrepareAccount(const account_object* account_object);
      void PrepareAsset(const asset_object* asset_object);
      void PrepareBalance(const balance_object* balance_object);
      void SendBulk();
      void createBulk(std::string type, std::string data, std::string id);
};

void es_objects_plugin_impl::updateDatabase( const vector<object_id_type>& ids , bool isNew)
{

   graphene::chain::database &db = database();

   auto block_time = db.head_block_time();

   // check if we are in replay or in sync and change number of bulk documents accordingly
   uint32_t limit_documents = 0;
   if((fc::time_point::now() - block_time) < fc::seconds(30))
      limit_documents = _es_objects_bulk_sync;
   else
      limit_documents = _es_objects_bulk_replay;

   if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech
      SendBulk();
   }

   for(auto const& value: ids) {
      if(value.space() == 1 && value.type() == 10 && _es_objects_proposals) {
         auto obj = db.find_object(value);
         auto p = static_cast<const proposal_object*>(obj);
         PrepareProposal(p);
      }
      else if(value.space() == 1 && value.type() == 2 && _es_objects_accounts) {
         auto obj = db.find_object(value);
         auto a = static_cast<const account_object*>(obj);
         PrepareAccount(a);
      }
      else if(value.space() == 1 && value.type() == 3 && _es_objects_assets) {
         auto obj = db.find_object(value);
         auto a = static_cast<const asset_object*>(obj);
         PrepareAsset(a);
      }
      else if(value.space() == 1 && value.type() == 15 && _es_objects_balances) {
         auto obj = db.find_object(value);
         auto b = static_cast<const balance_object*>(obj);
         PrepareBalance(b);
      }
   }
}

void es_objects_plugin_impl::createBulk(std::string type, std::string data, std::string id)
{
   bulk.push_back("{ \"index\" : { \"_index\" : \"bitshares-"+type+"\", \"_type\" : \"data\", \"_id\" : "+id+" } }");
   bulk.push_back(data);
}
<commit_msg>remove magic numbers and some commented includes<commit_after>/*
 * Copyright (c) 2017 Cryptonomex, Inc., and contributors.
 *
 * The MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <graphene/es_objects/es_objects.hpp>

#include <graphene/chain/config.hpp>
#include <graphene/chain/database.hpp>

#include <fc/smart_ref_impl.hpp>
#include <fc/thread/thread.hpp>

#include <curl/curl.h>
#include <boost/algorithm/string/replace.hpp>
#include <graphene/chain/proposal_object.hpp>
#include <boost/algorithm/string/join.hpp>
#include <graphene/chain/balance_object.hpp>


namespace graphene { namespace es_objects {

namespace detail
{

class es_objects_plugin_impl
{
   public:
      es_objects_plugin_impl(es_objects_plugin& _plugin)
         : _self( _plugin )
      {  curl = curl_easy_init(); }
      virtual ~es_objects_plugin_impl();

      void updateDatabase( const vector<object_id_type>& ids , bool isNew);

      graphene::chain::database& database()
      {
         return _self.database();
      }

      es_objects_plugin& _self;
      std::string _es_objects_elasticsearch_url = "http://localhost:9200/";
      uint32_t _es_objects_bulk_replay = 1000;
      uint32_t _es_objects_bulk_sync = 2;
      bool _es_objects_proposals = true;
      bool _es_objects_accounts = true;
      bool _es_objects_assets = true;
      bool _es_objects_balances = true;
      bool _es_objects_logs = true;
      CURL *curl; // curl handler
      vector <string> bulk;
   private:
      void PrepareProposal(const proposal_object* proposal_object);
      void PrepareAccount(const account_object* account_object);
      void PrepareAsset(const asset_object* asset_object);
      void PrepareBalance(const balance_object* balance_object);
      void SendBulk();
      void createBulk(std::string type, std::string data, std::string id);
};

void es_objects_plugin_impl::updateDatabase( const vector<object_id_type>& ids , bool isNew)
{

   graphene::chain::database &db = database();

   auto block_time = db.head_block_time();

   // check if we are in replay or in sync and change number of bulk documents accordingly
   uint32_t limit_documents = 0;
   if((fc::time_point::now() - block_time) < fc::seconds(30))
      limit_documents = _es_objects_bulk_sync;
   else
      limit_documents = _es_objects_bulk_replay;

   if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech
      SendBulk();
   }

   for(auto const& value: ids) {
      if(value.is<proposal_object>() && _es_objects_proposals) {
         auto obj = db.find_object(value);
         auto p = static_cast<const proposal_object*>(obj);
         PrepareProposal(p);
      }
      else if(value.is<account_object>() && _es_objects_accounts) {
         auto obj = db.find_object(value);
         auto a = static_cast<const account_object*>(obj);
         PrepareAccount(a);
      }
      else if(value.is<asset_object>() && _es_objects_assets) {
         auto obj = db.find_object(value);
         auto a = static_cast<const asset_object*>(obj);
         PrepareAsset(a);
      }
      else if(value.is<balance_object>() && _es_objects_balances) {
         auto obj = db.find_object(value);
         auto b = static_cast<const balance_object*>(obj);
         PrepareBalance(b);
      }
   }
}

void es_objects_plugin_impl::createBulk(std::string type, std::string data, std::string id)
{
   bulk.push_back("{ \"index\" : { \"_index\" : \"bitshares-"+type+"\", \"_type\" : \"data\", \"_id\" : "+id+" } }");
   bulk.push_back(data);
}

{'fraction_non_alphanumeric': 0.10834122225728411, 'fraction_numerical': 0.005680025244556642, 'mean_word_length': 3.6516634050880628, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2879217', 'n_tokens_mistral': 2988, 'n_tokens_neox': 2862, 'n_words': 828}