<commit_before>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2014 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include <map>
#include <string>
#include <vector>
#include "burst_serv.hpp"
#include "../framework/mixer/mixer_factory.hpp"
#include "jubatus/core/storage/storage_factory.hpp"

using jubatus::util::lang::lexical_cast;
using jubatus::util::lang::shared_ptr;
using jubatus::util::text::json::json;
using jubatus::core::common::jsonconfig::config_cast_check;
using jubatus::core::burst::burst_options;
using jubatus::core::burst::burst_result;
using jubatus::core::burst::batch_result;
using jubatus::server::framework::server_argv;
using jubatus::server::framework::mixer::create_mixer;

namespace jubatus {
namespace server {

namespace {

struct burst_serv_config {
  std::string method;
  core::common::jsonconfig::config parameter;

  template<typename Ar>
  void serialize(Ar& ar) {
    ar & JUBA_MEMBER(method) & JUBA_MEMBER(parameter);
  }
};

st_window to_st_window(const burst_result& x) {
  st_window result;
  result.start_pos = x.get_start_pos();
  const std::vector<batch_result>& batches = x.get_batches();

  result.batches.reserve(batches.size());
  for (size_t i = 0; i < batches.size(); ++i) {
    st_batch batch;
    batch.d = batches[i].d;
    batch.r = batches[i].r;
    batch.burst_weight = batches[i].burst_weight;
    result.batches.push_back(batch);
  }

  return result;
}

std::map<std::string, st_window>
    to_st_window_map(const core::driver::burst::result_map& x) {
  std::map<std::string, st_window> result;
  for (core::driver::burst::result_map::const_iterator iter = x.begin();
       iter != x.end(); ++iter) {
    result.insert(std::make_pair(iter->first, to_st_window(iter->second)));
  }
  return result;
}

}  // namespace


burst_serv::burst_serv(
    const jubatus::server::framework::server_argv& a,
    const jubatus::util::lang::shared_ptr<
        jubatus::server::common::lock_service>& zk)
    : jubatus::server::framework::server_base(a),
      mixer_(create_mixer(a, zk, rw_mutex())) {
  // somemixable* mi = new somemixable;
  // somemixable_.set_model(mi);
  // get_mixable_holder()->register_mixable(mi);
}

burst_serv::~burst_serv() {
}

void burst_serv::get_status(status_t& status) const {
  burst_->get_status(status);
}

void burst_serv::set_config(const std::string& config) {
  core::common::jsonconfig::config config_root(lexical_cast<json>(config));
  burst_serv_config conf = config_cast_check<burst_serv_config>(config_root);

  config_ = config;

  burst_options options = config_cast_check<burst_options>(conf.parameter);

  burst_.reset(new core::driver::burst(new core::burst::burst(options)));

  LOG(INFO) << "config loaded: " << config;
}

std::string burst_serv::get_config() const {
  return config_;
}

bool burst_serv::add_documents(const std::vector<st_document>& data) {
  size_t processed = 0;
  for (size_t i = 0; i < data.size(); i++) {
    const st_document& doc = data[i];
    if (burst_->add_document(doc.text, doc.pos)) {
      DLOG(INFO) << "add_document done: "
                 << "pos="
                 << doc.pos
                 << ", "
                 << "document="
                 << doc.text;
      ++processed;
    } else {
      DLOG(INFO) << "add_document failed: "
                 << "pos="
                 << doc.pos
                 << ", "
                 << "document="
                 << doc.text;
    }
  }
  if (processed > 0) {
    burst_->calculate_results();
  }<commit_msg>Disable logging on add_documents when successed<commit_after>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2014 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include <map>
#include <string>
#include <vector>
#include "burst_serv.hpp"
#include "../framework/mixer/mixer_factory.hpp"
#include "jubatus/core/storage/storage_factory.hpp"

using jubatus::util::lang::lexical_cast;
using jubatus::util::lang::shared_ptr;
using jubatus::util::text::json::json;
using jubatus::core::common::jsonconfig::config_cast_check;
using jubatus::core::burst::burst_options;
using jubatus::core::burst::burst_result;
using jubatus::core::burst::batch_result;
using jubatus::server::framework::server_argv;
using jubatus::server::framework::mixer::create_mixer;

namespace jubatus {
namespace server {

namespace {

struct burst_serv_config {
  std::string method;
  core::common::jsonconfig::config parameter;

  template<typename Ar>
  void serialize(Ar& ar) {
    ar & JUBA_MEMBER(method) & JUBA_MEMBER(parameter);
  }
};

st_window to_st_window(const burst_result& x) {
  st_window result;
  result.start_pos = x.get_start_pos();
  const std::vector<batch_result>& batches = x.get_batches();

  result.batches.reserve(batches.size());
  for (size_t i = 0; i < batches.size(); ++i) {
    st_batch batch;
    batch.d = batches[i].d;
    batch.r = batches[i].r;
    batch.burst_weight = batches[i].burst_weight;
    result.batches.push_back(batch);
  }

  return result;
}

std::map<std::string, st_window>
    to_st_window_map(const core::driver::burst::result_map& x) {
  std::map<std::string, st_window> result;
  for (core::driver::burst::result_map::const_iterator iter = x.begin();
       iter != x.end(); ++iter) {
    result.insert(std::make_pair(iter->first, to_st_window(iter->second)));
  }
  return result;
}

}  // namespace


burst_serv::burst_serv(
    const jubatus::server::framework::server_argv& a,
    const jubatus::util::lang::shared_ptr<
        jubatus::server::common::lock_service>& zk)
    : jubatus::server::framework::server_base(a),
      mixer_(create_mixer(a, zk, rw_mutex())) {
  // somemixable* mi = new somemixable;
  // somemixable_.set_model(mi);
  // get_mixable_holder()->register_mixable(mi);
}

burst_serv::~burst_serv() {
}

void burst_serv::get_status(status_t& status) const {
  burst_->get_status(status);
}

void burst_serv::set_config(const std::string& config) {
  core::common::jsonconfig::config config_root(lexical_cast<json>(config));
  burst_serv_config conf = config_cast_check<burst_serv_config>(config_root);

  config_ = config;

  burst_options options = config_cast_check<burst_options>(conf.parameter);

  burst_.reset(new core::driver::burst(new core::burst::burst(options)));

  LOG(INFO) << "config loaded: " << config;
}

std::string burst_serv::get_config() const {
  return config_;
}

bool burst_serv::add_documents(const std::vector<st_document>& data) {
  size_t processed = 0;
  for (size_t i = 0; i < data.size(); i++) {
    const st_document& doc = data[i];
    if (burst_->add_document(doc.text, doc.pos)) {
      ++processed;
    } else {
      DLOG(INFO) << "add_document failed: "
                 << "pos="
                 << doc.pos
                 << ", "
                 << "document="
                 << doc.text;
    }
  }
  if (processed > 0) {
    burst_->calculate_results();
  }
{'fraction_non_alphanumeric': 0.13822403063666827, 'fraction_numerical': 0.005026328386787937, 'mean_word_length': 4.142769230769231, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1349256', 'n_tokens_mistral': 2743, 'n_tokens_neox': 2642, 'n_words': 630}