<commit_before>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2014 Preferred Networks and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include <fenv.h>

#include <vector>
#include <string>
#include <set>
#include <utility>
#include <gtest/gtest.h>

#include "clustering.hpp"

#include "jubatus/util/lang/shared_ptr.h"
#include "jubatus/util/math/random.h"
#include "jubatus/util/lang/cast.h"
#include "../clustering/clustering_config.hpp"
#include "../clustering/clustering.hpp"
#include "../clustering/kmeans_clustering_method.hpp"
#include "../clustering/gmm_clustering_method.hpp"
#include "../framework/stream_writer.hpp"
#include "test_util.hpp"
#include "../fv_converter/datum.hpp"

using std::vector;
using std::string;
using std::pair;
using std::set;
using std::make_pair;
using jubatus::util::lang::shared_ptr;
using jubatus::util::lang::lexical_cast;
using jubatus::core::fv_converter::datum;
using jubatus::core::fv_converter::datum_to_fv_converter;
using jubatus::core::clustering::clustering_method;
using jubatus::core::clustering::clustering_config;

namespace jubatus {
namespace core {
namespace driver {

class clustering_test
    : public ::testing::TestWithParam<pair<string, string> > {
 protected:
  shared_ptr<driver::clustering> create_driver() const {
    pair<string, string> param = GetParam();
    return shared_ptr<driver::clustering>(
        new driver::clustering(
            shared_ptr<core::clustering::clustering>(
                new core::clustering::clustering("dummy",
                                                 param.second,
                                                 conf_)),
            make_fv_converter()));
  }
  void SetUp() {
    feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW);
    pair<string, string> param = GetParam();
    conf_.k = 2;
    conf_.compressor_method = param.first;
    conf_.bucket_size = 200;
    conf_.compressed_bucket_size = conf_.bucket_size / 10;
    conf_.bicriteria_base_size = conf_.bucket_size / 100;
    conf_.bucket_length = 2;
    conf_.forgetting_factor = 0.0;
    conf_.forgetting_threshold = 0.5;
    clustering_ = create_driver();
  }
  void TearDown() {
    clustering_.reset();
  }
  shared_ptr<driver::clustering> clustering_;
  clustering_config conf_;
};

namespace {  // testing util
datum single_datum(string key, double v) {
  datum d;
  d.num_values_.push_back(make_pair(key, v));
  return d;
}
}

TEST_P(clustering_test, get_revision) {
  const int num = conf_.bucket_size * 10;
  for (int i = 0; i < num; ++i) {
    vector<datum> datums;
    datums.push_back(single_datum("a", 1));
    clustering_->push(datums);
  }
  ASSERT_EQ(num / conf_.bucket_size, clustering_->get_revision());
}

TEST_P(clustering_test, push) {
  for (int j = 0; j < conf_.bucket_size / 5; ++j) {
    vector<datum> datums;
    for (int i = 0; i < 100; i += 5) {
      datums.push_back(single_datum("a", i * 2));
      datums.push_back(single_datum("b", i * 100));
    }
    clustering_->push(datums);
  }
}

TEST_P(clustering_test, save_load) {
  {
    core::fv_converter::datum d;
    vector<datum> datums;
    datums.push_back(single_datum("a", 1));
    clustering_->push(datums);
  }

  // save to a buffer
  msgpack::sbuffer sbuf;
  framework::stream_writer<msgpack::sbuffer> st(sbuf);
  framework::jubatus_packer jp(st);
  framework::packer packer(jp);
  clustering_->pack(packer);

  // restart the driver
  TearDown();
  SetUp();

  // unpack the buffer
  msgpack::unpacked unpacked;
  msgpack::unpack(&unpacked, sbuf.data(), sbuf.size());
  clustering_->unpack(unpacked.get());
}

TEST_P(clustering_test, get_k_center) {
  jubatus::util::math::random::mtrand r(0);
  vector<datum> one;
  vector<datum> two;

  for (int j = 0; j < conf_.bucket_size / 2; ++j) {
    datum a, b;
    a.num_values_.push_back(make_pair("a", -10 + r.next_gaussian() * 20));
    a.num_values_.push_back(make_pair("b", -200 + r.next_gaussian() * 400));
    b.num_values_.push_back(make_pair("c", -50 + r.next_gaussian() * 100));
    b.num_values_.push_back(make_pair("d", -25 + r.next_gaussian() * 50));
    one.push_back(a);
    two.push_back(b);
  }
  clustering_->push(one);
  clustering_->push(two);

  clustering_->do_clustering();
  {
    vector<datum> result = clustering_->get_k_center();
    ASSERT_EQ(conf_.k, result.size());
    ASSERT_LT(1, result[0].num_values_.size());
    if (result[0].num_values_[0].first == "a"
        || result[0].num_values_[0].first == "b") {
      // result[0] is {"a":xx, "b":yy} cluster
      if (result[0].num_values_[0].first == "a") {
        ASSERT_EQ("b", result[0].num_values_[1].first);
      } else {
        ASSERT_EQ("a", result[0].num_values_[1].first);
      }
    } else {
      // result[1] is {"a":xx, "b":yy} cluster
      if (result[1].num_values_[0].first == "a") {
        ASSERT_EQ("b", result[1].num_values_[1].first);
      } else {
        ASSERT_EQ("a", result[1].num_values_[1].first);
      }
    }
  }
}

TEST_P(clustering_test, integer_center) {
  jubatus::util::math::random::mtrand r(0);
  const int quantity = conf_.bucket_size * 5;
  std::vector<datum> data(quantity);

  for (int i = 0; i < quantity ; ++i) {
    data[i].num_values_.push_back(make_pair("x", 100 + r.next_int(-10, 10)));
  }

  clustering_->push(data);
  const std::vector<fv_converter::datum> centers = clustering_->get_k_center();

  ASSERT_EQ(conf_.k, centers.size());
  /*  debug out
  for (size_t i = 0; i < centers.size(); ++i) {
    std::cout << i << " :[";
    for (size_t j = 0; j < centers[i].num_values_.size(); ++j) {
      std::cout
        << centers[i].num_values_[j].first << " => "
        << centers[i].num_values_[j].second << ", ";
    }
    std::cout << "]" << std::endl;;
  }
  //*/
}

struct check_points {
  float a;
  float b;
  check_points(float a_, float b_)
    : a(a_), b(b_) {}
  bool operator<(const check_points& rhs) const {
    if (a < rhs.a) {
      return true;
    } else if (rhs.a < a) {
      return false;
    } else if (b < rhs.b) {
      return true;
    } else {
      return false;
    }
  }
};

struct check_point_compare {
  bool operator()(const check_points& lhs, const check_points& rhs) {
    return lhs < rhs;
  }
};


TEST_P(clustering_test, get_nearest_members) {
  jubatus::util::math::random::mtrand r(0);
  vector<datum> one;
  vector<datum> two;

  set<check_points, check_point_compare> points;

  for (int i = 0; i < conf_.bucket_size * 2 + 1; ++i) {
    datum x, y;
    float a = 10 + r.next_gaussian() * 20;
    float b = 1000 + r.next_gaussian() * 400;
    points.insert(check_points(a, b));

    x.num_values_.push_back(make_pair("a", a));
    x.num_values_.push_back(make_pair("b", b));
    y.num_values_.push_back(make_pair("c", -500000 - r.next_gaussian() * 100));
    y.num_values_.push_back(make_pair("d", -10000 - r.next_gaussian() * 50));
    one.push_back(x);
    two.push_back(y);
  }
  clustering_->push(one);
  clustering_->push(two);

  clustering_->do_clustering();

  {
    vector<datum> result = clustering_->get_k_center();
    ASSERT_EQ(conf_.k, result.size());
  }

  set<check_points, check_point_compare>::const_iterator it;
  for (it = points.begin(); it != points.end(); ++it) {
    datum x;
    x.num_values_.push_back(make_pair("a", it->a));
    x.num_values_.push_back(make_pair("b", it->b));
    core::clustering::cluster_unit result =
        clustering_->get_nearest_members(x);

    ASSERT_LT(1, result.size());
    for (size_t i = 0; i < result.size(); ++i) {
      const vector<pair<string, double> >& near_points =
          result[i].second.num_values_;
      ASSERT_EQ(2, near_points.size());  // must be 2-dimentional
      ASSERT_EQ("a", near_points[0].first);
      ASSERT_EQ("b", near_points[1].first);
      ASSERT_NE(points.end(),
                points.find(check_points(near_points[0].second,
                                         near_points[1].second)));
    }
  }
}



TEST_P(clustering_test, get_nearest_center) {
  jubatus::util::math::random::mtrand r(0);
  vector<datum> one;
  vector<datum> two;

  for (int i = 0; i < conf_.bucket_size * 2; ++i) {
    datum x, y;
    x.num_values_.push_back(make_pair("a", 10 + r.next_gaussian() * 20));
    x.num_values_.push_back(make_pair("b", 1000 + r.next_gaussian() * 400));
    y.num_values_.push_back(make_pair("c", -500000 - r.next_gaussian() * 100));
    y.num_values_.push_back(make_pair("d", -10000 - r.next_gaussian() * 50));
    one.push_back(x);
    two.push_back(y);
  }
  clustering_->push(one);
  clustering_->push(two);

  clustering_->do_clustering();

  {
    vector<datum> result = clustering_->get_k_center();
    ASSERT_EQ(conf_.k, result.size());
  }

  for (int i = 0; i < 100; ++i) {
    {
      datum x;
      x.num_values_.push_back(make_pair("a", 10 + r.next_gaussian() * 20));
      x.num_values_.push_back(make_pair("b", 1000 + r.next_gaussian() * 400));
      datum expect_near_x = clustering_->get_nearest_center(x);
      const vector<pair<string, double> >& result = expect_near_x.num_values_;
      for (size_t i = 0; i < result.size(); ++i) {
        // Check if the result is belonging to the same cluster.
        // Difference of the value and the mean of its distribution are expected
        // to be lesser than 2-sigma.
        if (result[i].first == "a") {
          const double diff = std::abs(result[i].second - 10);
          ASSERT_GT(20 * 2, diff);
        } else if (result[i].first == "b") {
          const double diff = std::abs(result[i].second - 1000);
          ASSERT_GT(400 * 2, diff);
        }
      }
    }
  }
}

TEST_P(clustering_test, empty_mix) {
  framework::linear_mixable* clustering_mixable =
    dynamic_cast<framework::linear_mixable*>(clustering_->get_mixable());
  shared_ptr<driver::clustering> other = create_driver();
  framework::linear_mixable* other_mixable =
    dynamic_cast<framework::linear_mixable*>(other->get_mixable());
  ASSERT_TRUE(clustering_mixable);
  ASSERT_TRUE(other_mixable);

  msgpack::sbuffer data;
  {
    core::framework::stream_writer<msgpack::sbuffer> st(data);
    core::framework::jubatus_packer jp(st);
    core::framework::packer pk(jp);
    clustering_mixable->get_diff(pk);
  }
  {
    msgpack::sbuffer sbuf;
    core::framework::stream_writer<msgpack::sbuffer> st(sbuf);
    core::framework::jubatus_packer jp(st);
    core::framework::packer pk(jp);
    other_mixable->get_diff(pk);

    msgpack::unpacked msg;
    msgpack::unpack(&msg, sbuf.data(), sbuf.size());
    framework::diff_object diff = other_mixable->convert_diff_object(msg.get());

    msgpack::unpacked data_msg;
    msgpack::unpack(&data_msg, data.data(), data.size());

    other_mixable->mix(data_msg.get(), diff);
    other_mixable->put_diff(diff);
  }
}

vector<pair<string, string> > parameter_list() {
  vector<pair<string, string> > ret;
  ret.push_back(make_pair("simple", "kmeans"));
  ret.push_back(make_pair("compressive_kmeans", "kmeans"));
#ifdef JUBATUS_USE_EIGEN
  ret.push_back(make_pair("simple", "gmm"));
  ret.push_back(make_pair("compressive_gmm", "gmm"));
#endif
  return ret;
}

INSTANTIATE_TEST_CASE_P(clustering_test_instance,
                        clustering_test,
                        testing::ValuesIn(parameter_list()));



class clustering_with_idf_test
    : public ::testing::TestWithParam<pair<string, string> > {
 protected:
  shared_ptr<driver::clustering> create_driver() const {
    pair<string, string> param = GetParam();
    clustering_config conf;
    conf.compressor_method = param.first;
    return shared_ptr<driver::clustering>(
        new driver::clustering(
            shared_ptr<core::clustering::clustering>(
                new core::clustering::clustering("dummy",
                                                 param.second,
                                                 conf)),
            make_tf_idf_fv_converter()));
  }
  void SetUp() {
    pair<string, string> param = GetParam();
    clustering_config conf;
    conf.compressor_method = param.first;
    conf.bucket_size = 50;
    conf.bicriteria_base_size = 5;
    conf.compressed_bucket_size = 10;
    clustering_ = create_driver();
  }
  void TearDown() {
    clustering_.reset();
  }
  shared_ptr<driver::clustering> clustering_;
};

fv_converter::datum create_datum_str(const string& key, const string& value) {
  fv_converter::datum d;
  d.string_values_.push_back(make_pair(key, value));
  return d;
}

TEST_P(clustering_with_idf_test, get_nearest_members) {
  jubatus::util::math::random::mtrand r(0);
  vector<datum> one;
  vector<datum> two;

  for (int i = 0; i < 200 ; ++i) {
    const string i_str = lexical_cast<string>(i);
    one.push_back(create_datum_str("a" + i_str, "x y z " + i_str));
    two.push_back(create_datum_str("a" + i_str, "y z w " + i_str));
  }
  clustering_->push(one);
  clustering_->push(two);

  clustering_->do_clustering();

  {
    vector<datum> result = clustering_->get_k_center();
    ASSERT_EQ(2u, result.size());
  }
}

INSTANTIATE_TEST_CASE_P(clustering_with_idf_test_instance,
                        clustering_with_idf_test,
                        testing::ValuesIn(parameter_list()));

}  // namespace driver
}  // namespace core
}  // namespace jubatus
<commit_msg>remove feenableexcept<commit_after>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2014 Preferred Networks and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include <fenv.h>

#include <vector>
#include <string>
#include <set>
#include <utility>
#include <gtest/gtest.h>

#include "clustering.hpp"

#include "jubatus/util/lang/shared_ptr.h"
#include "jubatus/util/math/random.h"
#include "jubatus/util/lang/cast.h"
#include "../clustering/clustering_config.hpp"
#include "../clustering/clustering.hpp"
#include "../clustering/kmeans_clustering_method.hpp"
#include "../clustering/gmm_clustering_method.hpp"
#include "../framework/stream_writer.hpp"
#include "test_util.hpp"
#include "../fv_converter/datum.hpp"

using std::vector;
using std::string;
using std::pair;
using std::set;
using std::make_pair;
using jubatus::util::lang::shared_ptr;
using jubatus::util::lang::lexical_cast;
using jubatus::core::fv_converter::datum;
using jubatus::core::fv_converter::datum_to_fv_converter;
using jubatus::core::clustering::clustering_method;
using jubatus::core::clustering::clustering_config;

namespace jubatus {
namespace core {
namespace driver {

class clustering_test
    : public ::testing::TestWithParam<pair<string, string> > {
 protected:
  shared_ptr<driver::clustering> create_driver() const {
    pair<string, string> param = GetParam();
    return shared_ptr<driver::clustering>(
        new driver::clustering(
            shared_ptr<core::clustering::clustering>(
                new core::clustering::clustering("dummy",
                                                 param.second,
                                                 conf_)),
            make_fv_converter()));
  }
  void SetUp() {
    pair<string, string> param = GetParam();
    conf_.k = 2;
    conf_.compressor_method = param.first;
    conf_.bucket_size = 200;
    conf_.compressed_bucket_size = conf_.bucket_size / 10;
    conf_.bicriteria_base_size = conf_.bucket_size / 100;
    conf_.bucket_length = 2;
    conf_.forgetting_factor = 0.0;
    conf_.forgetting_threshold = 0.5;
    clustering_ = create_driver();
  }
  void TearDown() {
    clustering_.reset();
  }
  shared_ptr<driver::clustering> clustering_;
  clustering_config conf_;
};

namespace {  // testing util
datum single_datum(string key, double v) {
  datum d;
  d.num_values_.push_back(make_pair(key, v));
  return d;
}
}

TEST_P(clustering_test, get_revision) {
  const int num = conf_.bucket_size * 10;
  for (int i = 0; i < num; ++i) {
    vector<datum> datums;
    datums.push_back(single_datum("a", 1));
    clustering_->push(datums);
  }
  ASSERT_EQ(num / conf_.bucket_size, clustering_->get_revision());
}

TEST_P(clustering_test, push) {
  for (int j = 0; j < conf_.bucket_size / 5; ++j) {
    vector<datum> datums;
    for (int i = 0; i < 100; i += 5) {
      datums.push_back(single_datum("a", i * 2));
      datums.push_back(single_datum("b", i * 100));
    }
    clustering_->push(datums);
  }
}

TEST_P(clustering_test, save_load) {
  {
    core::fv_converter::datum d;
    vector<datum> datums;
    datums.push_back(single_datum("a", 1));
    clustering_->push(datums);
  }

  // save to a buffer
  msgpack::sbuffer sbuf;
  framework::stream_writer<msgpack::sbuffer> st(sbuf);
  framework::jubatus_packer jp(st);
  framework::packer packer(jp);
  clustering_->pack(packer);

  // restart the driver
  TearDown();
  SetUp();

  // unpack the buffer
  msgpack::unpacked unpacked;
  msgpack::unpack(&unpacked, sbuf.data(), sbuf.size());
  clustering_->unpack(unpacked.get());
}

TEST_P(clustering_test, get_k_center) {
  jubatus::util::math::random::mtrand r(0);
  vector<datum> one;
  vector<datum> two;

  for (int j = 0; j < conf_.bucket_size / 2; ++j) {
    datum a, b;
    a.num_values_.push_back(make_pair("a", -10 + r.next_gaussian() * 20));
    a.num_values_.push_back(make_pair("b", -200 + r.next_gaussian() * 400));
    b.num_values_.push_back(make_pair("c", -50 + r.next_gaussian() * 100));
    b.num_values_.push_back(make_pair("d", -25 + r.next_gaussian() * 50));
    one.push_back(a);
    two.push_back(b);
  }
  clustering_->push(one);
  clustering_->push(two);

  clustering_->do_clustering();
  {
    vector<datum> result = clustering_->get_k_center();
    ASSERT_EQ(conf_.k, result.size());
    ASSERT_LT(1, result[0].num_values_.size());
    if (result[0].num_values_[0].first == "a"
        || result[0].num_values_[0].first == "b") {
      // result[0] is {"a":xx, "b":yy} cluster
      if (result[0].num_values_[0].first == "a") {
        ASSERT_EQ("b", result[0].num_values_[1].first);
      } else {
        ASSERT_EQ("a", result[0].num_values_[1].first);
      }
    } else {
      // result[1] is {"a":xx, "b":yy} cluster
      if (result[1].num_values_[0].first == "a") {
        ASSERT_EQ("b", result[1].num_values_[1].first);
      } else {
        ASSERT_EQ("a", result[1].num_values_[1].first);
      }
    }
  }
}

TEST_P(clustering_test, integer_center) {
  jubatus::util::math::random::mtrand r(0);
  const int quantity = conf_.bucket_size * 5;
  std::vector<datum> data(quantity);

  for (int i = 0; i < quantity ; ++i) {
    data[i].num_values_.push_back(make_pair("x", 100 + r.next_int(-10, 10)));
  }

  clustering_->push(data);
  const std::vector<fv_converter::datum> centers = clustering_->get_k_center();

  ASSERT_EQ(conf_.k, centers.size());
  /*  debug out
  for (size_t i = 0; i < centers.size(); ++i) {
    std::cout << i << " :[";
    for (size_t j = 0; j < centers[i].num_values_.size(); ++j) {
      std::cout
        << centers[i].num_values_[j].first << " => "
        << centers[i].num_values_[j].second << ", ";
    }
    std::cout << "]" << std::endl;;
  }
  //*/
}

struct check_points {
  float a;
  float b;
  check_points(float a_, float b_)
    : a(a_), b(b_) {}
  bool operator<(const check_points& rhs) const {
    if (a < rhs.a) {
      return true;
    } else if (rhs.a < a) {
      return false;
    } else if (b < rhs.b) {
      return true;
    } else {
      return false;
    }
  }
};

struct check_point_compare {
  bool operator()(const check_points& lhs, const check_points& rhs) {
    return lhs < rhs;
  }
};


TEST_P(clustering_test, get_nearest_members) {
  jubatus::util::math::random::mtrand r(0);
  vector<datum> one;
  vector<datum> two;

  set<check_points, check_point_compare> points;

  for (int i = 0; i < conf_.bucket_size * 2 + 1; ++i) {
    datum x, y;
    float a = 10 + r.next_gaussian() * 20;
    float b = 1000 + r.next_gaussian() * 400;
    points.insert(check_points(a, b));

    x.num_values_.push_back(make_pair("a", a));
    x.num_values_.push_back(make_pair("b", b));
    y.num_values_.push_back(make_pair("c", -500000 - r.next_gaussian() * 100));
    y.num_values_.push_back(make_pair("d", -10000 - r.next_gaussian() * 50));
    one.push_back(x);
    two.push_back(y);
  }
  clustering_->push(one);
  clustering_->push(two);

  clustering_->do_clustering();

  {
    vector<datum> result = clustering_->get_k_center();
    ASSERT_EQ(conf_.k, result.size());
  }

  set<check_points, check_point_compare>::const_iterator it;
  for (it = points.begin(); it != points.end(); ++it) {
    datum x;
    x.num_values_.push_back(make_pair("a", it->a));
    x.num_values_.push_back(make_pair("b", it->b));
    core::clustering::cluster_unit result =
        clustering_->get_nearest_members(x);

    ASSERT_LT(1, result.size());
    for (size_t i = 0; i < result.size(); ++i) {
      const vector<pair<string, double> >& near_points =
          result[i].second.num_values_;
      ASSERT_EQ(2, near_points.size());  // must be 2-dimentional
      ASSERT_EQ("a", near_points[0].first);
      ASSERT_EQ("b", near_points[1].first);
      ASSERT_NE(points.end(),
                points.find(check_points(near_points[0].second,
                                         near_points[1].second)));
    }
  }
}



TEST_P(clustering_test, get_nearest_center) {
  jubatus::util::math::random::mtrand r(0);
  vector<datum> one;
  vector<datum> two;

  for (int i = 0; i < conf_.bucket_size * 2; ++i) {
    datum x, y;
    x.num_values_.push_back(make_pair("a", 10 + r.next_gaussian() * 20));
    x.num_values_.push_back(make_pair("b", 1000 + r.next_gaussian() * 400));
    y.num_values_.push_back(make_pair("c", -500000 - r.next_gaussian() * 100));
    y.num_values_.push_back(make_pair("d", -10000 - r.next_gaussian() * 50));
    one.push_back(x);
    two.push_back(y);
  }
  clustering_->push(one);
  clustering_->push(two);

  clustering_->do_clustering();

  {
    vector<datum> result = clustering_->get_k_center();
    ASSERT_EQ(conf_.k, result.size());
  }

  for (int i = 0; i < 100; ++i) {
    {
      datum x;
      x.num_values_.push_back(make_pair("a", 10 + r.next_gaussian() * 20));
      x.num_values_.push_back(make_pair("b", 1000 + r.next_gaussian() * 400));
      datum expect_near_x = clustering_->get_nearest_center(x);
      const vector<pair<string, double> >& result = expect_near_x.num_values_;
      for (size_t i = 0; i < result.size(); ++i) {
        // Check if the result is belonging to the same cluster.
        // Difference of the value and the mean of its distribution are expected
        // to be lesser than 2-sigma.
        if (result[i].first == "a") {
          const double diff = std::abs(result[i].second - 10);
          ASSERT_GT(20 * 2, diff);
        } else if (result[i].first == "b") {
          const double diff = std::abs(result[i].second - 1000);
          ASSERT_GT(400 * 2, diff);
        }
      }
    }
  }
}

TEST_P(clustering_test, empty_mix) {
  framework::linear_mixable* clustering_mixable =
    dynamic_cast<framework::linear_mixable*>(clustering_->get_mixable());
  shared_ptr<driver::clustering> other = create_driver();
  framework::linear_mixable* other_mixable =
    dynamic_cast<framework::linear_mixable*>(other->get_mixable());
  ASSERT_TRUE(clustering_mixable);
  ASSERT_TRUE(other_mixable);

  msgpack::sbuffer data;
  {
    core::framework::stream_writer<msgpack::sbuffer> st(data);
    core::framework::jubatus_packer jp(st);
    core::framework::packer pk(jp);
    clustering_mixable->get_diff(pk);
  }
  {
    msgpack::sbuffer sbuf;
    core::framework::stream_writer<msgpack::sbuffer> st(sbuf);
    core::framework::jubatus_packer jp(st);
    core::framework::packer pk(jp);
    other_mixable->get_diff(pk);

    msgpack::unpacked msg;
    msgpack::unpack(&msg, sbuf.data(), sbuf.size());
    framework::diff_object diff = other_mixable->convert_diff_object(msg.get());

    msgpack::unpacked data_msg;
    msgpack::unpack(&data_msg, data.data(), data.size());

    other_mixable->mix(data_msg.get(), diff);
    other_mixable->put_diff(diff);
  }
}

vector<pair<string, string> > parameter_list() {
  vector<pair<string, string> > ret;
  ret.push_back(make_pair("simple", "kmeans"));
  ret.push_back(make_pair("compressive_kmeans", "kmeans"));
#ifdef JUBATUS_USE_EIGEN
  ret.push_back(make_pair("simple", "gmm"));
  ret.push_back(make_pair("compressive_gmm", "gmm"));
#endif
  return ret;
}

INSTANTIATE_TEST_CASE_P(clustering_test_instance,
                        clustering_test,
                        testing::ValuesIn(parameter_list()));



class clustering_with_idf_test
    : public ::testing::TestWithParam<pair<string, string> > {
 protected:
  shared_ptr<driver::clustering> create_driver() const {
    pair<string, string> param = GetParam();
    clustering_config conf;
    conf.compressor_method = param.first;
    return shared_ptr<driver::clustering>(
        new driver::clustering(
            shared_ptr<core::clustering::clustering>(
                new core::clustering::clustering("dummy",
                                                 param.second,
                                                 conf)),
            make_tf_idf_fv_converter()));
  }
  void SetUp() {
    pair<string, string> param = GetParam();
    clustering_config conf;
    conf.compressor_method = param.first;
    conf.bucket_size = 50;
    conf.bicriteria_base_size = 5;
    conf.compressed_bucket_size = 10;
    clustering_ = create_driver();
  }
  void TearDown() {
    clustering_.reset();
  }
  shared_ptr<driver::clustering> clustering_;
};

fv_converter::datum create_datum_str(const string& key, const string& value) {
  fv_converter::datum d;
  d.string_values_.push_back(make_pair(key, value));
  return d;
}

TEST_P(clustering_with_idf_test, get_nearest_members) {
  jubatus::util::math::random::mtrand r(0);
  vector<datum> one;
  vector<datum> two;

  for (int i = 0; i < 200 ; ++i) {
    const string i_str = lexical_cast<string>(i);
    one.push_back(create_datum_str("a" + i_str, "x y z " + i_str));
    two.push_back(create_datum_str("a" + i_str, "y z w " + i_str));
  }
  clustering_->push(one);
  clustering_->push(two);

  clustering_->do_clustering();

  {
    vector<datum> result = clustering_->get_k_center();
    ASSERT_EQ(2u, result.size());
  }
}

INSTANTIATE_TEST_CASE_P(clustering_with_idf_test_instance,
                        clustering_with_idf_test,
                        testing::ValuesIn(parameter_list()));

}  // namespace driver
}  // namespace core
}  // namespace jubatus

{'fraction_non_alphanumeric': 0.15312848960772307, 'fraction_numerical': 0.015201181513634235, 'mean_word_length': 3.4312849162011174, 'pattern_counts': {'":': 8, '<': 183, '<?xml version=': 0, '>': 187, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4725599', 'n_tokens_mistral': 10266, 'n_tokens_neox': 9608, 'n_words': 1864}