<commit_before>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2011 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include "converter_config.hpp"

#include <map>
#include <string>
#include <vector>
#include <pficommon/text/json.h>
#include "except_match.hpp"
#include "datum_to_fv_converter.hpp"
#include "exception.hpp"
#include "key_matcher.hpp"
#include "key_matcher_factory.hpp"
#include "num_feature.hpp"
#include "num_feature_impl.hpp"
#include "num_feature_factory.hpp"
#include "num_filter.hpp"
#include "num_filter_factory.hpp"
#include "space_splitter.hpp"
#include "splitter_factory.hpp"
#include "string_filter.hpp"
#include "string_filter_factory.hpp"
#include "without_split.hpp"

namespace jubatus {
namespace fv_converter {

namespace {

typedef pfi::lang::shared_ptr<word_splitter> splitter_ptr;
typedef pfi::lang::shared_ptr<key_matcher> matcher_ptr;
typedef pfi::lang::shared_ptr<num_feature> num_feature_ptr;
typedef pfi::lang::shared_ptr<string_filter> string_filter_ptr;
typedef pfi::lang::shared_ptr<num_filter> num_filter_ptr;

splitter_weight_type make_weight_type(
    const std::string& sample, const std::string& global) {
  frequency_weight_type sample_type;
  if (sample == "bin") {
    sample_type = FREQ_BINARY;
  } else if (sample == "tf") {
    sample_type = TERM_FREQUENCY;
  } else if (sample == "log_tf") {
    sample_type = LOG_TERM_FREQUENCY;
  } else {
    throw JUBATUS_EXCEPTION(
        converter_exception(std::string("unknown sample weight: ") + sample));
  }

  term_weight_type global_type;
  if (global == "bin") {
    global_type = TERM_BINARY;
  } else if (global == "idf") {
    global_type = IDF;
  } else if (global == "weight") {
    global_type = WITH_WEIGHT_FILE;
  } else {
    throw JUBATUS_EXCEPTION(
        converter_exception(std::string("unknown global weight: ") + global));
  }
  return splitter_weight_type(sample_type, global_type);
}

std::string get_or_die(
    const std::map<std::string, std::string>& m,
    const std::string& key) {
  std::map<std::string, std::string>::const_iterator it = m.find(key);
  if (it == m.end()) {
    throw JUBATUS_EXCEPTION(
        converter_exception(std::string("unknown parameter: ") + key));
  } else {
    return it->second;
  }
}

matcher_ptr create_key_matcher(
    const std::string& key,
    const pfi::data::optional<std::string>& except) {
  key_matcher_factory f;
  if (except) {
    matcher_ptr m1(f.create_matcher(key));
    matcher_ptr m2(f.create_matcher(*except.get()));
    return matcher_ptr(new except_match(m1, m2));
  } else {
    return matcher_ptr(f.create_matcher(key));
  }
}

void init_string_filter_types(
    const std::map<std::string, param_t>& filter_types,
    std::map<std::string, string_filter_ptr>& filters) {
  string_filter_factory f;
  for (std::map<std::string, param_t>::const_iterator it = filter_types.begin();
      it != filter_types.end(); ++it) {
    const std::string& name = it->first;
    const std::map<std::string, std::string>& param = it->second;

    std::string method = get_or_die(param, "method");
    string_filter_ptr filter(f.create(method, param));
    filters[name] = filter;
  }
}
<commit_msg>Use operator * instead of get method<commit_after>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2011 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include "converter_config.hpp"

#include <map>
#include <string>
#include <vector>
#include <pficommon/text/json.h>
#include "except_match.hpp"
#include "datum_to_fv_converter.hpp"
#include "exception.hpp"
#include "key_matcher.hpp"
#include "key_matcher_factory.hpp"
#include "num_feature.hpp"
#include "num_feature_impl.hpp"
#include "num_feature_factory.hpp"
#include "num_filter.hpp"
#include "num_filter_factory.hpp"
#include "space_splitter.hpp"
#include "splitter_factory.hpp"
#include "string_filter.hpp"
#include "string_filter_factory.hpp"
#include "without_split.hpp"

namespace jubatus {
namespace fv_converter {

namespace {

typedef pfi::lang::shared_ptr<word_splitter> splitter_ptr;
typedef pfi::lang::shared_ptr<key_matcher> matcher_ptr;
typedef pfi::lang::shared_ptr<num_feature> num_feature_ptr;
typedef pfi::lang::shared_ptr<string_filter> string_filter_ptr;
typedef pfi::lang::shared_ptr<num_filter> num_filter_ptr;

splitter_weight_type make_weight_type(
    const std::string& sample, const std::string& global) {
  frequency_weight_type sample_type;
  if (sample == "bin") {
    sample_type = FREQ_BINARY;
  } else if (sample == "tf") {
    sample_type = TERM_FREQUENCY;
  } else if (sample == "log_tf") {
    sample_type = LOG_TERM_FREQUENCY;
  } else {
    throw JUBATUS_EXCEPTION(
        converter_exception(std::string("unknown sample weight: ") + sample));
  }

  term_weight_type global_type;
  if (global == "bin") {
    global_type = TERM_BINARY;
  } else if (global == "idf") {
    global_type = IDF;
  } else if (global == "weight") {
    global_type = WITH_WEIGHT_FILE;
  } else {
    throw JUBATUS_EXCEPTION(
        converter_exception(std::string("unknown global weight: ") + global));
  }
  return splitter_weight_type(sample_type, global_type);
}

std::string get_or_die(
    const std::map<std::string, std::string>& m,
    const std::string& key) {
  std::map<std::string, std::string>::const_iterator it = m.find(key);
  if (it == m.end()) {
    throw JUBATUS_EXCEPTION(
        converter_exception(std::string("unknown parameter: ") + key));
  } else {
    return it->second;
  }
}

matcher_ptr create_key_matcher(
    const std::string& key,
    const pfi::data::optional<std::string>& except) {
  key_matcher_factory f;
  if (except) {
    matcher_ptr m1(f.create_matcher(key));
    matcher_ptr m2(f.create_matcher(*except));
    return matcher_ptr(new except_match(m1, m2));
  } else {
    return matcher_ptr(f.create_matcher(key));
  }
}

void init_string_filter_types(
    const std::map<std::string, param_t>& filter_types,
    std::map<std::string, string_filter_ptr>& filters) {
  string_filter_factory f;
  for (std::map<std::string, param_t>::const_iterator it = filter_types.begin();
      it != filter_types.end(); ++it) {
    const std::string& name = it->first;
    const std::map<std::string, std::string>& param = it->second;

    std::string method = get_or_die(param, "method");
    string_filter_ptr filter(f.create(method, param));
    filters[name] = filter;
  }
}

{'fraction_non_alphanumeric': 0.12265714650006375, 'fraction_numerical': 0.005355093714139997, 'mean_word_length': 4.793205317577548, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7134418', 'n_tokens_mistral': 2619, 'n_tokens_neox': 2552, 'n_words': 669}