<commit_before>/*
Resembla
https://github.com/tuem/resembla

Copyright 2017 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "resembla_util.hpp"

#include <simstring/simstring.h>

#include "simstring_database.hpp"

#include "measure/asis_preprocessor.hpp"

#include "measure/word_preprocessor.hpp"
#include "measure/word_weight.hpp"
#include "measure/word_mismatch_cost.hpp"

#include "measure/pronunciation_preprocessor.hpp"
#include "measure/letter_weight.hpp"
#include "measure/kana_mismatch_cost.hpp"

#include "measure/romaji_preprocessor.hpp"
#include "measure/romaji_weight.hpp"
#include "measure/romaji_mismatch_cost.hpp"

#include "measure/keyword_match_preprocessor.hpp"
#include "measure/keyword_matcher.hpp"

#include "measure/weighted_sequence_builder.hpp"
#include "measure/weighted_sequence_serializer.hpp"

#include "measure/edit_distance.hpp"
#include "measure/weighted_edit_distance.hpp"

#include "measure/weighted_l2_norm.hpp"
#include "resembla_ensemble.hpp"

#include "regression/extractor/feature_extractor.hpp"
#include "regression/extractor/regex_feature_extractor.hpp"
#include "regression/extractor/date_period_feature_extractor.hpp"
#include "regression/extractor/time_period_feature_extractor.hpp"

#include "regression/aggregator/flag_feature_aggregator.hpp"
#include "regression/aggregator/real_feature_aggregator.hpp"
#include "regression/aggregator/interval_feature_aggregator.hpp"

namespace resembla {

const std::string SIMSTRING_DB_FILE_COMMON_SUFFIX = ".simstring_db.";
const std::string SIMSTRING_INVERSE_FILE_COMMON_SUFFIX = ".inverse.";

// utility function for converting string that represents a simstring measure to int
int simstring_measure_from_string(const std::string& simstring_measure_str)
{
    if(simstring_measure_str == "exact"){
        return simstring::exact;
    }
    else if(simstring_measure_str == "dice"){
        return simstring::dice;
    }
    else if(simstring_measure_str == "cosine"){
        return  simstring::cosine;
    }
    else if(simstring_measure_str == "jaccard"){
        return  simstring::jaccard;
    }
    else if(simstring_measure_str == "overlap"){
        return simstring::overlap;
    }
    else{
        throw std::invalid_argument("unknown simstring measure: " + simstring_measure_str);
    }
}

/*
std::string read_value_with_rest(paramset::manager& pm, const std::string key, const char* throw_if)
{
    return read_value_with_rest(pm, key, std::string(throw_if));
}
*/

std::string db_path_from_resembla_measure(const std::string& corpus_path, const measure resembla_measure)
{
    if(resembla_measure == edit_distance){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(edit_distance);
    }
    else if(resembla_measure == weighted_word_edit_distance){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(weighted_word_edit_distance);
    }
    else if(resembla_measure == weighted_pronunciation_edit_distance){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(weighted_pronunciation_edit_distance);
    }
    else if(resembla_measure == weighted_romaji_edit_distance){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(weighted_romaji_edit_distance);
    }
    else if(resembla_measure == keyword_match){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(keyword_match);
    }
    else if(resembla_measure == ensemble){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(ensemble);
    }
    else if(resembla_measure == svr){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(svr);
    }
    else{<commit_msg>delete old function<commit_after>/*
Resembla
https://github.com/tuem/resembla

Copyright 2017 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "resembla_util.hpp"

#include <simstring/simstring.h>

#include "simstring_database.hpp"

#include "measure/asis_preprocessor.hpp"

#include "measure/word_preprocessor.hpp"
#include "measure/word_weight.hpp"
#include "measure/word_mismatch_cost.hpp"

#include "measure/pronunciation_preprocessor.hpp"
#include "measure/letter_weight.hpp"
#include "measure/kana_mismatch_cost.hpp"

#include "measure/romaji_preprocessor.hpp"
#include "measure/romaji_weight.hpp"
#include "measure/romaji_mismatch_cost.hpp"

#include "measure/keyword_match_preprocessor.hpp"
#include "measure/keyword_matcher.hpp"

#include "measure/weighted_sequence_builder.hpp"
#include "measure/weighted_sequence_serializer.hpp"

#include "measure/edit_distance.hpp"
#include "measure/weighted_edit_distance.hpp"

#include "measure/weighted_l2_norm.hpp"
#include "resembla_ensemble.hpp"

#include "regression/extractor/feature_extractor.hpp"
#include "regression/extractor/regex_feature_extractor.hpp"
#include "regression/extractor/date_period_feature_extractor.hpp"
#include "regression/extractor/time_period_feature_extractor.hpp"

#include "regression/aggregator/flag_feature_aggregator.hpp"
#include "regression/aggregator/real_feature_aggregator.hpp"
#include "regression/aggregator/interval_feature_aggregator.hpp"

namespace resembla {

const std::string SIMSTRING_DB_FILE_COMMON_SUFFIX = ".simstring_db.";
const std::string SIMSTRING_INVERSE_FILE_COMMON_SUFFIX = ".inverse.";

// utility function for converting string that represents a simstring measure to int
int simstring_measure_from_string(const std::string& simstring_measure_str)
{
    if(simstring_measure_str == "exact"){
        return simstring::exact;
    }
    else if(simstring_measure_str == "dice"){
        return simstring::dice;
    }
    else if(simstring_measure_str == "cosine"){
        return  simstring::cosine;
    }
    else if(simstring_measure_str == "jaccard"){
        return  simstring::jaccard;
    }
    else if(simstring_measure_str == "overlap"){
        return simstring::overlap;
    }
    else{
        throw std::invalid_argument("unknown simstring measure: " + simstring_measure_str);
    }
}

std::string db_path_from_resembla_measure(const std::string& corpus_path, const measure resembla_measure)
{
    if(resembla_measure == edit_distance){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(edit_distance);
    }
    else if(resembla_measure == weighted_word_edit_distance){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(weighted_word_edit_distance);
    }
    else if(resembla_measure == weighted_pronunciation_edit_distance){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(weighted_pronunciation_edit_distance);
    }
    else if(resembla_measure == weighted_romaji_edit_distance){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(weighted_romaji_edit_distance);
    }
    else if(resembla_measure == keyword_match){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(keyword_match);
    }
    else if(resembla_measure == ensemble){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(ensemble);
    }
    else if(resembla_measure == svr){
        return corpus_path + SIMSTRING_DB_FILE_COMMON_SUFFIX + STR(svr);
    }
    else{
{'fraction_non_alphanumeric': 0.09307225182798363, 'fraction_numerical': 0.00223075969760813, 'mean_word_length': 5.329411764705882, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3355727', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2550, 'n_words': 518}