<commit_before>/*
Resembla: Word-based Japanese similar sentence search library
https://github.com/tuem/resembla

Copyright 2017 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#ifndef RESEMBLA_RESEMBLA_WITH_ID_HPP
#define RESEMBLA_RESEMBLA_WITH_ID_HPP

#include <string>
#include <vector>
#include <memory>
#include <unordered_map>

#include "resembla_interface.hpp"
#include "regression/feature.hpp"

namespace resembla {

class ResemblaWithId
{
public:
    using id_type = int;

    struct output_type: public ResemblaInterface::output_type
    {
        id_type id;

        output_type(const ResemblaInterface::output_type& output, id_type id):
            ResemblaInterface::output_type(output), id(id)
        {}
    };

    ResemblaWithId(const std::shared_ptr<ResemblaInterface> resembla,
            std::string corpus_path, size_t id_col = 1, size_t text_col = 2);

    std::vector<output_type> find(const string_type& query, double threshold = 0.0,
            size_t max_response = 0) const;
    std::vector<output_type> eval(const string_type& query, const std::vector<string_type>& targets,
            double threshold = 0.0, size_t max_response = 0) const;

protected:
    std::shared_ptr<ResemblaInterface> resembla;
    std::unordered_map<string_type, id_type> ids;

    void loadCorpus(const std::string& corpus_path, size_t id_col, size_t text_col);
};

}
#endif<commit_msg>delete unused include declaration<commit_after>/*
Resembla: Word-based Japanese similar sentence search library
https://github.com/tuem/resembla

Copyright 2017 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#ifndef RESEMBLA_RESEMBLA_WITH_ID_HPP
#define RESEMBLA_RESEMBLA_WITH_ID_HPP

#include <string>
#include <vector>
#include <memory>
#include <unordered_map>

#include "resembla_interface.hpp"

namespace resembla {

class ResemblaWithId
{
public:
    using id_type = int;

    struct output_type: public ResemblaInterface::output_type
    {
        id_type id;

        output_type(const ResemblaInterface::output_type& output, id_type id):
            ResemblaInterface::output_type(output), id(id)
        {}
    };

    ResemblaWithId(const std::shared_ptr<ResemblaInterface> resembla,
            std::string corpus_path, size_t id_col = 1, size_t text_col = 2);

    std::vector<output_type> find(const string_type& query,
            double threshold = 0.0, size_t max_response = 0) const;
    std::vector<output_type> eval(const string_type& query, const std::vector<string_type>& targets,
            double threshold = 0.0, size_t max_response = 0) const;

protected:
    std::shared_ptr<ResemblaInterface> resembla;
    std::unordered_map<string_type, id_type> ids;

    void loadCorpus(const std::string& corpus_path, size_t id_col, size_t text_col);
};

}
#endif
{'fraction_non_alphanumeric': 0.08502132196162047, 'fraction_numerical': 0.008528784648187633, 'mean_word_length': 4.376790830945558, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4193963', 'n_tokens_mistral': 1205, 'n_tokens_neox': 1132, 'n_words': 356}