<commit_before>#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE test_first_letter

#include "first_letter/first_letter.h"
#include "type/data.h"
#include <boost/test/unit_test.hpp>
#include <vector>
#include <iostream>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <fstream>
#include<map>
#include<unordered_map>

namespace pt = boost::posix_time;
using namespace navitia::firstletter;

BOOST_AUTO_TEST_CASE(parse_state_test){
    FirstLetter<unsigned int> fl;
    fl.add_string("rue jean jaures", 0);
    fl.add_string("place jean jaures", 1);
    fl.add_string("rue jeanne d'arc", 2);
    fl.add_string("avenue jean jaures", 3);
    fl.add_string("<NAME>", 4);
    fl.add_string("<NAME>", 5);
    fl.build();    

    auto res = fl.find("rue jean jaures");
    std::vector<int> expected = {0};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("jaures");
    expected = {0, 1, 3};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("avenue");
    expected = {3};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("av");
    expected = {3};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("r jean");
    expected = {0, 2};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("jean r");
    expected = {0, 2};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("jEaN r");
    expected = {0, 2};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("ponia");
    expected = {4};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("ru je jau");
    expected = {0};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());    
}

/*
    > Le fonctionnement d'alias :


  */

BOOST_AUTO_TEST_CASE(alias_test){
    FirstLetter<unsigned int> fl;
    std::map <std::string,std::string> map;
    map["st"]="saint";
    std::string str = fl.get_alias("st", map);
    BOOST_CHECK_EQUAL("saint",str);

    map["de"]="";
    str = fl.get_alias("de", map);
    BOOST_CHECK_EQUAL("", str);
}

/*
    > Le fonctionnement partiel :> On prends tous les firstletter s'il y au moins un match
      et trie la liste des FirstLetter par la qualité.
    > La qualité est calculé avec le nombre des mots dans la recherche et le nombre des matchs dans FirstLetter.

    Exemple:
    vector[] = {0,1,2,3,4,5};
    Recherche : "<NAME>";
    Vector      nb_found    Quality
        0           1       33
        1           2       50
        2           1       33
        5           2       66

    Result [] = {1,5,0,2}

    Recherche : "jean";
    Vector      nb_found    Quality
        0           1       100
        1           1       100
        2           1       100
        5           1       100

    > Le résultat est trié par qualité
    Result [] = {0,1,2,5}
*/

BOOST_AUTO_TEST_CASE(parse_state_nb_found_test){
        FirstLetter<unsigned int> fl;

        fl.add_string("place jean jaures", 0);
        fl.add_string("rue je<NAME>", 1);
        fl.add_string("aven<NAME>", 2);
        fl.add_string("<NAME>", 3);
        fl.add_string("pente de Bray", 4);
        fl.add_string("rue jean jaures", 5);

        fl.build();

        auto res = fl.find_partial("rue jean");
        BOOST_REQUIRE_EQUAL(res.size(), 4);
        BOOST_REQUIRE_EQUAL(res.at(0).idx, 5);
        BOOST_REQUIRE_EQUAL(res.at(1).idx, 1);
        BOOST_REQUIRE_EQUAL(res.at(2).idx, 0);
        BOOST_REQUIRE_EQUAL(res.at(3).idx, 2);

        BOOST_REQUIRE_EQUAL(res.at(0).quality, 66);
        BOOST_REQUIRE_EQUAL(res.at(1).quality, 50);
        BOOST_REQUIRE_EQUAL(res.at(2).quality, 33);
        BOOST_REQUIRE_EQUAL(res.at(3).quality, 33);
    }

/*<commit_msg>FirstLetter : les tests portaient sur un tri indéterminé.<commit_after>#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE test_first_letter

#include "first_letter/first_letter.h"
#include "type/data.h"
#include <boost/test/unit_test.hpp>
#include <vector>
#include <iostream>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <fstream>
#include<map>
#include<unordered_map>

namespace pt = boost::posix_time;
using namespace navitia::firstletter;

BOOST_AUTO_TEST_CASE(parse_state_test){
    FirstLetter<unsigned int> fl;
    fl.add_string("rue jean jaures", 0);
    fl.add_string("place jean jaures", 1);
    fl.add_string("rue jeanne d'arc", 2);
    fl.add_string("avenue jean jaures", 3);
    fl.add_string("<NAME>", 4);
    fl.add_string("<NAME>", 5);
    fl.build();    

    auto res = fl.find("rue jean jaures");
    std::vector<int> expected = {0};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("jaures");
    expected = {0, 1, 3};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("avenue");
    expected = {3};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("av");
    expected = {3};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("r jean");
    expected = {0, 2};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("jean r");
    expected = {0, 2};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("jEaN r");
    expected = {0, 2};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("ponia");
    expected = {4};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());

    res = fl.find("ru je jau");
    expected = {0};
    BOOST_CHECK_EQUAL_COLLECTIONS(res.begin(), res.end(), expected.begin(), expected.end());    
}

/*
    > Le fonctionnement d'alias :


  */

BOOST_AUTO_TEST_CASE(alias_test){
    FirstLetter<unsigned int> fl;
    std::map <std::string,std::string> map;
    map["st"]="saint";
    std::string str = fl.get_alias("st", map);
    BOOST_CHECK_EQUAL("saint",str);

    map["de"]="";
    str = fl.get_alias("de", map);
    BOOST_CHECK_EQUAL("", str);
}

/*
    > Le fonctionnement partiel :> On prends tous les firstletter s'il y au moins un match
      et trie la liste des FirstLetter par la qualité.
    > La qualité est calculé avec le nombre des mots dans la recherche et le nombre des matchs dans FirstLetter.

    Exemple:
    vector[] = {0,1,2,3,4,5};
    Recherche : "rue jean";
    Vector      nb_found    Quality
        0           1       33
        1           2       50
        2           1       33
        5           2       66

    Result [] = {1,5,0,2}

    Recherche : "jean";
    Vector      nb_found    Quality
        0           1       100
        1           1       100
        2           1       100
        5           1       100

    > Le résultat est trié par qualité
    Result [] = {0,1,2,5}
*/

BOOST_AUTO_TEST_CASE(parse_state_nb_found_test){
        FirstLetter<unsigned int> fl;

        fl.add_string("place jean jaures", 0);
        fl.add_string("rue jeanne d'arc", 1);
        fl.add_string("avenue jean jaures", 2);
        fl.add_string("<NAME>", 3);
        fl.add_string("pente de Bray", 4);
        fl.add_string("rue jean jaures", 5);

        fl.build();

        auto res = fl.find_partial("rue jean");
        BOOST_REQUIRE_EQUAL(res.size(), 4);
        BOOST_CHECK_EQUAL(res.at(0).idx, 5);
        BOOST_CHECK_EQUAL(res.at(1).idx, 1);
        //Ils on la même qualité, l'ordre est donc indéterminé
        BOOST_CHECK(res.at(2).idx == 0 || res.at(2).idx == 2);
        BOOST_CHECK(res.at(3).idx == 0 || res.at(3).idx == 2);

        BOOST_CHECK_EQUAL(res.at(0).quality, 66);
        BOOST_CHECK_EQUAL(res.at(1).quality, 50);
        BOOST_CHECK_EQUAL(res.at(2).quality, 33);
        BOOST_CHECK_EQUAL(res.at(3).quality, 33);
    }

/*
{'fraction_non_alphanumeric': 0.155877034358047, 'fraction_numerical': 0.023869801084990958, 'mean_word_length': 2.8496519721577727, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228570', 'n_tokens_mistral': 3071, 'n_tokens_neox': 2845, 'n_words': 614}