<commit_before>#include "fare_parser.h"
#include "utils/csv.h"
#include "ed/data.h"
#include "fare_utils.h"

#include <boost/foreach.hpp>
#include <boost/algorithm/string.hpp>

/// Wrapper pour pouvoir parser une condition en une seule fois avec boost::spirit::qi
namespace fa = navitia::fare;

namespace greg = boost::gregorian;

namespace ed { namespace connectors {

void fare_parser::load() {
    load_prices();

    load_transitions();

    load_od();
}

void fare_parser::load_transitions() {
    CsvReader reader(state_transition_filename);
    std::vector<std::string> row;
    reader.next(); //header

    for (row = reader.next(); !reader.eof(); row = reader.next()) {
        bool symetric = false;

        if (row.size() != 6) {
            LOG4CPLUS_ERROR(logger, "Wrongly formated line " << row.size() << " columns, we skip the line");
            continue;
        }

        fa::State start = parse_state(row.at(0));
        fa::State end = parse_state(row.at(1));

        fa::Transition transition;
        transition.start_conditions = parse_conditions(row.at(2));
        transition.end_conditions = parse_conditions(row.at(3));
        std::vector<std::string> global_conditions;
        std::string str_condition = boost::algorithm::trim_copy(row.at(4));
        boost::algorithm::split(global_conditions, str_condition, boost::algorithm::is_any_of("&"));

        for (std::string cond : global_conditions) {
           if (cond == "symetric") {
               symetric = true;
           } else {
               transition.global_condition = ed::connectors::to_global_condition(cond);
           }
        }
        transition.ticket_key = boost::algorithm::trim_copy(row[5]);

        //coherence check
        if (! transition.ticket_key.empty() && data.fare_map.find(transition.ticket_key) == data.fare_map.end()) {
            LOG4CPLUS_WARN(logger, "impossible to find ticket " << transition.ticket_key << ", transition skipped");
            continue;
        }

        data.transitions.push_back(std::make_tuple(start, end, transition));

        if (symetric) {
            fa::Transition sym_transition = transition;
            sym_transition.start_conditions = transition.end_conditions;
            sym_transition.end_conditions = transition.start_conditions;
            data.transitions.push_back(std::make_tuple(start, end, sym_transition));
        }
    }

}


void fare_parser::load_prices() {
    CsvReader reader(prices_filename);
    for (std::vector<std::string> row = reader.next() ; ! reader.eof() ; row = reader.next()) {
        // csv structure is:
        //key; begin; end; price; name; unknown field :); comment; currency(optional)
        greg::date start(greg::from_undelimited_string(row.at(1)));
        greg::date end(greg::from_undelimited_string(row.at(2)));

        fa::Ticket dated_ticket(row.at(0), row.at(4), boost::lexical_cast<int>(row.at(3)), row.at(5));
        if (row.size() > 7) {
            dated_ticket.currency = row[8];
        }
        data.fare_map[row.at(0)].add(start, end, dated_ticket);
    }
}

void fare_parser::load_od() {
    CsvReader reader(od_filename);
    std::vector<std::string> row;
    reader.next(); //header

    // file format is :
    // Origin ID, Origin name, Origin mode, Destination ID, Destination name, Destination mode, ticket_id, ticket id, .... (with any number of ticket)

    int count = 0;
    for (row=reader.next(); !reader.eof(); row = reader.next()) {

        if (row.size() < 7) {
            LOG4CPLUS_WARN(logger, "wrongly formated OD line : " << boost::algorithm::join(row, ", "));
            continue;
        }
        std::string start_saec = boost::algorithm::trim_copy(row[0]);
        std::string dest_saec = boost::algorithm::trim_copy(row[3]);
        //col 1 and 4 are the human readable name of the start/end, and are not used

        std::string start_mode = boost::algorithm::trim_copy(row[2]);
        std::string dest_mode = boost::algorithm::trim_copy(row[5]);

        std::vector<std::string> price_keys;
        for (size_t i = 6; i < row.size(); ++i) {
            std::string price_key = boost::algorithm::trim_copy(row[i]);
<commit_msg>Kraken: fare bug correction<commit_after>#include "fare_parser.h"
#include "utils/csv.h"
#include "ed/data.h"
#include "fare_utils.h"

#include <boost/foreach.hpp>
#include <boost/algorithm/string.hpp>

/// Wrapper pour pouvoir parser une condition en une seule fois avec boost::spirit::qi
namespace fa = navitia::fare;

namespace greg = boost::gregorian;

namespace ed { namespace connectors {

void fare_parser::load() {
    load_prices();

    load_transitions();

    load_od();
}

void fare_parser::load_transitions() {
    CsvReader reader(state_transition_filename);
    std::vector<std::string> row;
    reader.next(); //header

    for (row = reader.next(); !reader.eof(); row = reader.next()) {
        bool symetric = false;

        if (row.size() != 6) {
            LOG4CPLUS_ERROR(logger, "Wrongly formated line " << row.size() << " columns, we skip the line");
            continue;
        }

        fa::State start = parse_state(row.at(0));
        fa::State end = parse_state(row.at(1));

        fa::Transition transition;
        transition.start_conditions = parse_conditions(row.at(2));
        transition.end_conditions = parse_conditions(row.at(3));
        std::vector<std::string> global_conditions;
        std::string str_condition = boost::algorithm::trim_copy(row.at(4));
        boost::algorithm::split(global_conditions, str_condition, boost::algorithm::is_any_of("&"));

        for (std::string cond : global_conditions) {
           if (cond == "symetric") {
               symetric = true;
           } else {
               transition.global_condition = ed::connectors::to_global_condition(cond);
           }
        }
        transition.ticket_key = boost::algorithm::trim_copy(row[5]);

        //coherence check
        if (! transition.ticket_key.empty() && data.fare_map.find(transition.ticket_key) == data.fare_map.end()) {
            LOG4CPLUS_WARN(logger, "impossible to find ticket " << transition.ticket_key << ", transition skipped");
            continue;
        }

        data.transitions.push_back(std::make_tuple(start, end, transition));

        if (symetric) {
            fa::Transition sym_transition = transition;
            sym_transition.start_conditions = transition.end_conditions;
            sym_transition.end_conditions = transition.start_conditions;
            data.transitions.push_back(std::make_tuple(start, end, sym_transition));
        }
    }

}


void fare_parser::load_prices() {
    CsvReader reader(prices_filename);
    for (std::vector<std::string> row = reader.next() ; ! reader.eof() ; row = reader.next()) {
        // csv structure is:
        //key; begin; end; price; name; unknown field :); comment; currency(optional)
        greg::date start(greg::from_undelimited_string(row.at(1)));
        greg::date end(greg::from_undelimited_string(row.at(2)));

        fa::Ticket dated_ticket(row.at(0), row.at(4), boost::lexical_cast<int>(row.at(3)), row.at(5));
        if (row.size() > 7) {
            dated_ticket.currency = row[7];
        }
        data.fare_map[row.at(0)].add(start, end, dated_ticket);
    }
}

void fare_parser::load_od() {
    CsvReader reader(od_filename);
    std::vector<std::string> row;
    reader.next(); //header

    // file format is :
    // Origin ID, Origin name, Origin mode, Destination ID, Destination name, Destination mode, ticket_id, ticket id, .... (with any number of ticket)

    int count = 0;
    for (row=reader.next(); !reader.eof(); row = reader.next()) {

        if (row.size() < 7) {
            LOG4CPLUS_WARN(logger, "wrongly formated OD line : " << boost::algorithm::join(row, ", "));
            continue;
        }
        std::string start_saec = boost::algorithm::trim_copy(row[0]);
        std::string dest_saec = boost::algorithm::trim_copy(row[3]);
        //col 1 and 4 are the human readable name of the start/end, and are not used

        std::string start_mode = boost::algorithm::trim_copy(row[2]);
        std::string dest_mode = boost::algorithm::trim_copy(row[5]);

        std::vector<std::string> price_keys;
        for (size_t i = 6; i < row.size(); ++i) {
            std::string price_key = boost::algorithm::trim_copy(row[i]);

{'fraction_non_alphanumeric': 0.14733953710331663, 'fraction_numerical': 0.006680983058935815, 'mean_word_length': 3.022552783109405, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5648988', 'n_tokens_mistral': 2633, 'n_tokens_neox': 2542, 'n_words': 529}