<commit_before>#include "parser.hpp"

using namespace std;

#include <limits>

///////////////////////////////////////////////////////////////////////////////
//  Implementation of parser class
///////////////////////////////////////////////////////////////////////////////
parser::parser(lexer& scanner, error_container& err_repo)
    : scanner(scanner), err_repo(err_repo) {
        error::line = scanner.getline();
}

shared_ptr<rpl_node> parser::parse()
{
    token tok = scanner.next();
    rpl_node* tree = start_rule(tok);
    if (scanner.has_next() && err_repo.size() > 0)
        return nullptr;
    return shared_ptr<rpl_node>(tree);
}

//  Useful functions in parsing: check if the token tok has the expected output
//  and advance the scanner returning true, otherwise add error message to the
//  error_repo and return false
bool parser::expect(token& tok, token::type exp)
{
    if (tok.kind == exp) {
        tok = scanner.next();
        return true;
    } else {
        err_repo.add( make_shared<error_unexp>(tok.data, tok.pos) );
        return false;
    }
}

bool parser::expect(token& tok, token::type exp, string& data)
{
    data = tok.data;
    return expect(tok, exp);
}

bool parser::expect(token& tok, token::type exp, int& data)
{
    data = exp == token::integer ? stoi(tok.data) : 0;
    return expect(tok, exp);
}

bool parser::expect(token& tok, token::type exp, double& data)
{
    data = exp == token::integer ? stod(tok.data) : 0;
    return expect(tok, exp);
}

bool parser::expect(token& tok, token::type exp1, token::type exp2)
{
    return tok.kind == exp2 ? expect(tok, exp2) : expect(tok, exp1);
}

///////////////////////////////////////////////////////////////////////////////
//
//  The parsing rules actually implementing the recursive descent parser
//  of the shell language
//
///////////////////////////////////////////////////////////////////////////////
rpl_node* parser::start_rule(token& tok)
{
    switch (tok.kind) {
        case token::word:
            return assign_rule(tok);
        case token::show:
            return show_rule(tok);
        case token::set:
            return set_rule(tok);
        case token::annotate:
            return ann_rule(tok);
        case token::rewrite:
            return rwr_rule(tok);
        case token::optimize:
            return opt_rule(tok);
        default:
            err_repo.add( make_shared<error_unexp>(tok.data, tok.pos) );
            return nullptr;
    }
}

//  <assign> ::= <word> = <pattexp>
//  TODO add possibility to put rewriting/opt rules as functions in the right part
rpl_node* parser::assign_rule(token& tok)
{
    string word = tok.data;
    skel_node* pattexp;
    expect(tok, token::word);
    expect(tok, token::equals);
    pattexp = pattexp_rule(tok);
    expect(tok, token::eol);
    if (err_repo.size() > 0) {
        delete pattexp;
        pattexp = nullptr;
    }
    return new assign_node(word, pattexp);
}

//  <show> ::= show <word> [<parameter>]
//  <parameter> ::= servicetime | latency | ...
rpl_node* parser::show_rule(token& tok)
{
    string id, par = "show_default";
    vector<string> parameters;
    int num = numeric_limits<int>::max();

    expect(tok, token::show);
    if ( tok.kind == token::parameter ) {
        expect(tok, token::parameter, par);
        expect(tok, token::eol);
        return new show_node("", 0, par, parameters);
    }

    expect(tok, token::word, id);
    if (tok.kind == token::by) {
        expect(tok, token::by);
        expect(tok, token::parameter, par);
        while ( tok.kind == token::comma ) {
            parameters.push_back(par);
            expect(tok, token::comma);
            expect(tok, token::parameter, par);
        }
        parameters.push_back(par);
    }
    else if (tok.kind == token::parameter) {                     // optional
        expect(tok, token::parameter, par);
        parameters.push_back(par);
    }

    /* always put the default */
    parameters.push_back("show_default");

    if ( tok.kind == token::plus || tok.kind == token::minus ) {
        auto type = tok.kind;
        expect( tok, type );
        expect(tok, token::integer, num);
        num = ( type == token::minus )? -num : num;
    }

    expect(tok, token::eol);
    return new show_node(id, num, parameters[0], parameters);
}

//  <set> ::= set <word> with <pattern> <integer>
//  <pattern> ::= farm | pipe | ...
rpl_node* parser::set_rule(token& tok)
{
    string par; double value;
    expect(tok, token::set);
    expect(tok, token::parameter, par);
    expect(tok, token::with);
    if ( tok.kind == token::number || tok.kind == token::integer )
        expect(tok, tok.kind, value);
    return new set_node(par, par, value);
}

//  <ann> ::= set <word> with <parameter> <value>
rpl_node* parser::ann_rule(token& tok)
{
    string id, par; double value;
    expect(tok, token::annotate);
    expect(tok, token::word, id);
    expect(tok, token::with);
    expect(tok, token::parameter, par);
    expect(tok, token::number, value);
    return new ann_node(id, par, value);
}

// <rwr> ::= rewrite <word> with <rwr-rule>
// <rwr-rule> ::= farmelim | farmfarmelim | farmintro | ...<commit_msg>ann rule parses also integers<commit_after>#include "parser.hpp"

using namespace std;

#include <limits>

///////////////////////////////////////////////////////////////////////////////
//  Implementation of parser class
///////////////////////////////////////////////////////////////////////////////
parser::parser(lexer& scanner, error_container& err_repo)
    : scanner(scanner), err_repo(err_repo) {
        error::line = scanner.getline();
}

shared_ptr<rpl_node> parser::parse()
{
    token tok = scanner.next();
    rpl_node* tree = start_rule(tok);
    if (scanner.has_next() && err_repo.size() > 0)
        return nullptr;
    return shared_ptr<rpl_node>(tree);
}

//  Useful functions in parsing: check if the token tok has the expected output
//  and advance the scanner returning true, otherwise add error message to the
//  error_repo and return false
bool parser::expect(token& tok, token::type exp)
{
    if (tok.kind == exp) {
        tok = scanner.next();
        return true;
    } else {
        err_repo.add( make_shared<error_unexp>(tok.data, tok.pos) );
        return false;
    }
}

bool parser::expect(token& tok, token::type exp, string& data)
{
    data = tok.data;
    return expect(tok, exp);
}

bool parser::expect(token& tok, token::type exp, int& data)
{
    data = exp == token::integer ? stoi(tok.data) : 0;
    return expect(tok, exp);
}

bool parser::expect(token& tok, token::type exp, double& data)
{
    data = exp == token::integer ? stod(tok.data) : 0;
    return expect(tok, exp);
}

bool parser::expect(token& tok, token::type exp1, token::type exp2)
{
    return tok.kind == exp2 ? expect(tok, exp2) : expect(tok, exp1);
}

///////////////////////////////////////////////////////////////////////////////
//
//  The parsing rules actually implementing the recursive descent parser
//  of the shell language
//
///////////////////////////////////////////////////////////////////////////////
rpl_node* parser::start_rule(token& tok)
{
    switch (tok.kind) {
        case token::word:
            return assign_rule(tok);
        case token::show:
            return show_rule(tok);
        case token::set:
            return set_rule(tok);
        case token::annotate:
            return ann_rule(tok);
        case token::rewrite:
            return rwr_rule(tok);
        case token::optimize:
            return opt_rule(tok);
        default:
            err_repo.add( make_shared<error_unexp>(tok.data, tok.pos) );
            return nullptr;
    }
}

//  <assign> ::= <word> = <pattexp>
//  TODO add possibility to put rewriting/opt rules as functions in the right part
rpl_node* parser::assign_rule(token& tok)
{
    string word = tok.data;
    skel_node* pattexp;
    expect(tok, token::word);
    expect(tok, token::equals);
    pattexp = pattexp_rule(tok);
    expect(tok, token::eol);
    if (err_repo.size() > 0) {
        delete pattexp;
        pattexp = nullptr;
    }
    return new assign_node(word, pattexp);
}

//  <show> ::= show <word> [<parameter>]
//  <parameter> ::= servicetime | latency | ...
rpl_node* parser::show_rule(token& tok)
{
    string id, par = "show_default";
    vector<string> parameters;
    int num = numeric_limits<int>::max();

    expect(tok, token::show);
    if ( tok.kind == token::parameter ) {
        expect(tok, token::parameter, par);
        expect(tok, token::eol);
        return new show_node("", 0, par, parameters);
    }

    expect(tok, token::word, id);
    if (tok.kind == token::by) {
        expect(tok, token::by);
        expect(tok, token::parameter, par);
        while ( tok.kind == token::comma ) {
            parameters.push_back(par);
            expect(tok, token::comma);
            expect(tok, token::parameter, par);
        }
        parameters.push_back(par);
    }
    else if (tok.kind == token::parameter) {                     // optional
        expect(tok, token::parameter, par);
        parameters.push_back(par);
    }

    /* always put the default */
    parameters.push_back("show_default");

    if ( tok.kind == token::plus || tok.kind == token::minus ) {
        auto type = tok.kind;
        expect( tok, type );
        expect(tok, token::integer, num);
        num = ( type == token::minus )? -num : num;
    }

    expect(tok, token::eol);
    return new show_node(id, num, parameters[0], parameters);
}

//  <set> ::= set <word> with <pattern> <integer>
//  <pattern> ::= farm | pipe | ...
rpl_node* parser::set_rule(token& tok)
{
    string par; double value;
    expect(tok, token::set);
    expect(tok, token::parameter, par);
    expect(tok, token::with);
    if ( tok.kind == token::number || tok.kind == token::integer )
        expect(tok, tok.kind, value);
    return new set_node(par, par, value);
}

//  <ann> ::= set <word> with <parameter> <value>
rpl_node* parser::ann_rule(token& tok)
{
    string id, par; double value;
    expect(tok, token::annotate);
    expect(tok, token::word, id);
    expect(tok, token::with);
    expect(tok, token::parameter, par);
    if ( tok.kind == token::number || tok.kind == token::integer )
        expect(tok, tok.kind, value);
    return new ann_node(id, par, value);
}

// <rwr> ::= rewrite <word> with <rwr-rule>
// <rwr-rule> ::= farmelim | farmfarmelim | farmintro | ...
{'fraction_non_alphanumeric': 0.204302103250478, 'fraction_numerical': 0.002103250478011472, 'mean_word_length': 2.262944479101684, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2523957', 'n_tokens_mistral': 3245, 'n_tokens_neox': 3191, 'n_words': 815}