<commit_before>#include <functional>
#include <initializer_list>
#include <string>
#include <vector>
#include <unordered_map>
#include <algorithm>


#include <iostream>

#include <memory>

namespace parser{

    using namespace std;

    vector<Rule> first(Rule Rs){
        if(Rs->isTerm()){
            return {Rs};
        }
        vector<Rule> res; 
        for(auto& r : Rs->rules()){
            if(!r.empty()){
                auto ext = first(r[0]);
                if(find(ext.begin(), ext.end(), R("Epsilon")) != ext.end()){
                    ext.erase(remove(ext.begin(), ext.end(), R("Epsilon")), ext.end());
                    res.insert(res.end(), ext.begin(), ext.end());
                    if(r.size() >= 2){
                        auto nxt = first(r[1]);
                        res.insert(res.end(), nxt.begin(), nxt.end());
                    }else{
                        res.push_back(R("Epsilon"));
                    }
                }
                res.insert(res.end(), ext.begin(), ext.end());
            }
        }
        return res;
    }

    vector<Rule> first(initializer_list<Rule>& l){
        if(l.size() == 0)
            return {R("Epsilon")};

        vector<Rule> res;
        
        auto it = l.begin();
        if(*it == R("Epsilon")) return {R("Epsilon")};
        if((*it)->isTerm()) return {*it};

        auto ext = first(*it); 
        if(find(ext.begin(), ext.end(), R("Epsilon")) != ext.end()){
            ext.erase(remove(ext.begin(), ext.end(), R("Epsilon")), ext.end());
            res.insert(res.end(), ext.begin(), ext.end());                
            if(l.size() >= 2 ){
                it++;
                auto next = first(*it);
                res.insert(res.end(), next.begin(), next.end());
            }
            return res;
        }else{
            return ext;
        }
    }

    vector<Rule> follow(Rule& Rs){
        vector<Rule> res;
        
        if(Rs == R("E")){
            res.push_back(R("FIN"));
        }

        for(auto rit = rule_table.cbegin(); rit != rule_table.cend(); ++rit){
            auto rule = rit->second; 
            if(rule == Rs) continue;

            for(auto r : rule->rules()){
                for(size_t i = 1; i < r.size(); i++){
                    if(string(*r[i]) == string(*Rs)){
                        if(i + 1 < r.size()){                            
                            auto ext = first(r[i+1]);
                            if(find(ext.begin(), ext.end(), R("Epsilon")) != ext.end()){
                                auto left = follow(rule);
                                res.insert(res.end(), left.begin(), left.end());
                            }
                            ext.erase(remove(ext.begin(), ext.end(), R("Epsilon")), ext.end());
                            res.insert(res.end(), ext.begin(), ext.end());
                        }else{
                            auto left = follow(rule);
                            res.insert(res.end(), left.begin(), left.end());
                        }
                    }
                }
            }
        }
        return res;
    }

    class Item{

    };

    class Sign{

    };


    void setup(){

        string r1 = "E";
        string r2 = "Eq";
        string r3 = "T";
        string r4 = "Tq";
        string r5 = "F";

        cR(r1);
        cR(r2);
        cR(r3);
        cR(r4);
        cR(r5);

        cR("FIN",true);
        cR("Epsilon",true);

        R(r1)->add(
            {R(r3), R(r2)}
        );

        R(r2)->add(
            {R("+"), R(r3), R(r2)}
        );
        R(r2)->add(
            {R("Epsilon")}
        );
        R(r3)->add(<commit_msg>[Add] Item, Sign<commit_after>#include <functional>
#include <initializer_list>
#include <string>
#include <vector>
#include <unordered_map>
#include <algorithm>


#include <iostream>

#include <memory>

namespace parser{

    using namespace std;

    vector<Sing> first(Sign sig){
        if(Rs->isTerm()){
            return {Rs};
        }
        vector<Rule> res; 
        for(auto& r : Rs->rules()){
            if(!r.empty()){
                auto ext = first(r[0]);
                if(find(ext.begin(), ext.end(), R("Epsilon")) != ext.end()){
                    ext.erase(remove(ext.begin(), ext.end(), R("Epsilon")), ext.end());
                    res.insert(res.end(), ext.begin(), ext.end());
                    if(r.size() >= 2){
                        auto nxt = first(r[1]);
                        res.insert(res.end(), nxt.begin(), nxt.end());
                    }else{
                        res.push_back(R("Epsilon"));
                    }
                }
                res.insert(res.end(), ext.begin(), ext.end());
            }
        }
        return res;
    }

    vector<Sing> first(initializer_list<Sign>& l){
        if(l.size() == 0)
            return {R("Epsilon")};

        vector<Rule> res;
        
        auto it = l.begin();
        if(*it == R("Epsilon")) return {R("Epsilon")};
        if((*it)->isTerm()) return {*it};

        auto ext = first(*it); 
        if(find(ext.begin(), ext.end(), R("Epsilon")) != ext.end()){
            ext.erase(remove(ext.begin(), ext.end(), R("Epsilon")), ext.end());
            res.insert(res.end(), ext.begin(), ext.end());                
            if(l.size() >= 2 ){
                it++;
                auto next = first(*it);
                res.insert(res.end(), next.begin(), next.end());
            }
            return res;
        }else{
            return ext;
        }
    }

    vector<Sing> follow(Sign& Rs){
        vector<Rule> res;
        
        if(Rs == R("E")){
            res.push_back(R("FIN"));
        }

        for(auto rit = rule_table.cbegin(); rit != rule_table.cend(); ++rit){
            auto rule = rit->second; 
            if(rule == Rs) continue;

            for(auto r : rule->rules()){
                for(size_t i = 1; i < r.size(); i++){
                    if(string(*r[i]) == string(*Rs)){
                        if(i + 1 < r.size()){                            
                            auto ext = first(r[i+1]);
                            if(find(ext.begin(), ext.end(), R("Epsilon")) != ext.end()){
                                auto left = follow(rule);
                                res.insert(res.end(), left.begin(), left.end());
                            }
                            ext.erase(remove(ext.begin(), ext.end(), R("Epsilon")), ext.end());
                            res.insert(res.end(), ext.begin(), ext.end());
                        }else{
                            auto left = follow(rule);
                            res.insert(res.end(), left.begin(), left.end());
                        }
                    }
                }
            }
        }
        return res;
    }


    class Sign{
        string name_;
        bool isTerm_;
      public:
        Sign(string n, bool t){
            name_ = n;
            isTerm_ = t;
        }
        bool isTerm(){
            return isTerm_;
        }
        string name(){
            return name_;
        }
    };

    class Item{
        Sgin left;
        vector<Sign> rights;
        int pos;
      public:
        Item(Sing l, initializer_list<Rule>& r){
            left = l;
            rights(r.begin(), r.end());
        }
        Sign nextSign(){
            if(right.size() > pos)
                return right[pos];
            return nullptr;
        }
        void next(){
            pos++;
        }
    };

    void setup(){

        string r1 = "E";
        string r2 = "Eq";
        string r3 = "T";
        string r4 = "Tq";
        string r5 = "F";

        cR(r1);
        cR(r2);
        cR(r3);
        cR(r4);
        cR(r5);

        cR("FIN",true);
        cR("Epsilon",true);

        R(r1)->add(
            {R(r3), R(r2)}
        );

        R(r2)->add(
            {R("+"), R(r3), R(r2)}
        );
        R(r2)->add(
            {R("Epsilon")}
        );
        R(r3)->add(
{'fraction_non_alphanumeric': 0.1742697327532629, 'fraction_numerical': 0.006463642013673089, 'mean_word_length': 1.1588408907968877, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2161974', 'n_tokens_mistral': 2474, 'n_tokens_neox': 2364, 'n_words': 468}