<commit_before>#include "vtpath.h" // we implement this

#include "tinfra/tstring.h"
#include "tinfra/any.h"
#include "tinfra/trace.h"

#include <vector>
#include <stack>
#include <deque>
#include <iterator>
#include <string>
#include <stdexcept>

namespace tinfra {

//
// vtpath_visit
// 
std::vector<const variant*> vtpath_visit(variant const& v, tstring const& expression)
{
    vtpath_visitor vtpv(const_cast<variant*>(&v), expression);
    
    return std::vector<const variant*>(vtpv.begin(), vtpv.end());
}

std::vector<variant*> vtpath_visit(variant& v, tstring const& expression)
{
    vtpath_visitor vtpv(&v, expression);
    
    return std::vector<variant*>(vtpv.begin(), vtpv.end());
}

//
// vtpath_visitor
// 

enum vpath_token_type {
    ROOT,
    CURRENT_OBJECT,
    CHILD,
    RECURSIVE_CHILD,
    WILDCARD_ALL,
    WILDCARD_OTHER,
    TOKEN
};
struct vtpath_command {
    vpath_token_type type;
    
    variant expr;
    
    vtpath_command(vpath_token_type t): type(t) {}
    vtpath_command(vpath_token_type t, variant expr): type(t), expr(expr) {}
};

//
// vtpath_parse
//
tinfra::module_tracer vtpath_parse_tracer(tinfra::tinfra_tracer, "vtpath.parse");

void vtpath_parse_fail(std::string const& message)
{
    throw std::runtime_error(tsprintf("vtpath: %s", message)); 
}

void vtpath_parse_expect(char v, char expect, const char* message)
{
    if( v != expect ) {
        vtpath_parse_fail(tsprintf("expected '%s', but found '%s': %s", expect, v, message)); 
    }
}


tstring vpath_parse_selector(tstring expr, std::vector<vtpath_command>& result)
    // this function expects string is just after parsing starting [
    // it will consume ending ]
    // only [*] supported for now
{
     while( expr.size() > 0 ) {
        switch(expr[0]) {
            case '*':
                TINFRA_TRACE(vtpath_parse_tracer, "[*] -> WILDCARD_ALL");
                result.push_back(vtpath_command(WILDCARD_ALL));
                vtpath_parse_expect(expr[1],']', "* is supported only as alone token in selector");
                return expr.substr(2);
            default:
                vtpath_parse_fail("only [*] wildcard is supported for now");
                break;
        }
     }
     return expr;
}

void vtpath_parse(tstring expr, std::vector<vtpath_command>& result)
{
    while( expr.size() > 0 ) {
        switch(expr[0]) {
        case '$':
            TINFRA_TRACE(vtpath_parse_tracer, "$ -> ROOT");
            result.push_back(vtpath_command(ROOT));
            expr=expr.substr(1);
            continue;
        case '.':
            if( expr.size() > 1 && expr[1] == '.' ) {
                TINFRA_TRACE(vtpath_parse_tracer, ".. -> RECURSIVE_CHILD");
                result.push_back(vtpath_command(RECURSIVE_CHILD));
                expr=expr.substr(2);
            } else {
                TINFRA_TRACE(vtpath_parse_tracer, ". -> CHILD");
                result.push_back(vtpath_command(CHILD));
                expr=expr.substr(1);
            }
            continue;
        case '*':
            TINFRA_TRACE(vtpath_parse_tracer, "* -> WILDCARD_ALL");
            result.push_back(vtpath_command(WILDCARD_ALL));
            expr=expr.substr(1);
            continue;
        case '[':
            expr = vpath_parse_selector(expr.substr(1), result);
            continue;
        default:
            {
                std::string name;
                while( expr.size() > 0 && isalpha(expr[0]))  {
                        name += expr[0];
                        expr=expr.substr(1);
                }
                TINFRA_TRACE(vtpath_parse_tracer, name << " -> TOKEN(" << name << ")");
                result.push_back(vtpath_command(TOKEN, name));
                continue;
            }
        }
    }
    
}

tinfra::module_tracer vtpath_exec_tracer(tinfra::tinfra_tracer, "vtpath.exec");

struct vtpath_parse_state {
    variant*                      current;
    <commit_msg>vtpath_parse: fix, word tokens parsing matches all isalnum() and underscore<commit_after>#include "vtpath.h" // we implement this

#include "tinfra/tstring.h"
#include "tinfra/any.h"
#include "tinfra/trace.h"

#include <vector>
#include <stack>
#include <deque>
#include <iterator>
#include <string>
#include <stdexcept>

namespace tinfra {

//
// vtpath_visit
// 
std::vector<const variant*> vtpath_visit(variant const& v, tstring const& expression)
{
    vtpath_visitor vtpv(const_cast<variant*>(&v), expression);
    
    return std::vector<const variant*>(vtpv.begin(), vtpv.end());
}

std::vector<variant*> vtpath_visit(variant& v, tstring const& expression)
{
    vtpath_visitor vtpv(&v, expression);
    
    return std::vector<variant*>(vtpv.begin(), vtpv.end());
}

//
// vtpath_visitor
// 

enum vpath_token_type {
    ROOT,
    CURRENT_OBJECT,
    CHILD,
    RECURSIVE_CHILD,
    WILDCARD_ALL,
    WILDCARD_OTHER,
    TOKEN
};
struct vtpath_command {
    vpath_token_type type;
    
    variant expr;
    
    vtpath_command(vpath_token_type t): type(t) {}
    vtpath_command(vpath_token_type t, variant expr): type(t), expr(expr) {}
};

//
// vtpath_parse
//
tinfra::module_tracer vtpath_parse_tracer(tinfra::tinfra_tracer, "vtpath.parse");

void vtpath_parse_fail(std::string const& message)
{
    throw std::runtime_error(tsprintf("vtpath: %s", message)); 
}

void vtpath_parse_expect(char v, char expect, const char* message)
{
    if( v != expect ) {
        vtpath_parse_fail(tsprintf("expected '%s', but found '%s': %s", expect, v, message)); 
    }
}


tstring vpath_parse_selector(tstring expr, std::vector<vtpath_command>& result)
    // this function expects string is just after parsing starting [
    // it will consume ending ]
    // only [*] supported for now
{
     while( expr.size() > 0 ) {
        switch(expr[0]) {
            case '*':
                TINFRA_TRACE(vtpath_parse_tracer, "[*] -> WILDCARD_ALL");
                result.push_back(vtpath_command(WILDCARD_ALL));
                vtpath_parse_expect(expr[1],']', "* is supported only as alone token in selector");
                return expr.substr(2);
            default:
                vtpath_parse_fail("only [*] wildcard is supported for now");
                break;
        }
     }
     return expr;
}

void vtpath_parse(tstring expr, std::vector<vtpath_command>& result)
{
    while( expr.size() > 0 ) {
        switch(expr[0]) {
        case '$':
            TINFRA_TRACE(vtpath_parse_tracer, "$ -> ROOT");
            result.push_back(vtpath_command(ROOT));
            expr=expr.substr(1);
            continue;
        case '.':
            if( expr.size() > 1 && expr[1] == '.' ) {
                TINFRA_TRACE(vtpath_parse_tracer, ".. -> RECURSIVE_CHILD");
                result.push_back(vtpath_command(RECURSIVE_CHILD));
                expr=expr.substr(2);
            } else {
                TINFRA_TRACE(vtpath_parse_tracer, ". -> CHILD");
                result.push_back(vtpath_command(CHILD));
                expr=expr.substr(1);
            }
            continue;
        case '*':
            TINFRA_TRACE(vtpath_parse_tracer, "* -> WILDCARD_ALL");
            result.push_back(vtpath_command(WILDCARD_ALL));
            expr=expr.substr(1);
            continue;
        case '[':
            expr = vpath_parse_selector(expr.substr(1), result);
            continue;
        default:
            {
                std::string name;
                while( expr.size() > 0 && (isalnum(expr[0]) || expr[0] == '_'))  {
                        name += expr[0];
                        expr=expr.substr(1);
                }
                TINFRA_TRACE(vtpath_parse_tracer, name << " -> TOKEN(" << name << ")");
                result.push_back(vtpath_command(TOKEN, name));
                continue;
            }
        }
    }
    
}

tinfra::module_tracer vtpath_exec_tracer(tinfra::tinfra_tracer, "vtpath.exec");

struct vtpath_parse_state {
    variant*                      current;
    
{'fraction_non_alphanumeric': 0.13512833417211878, 'fraction_numerical': 0.004403623553095118, 'mean_word_length': 2.2983402489626554, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6841691', 'n_tokens_mistral': 2684, 'n_tokens_neox': 2576, 'n_words': 462}