<commit_before>#include "catch.hpp"
#include "util.h"
#include <grammar_loader.h>

using namespace shl;

TEST_CASE("GrammarLoader Test") {
    string data = load_string("fixture/valid.json");
    GrammarLoader loader;
    Grammar g = loader.load(data);

    SECTION("can load a valid grammar") {
        REQUIRE(g.desc == "C");
        REQUIRE(g.file_types.size() == 2);
        REQUIRE(g.name == "source.c");

        REQUIRE(g.repository.size() == 2);
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin.source() == "^\\s*(#(if)\\s+(0*1)\\b)");
    }

    SECTION("unused repository is ignored") {
        REQUIRE(g.repository.size() == 2);
    }

    SECTION("captures can be regard as begin_captures if is a begin/end rule and has no begin_captures") {
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin_captures.size() == 3);
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin_captures[2] == "keyword.control.import.if.c");
    }

    SECTION("match rule merged with begin, and captures merge with begin_captures") {
        REQUIRE(g.repository["string_escaped_char"].patterns[1].begin.source() == "\\\\.");
        REQUIRE(g.patterns[1].begin_captures.size() == 1);
        REQUIRE(g.patterns[1].begin.source() == "\"");
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin.source() == "^\\s*(#(if)\\s+(0*1)\\b)");
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin_captures.size() == 3);
    }


    SECTION("can resolve includes") {
        REQUIRE(g.patterns[0].include == &g.repository["preprocessor-rule-enabled"]);
        REQUIRE(g.patterns[1].patterns[0].include == &g.repository["string_escaped_char"]);
        REQUIRE(g.patterns[1].patterns[1].include == &g.patterns[1]);<commit_msg>test captures as end_captures behaviour<commit_after>#include "catch.hpp"
#include "util.h"
#include <grammar_loader.h>

using namespace shl;

TEST_CASE("GrammarLoader Test") {
    string data = load_string("fixture/valid.json");
    GrammarLoader loader;
    Grammar g = loader.load(data);

    SECTION("can load a valid grammar") {
        REQUIRE(g.desc == "C");
        REQUIRE(g.file_types.size() == 2);
        REQUIRE(g.name == "source.c");

        REQUIRE(g.repository.size() == 2);
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin.source() == "^\\s*(#(if)\\s+(0*1)\\b)");
    }

    SECTION("unused repository is ignored") {
        REQUIRE(g.repository.size() == 2);
    }

    SECTION("captures can be regard as begin_captures if is a begin/end rule and has no begin_captures") {
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin_captures.size() == 3);
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin_captures[2] == "keyword.control.import.if.c");
    }

    SECTION("captures can be regard as end_captures if is a begin/end rule and has no end_captures") {
        REQUIRE(g.repository["preprocessor-rule-enabled"].end_captures.size() == 3);
        REQUIRE(g.repository["preprocessor-rule-enabled"].end_captures[2] == "keyword.control.import.if.c");
    }

    SECTION("match rule merged with begin, and captures merge with begin_captures") {
        REQUIRE(g.repository["string_escaped_char"].patterns[1].begin.source() == "\\\\.");
        REQUIRE(g.patterns[1].begin_captures.size() == 1);
        REQUIRE(g.patterns[1].begin.source() == "\"");
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin.source() == "^\\s*(#(if)\\s+(0*1)\\b)");
        REQUIRE(g.repository["preprocessor-rule-enabled"].begin_captures.size() == 3);
    }


    SECTION("can resolve includes") {
        REQUIRE(g.patterns[0].include == &g.repository["preprocessor-rule-enabled"]);
        REQUIRE(g.patterns[1].patterns[0].include == &g.repository["string_escaped_char"]);
        REQUIRE(g.patterns[1].patterns[1].include == &g.patterns[1]);
{'fraction_non_alphanumeric': 0.18568464730290457, 'fraction_numerical': 0.010892116182572614, 'mean_word_length': 3.6081242532855438, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '636830', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1298, 'n_words': 194}