<commit_before>////////////////////////////////////////////////////////////////////////////
//
// Copyright 2015 Realm Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////

#include "parser.hpp"

#include <iostream>

#include <pegtl.hh>
#include <pegtl/analyze.hh>
#include <pegtl/trace.hh>

using namespace pegtl;

namespace realm {
namespace parser {

// strings
struct unicode : list< seq< one< 'u' >, rep< 4, must< xdigit > > >, one< '\\' > > {};
struct escaped_char : one< '"', '\\', '/', 'b', 'f', 'n', 'r', 't' > {};
struct escaped : sor< escaped_char, unicode > {};
struct unescaped : utf8::range< 0x20, 0x10FFFF > {};
struct char_ : if_then_else< one< '\\' >, must< escaped >, unescaped > {};

struct string_content : until< at< one< '"' > >, must< char_ > > {};
struct string : seq< one< '"' >, must< string_content >, any >
{
    using content = string_content;
};

// numbers
struct minus : opt< one< '-' > > {};
struct dot : one< '.' > {};

struct float_num : sor<
    seq< plus< digit >, dot, star< digit > >,
    seq< star< digit >, dot, plus< digit > >
> {};
struct hex_num : seq< one< '0' >, one< 'x', 'X' >, plus< xdigit > > {};
struct int_num : plus< digit > {};

struct number : seq< minus, sor< float_num, hex_num, int_num > > {};

// key paths
struct key_path : list< seq< sor< alpha, one< '_' > >, star< sor< alnum, one< '_', '-' > > > >, one< '.' > > {};

// expressions and operators
struct expr : sor< string, key_path, number > {};

struct eq : sor< two< '=' >, one< '=' > > {};
struct noteq : pegtl::string< '!', '=' > {};
struct lteq : pegtl::string< '<', '=' > {};
struct lt : one< '<' > {};
struct gteq : pegtl::string< '>', '=' > {};
struct gt : one< '<' > {};
struct begins : istring< 'b', 'e', 'g', 'i', 'n', 's', 'w', 'i', 't', 'h' > {};
struct ends : istring< 'e', 'n', 'd', 's', 'w', 'i', 't', 'h' > {};
struct contains : istring< 'c', 'o', 'n', 't', 'a', 'i', 'n', 's' > {};
struct oper : sor< eq, noteq, lteq, lt, gteq, gt, begins, ends, contains > {};

// predicates
struct comparison_pred : seq< expr, pad< oper, blank >, expr > {};

struct pred;
struct group_pred : if_must< one< '(' >, pad< pred, blank >, one< ')' > > {};

struct not_pre : sor< seq< one< '!' >, star< blank > >, seq< istring< 'N', 'O', 'T' >, plus< blank > > > {};
struct atom_pred : seq< opt< not_pre >, pad< sor< group_pred, comparison_pred >, blank > > {};

struct and_op : sor< two< '&' >, istring< 'A', 'N', 'D' > > {};
struct or_op : sor< two< '|' >, istring< 'O', 'R' > > {};

struct or_ext : seq< pad< or_op, blank >, pred > {};
struct and_ext : seq< pad< and_op, blank >, pred > {};
struct and_pred : seq< atom_pred, star< and_ext > > {};

struct pred : seq< and_pred, star< or_ext > > {};

// state
struct ParserState
{
    std::vector<Predicate *> predicate_stack;
    Predicate *current() { return predicate_stack.back(); }

    void addExpression(Expression exp)
    {
        if (current()->type == Predicate::Type::Comparison) {
            current()->sub_expressions.emplace_back(std::move(exp));
            predicate_stack.pop_back();
        }
        else {
            Predicate p;
            p.type = Predicate::Type::Comparison;
            p.sub_expressions.emplace_back(std::move(exp));
            current()->sub_predicates.emplace_back(std::move(p));
            predicate_stack.push_back(&current()->sub_predicates.back());
        }
    }
};

// rules
template< typename Rule >
struct action : nothing< Rule > {};

template<> struct action< and_ext >
{
    static void apply( const input & in, ParserState & state )
    {
        std::cout << "<and>" << in.string() << std::endl;
    }
};

template<> struct action< or_ext >
{
    static void apply( const input & in, ParserState & state )
    {
        std::cout << "<or>" << in.string() << std::endl;<commit_msg>support OR with proper precedence<commit_after>////////////////////////////////////////////////////////////////////////////
//
// Copyright 2015 Realm Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////

#include "parser.hpp"

#include <iostream>

#include <pegtl.hh>
#include <pegtl/analyze.hh>
#include <pegtl/trace.hh>

using namespace pegtl;

namespace realm {
namespace parser {

// strings
struct unicode : list< seq< one< 'u' >, rep< 4, must< xdigit > > >, one< '\\' > > {};
struct escaped_char : one< '"', '\\', '/', 'b', 'f', 'n', 'r', 't' > {};
struct escaped : sor< escaped_char, unicode > {};
struct unescaped : utf8::range< 0x20, 0x10FFFF > {};
struct char_ : if_then_else< one< '\\' >, must< escaped >, unescaped > {};

struct string_content : until< at< one< '"' > >, must< char_ > > {};
struct string : seq< one< '"' >, must< string_content >, any >
{
    using content = string_content;
};

// numbers
struct minus : opt< one< '-' > > {};
struct dot : one< '.' > {};

struct float_num : sor<
    seq< plus< digit >, dot, star< digit > >,
    seq< star< digit >, dot, plus< digit > >
> {};
struct hex_num : seq< one< '0' >, one< 'x', 'X' >, plus< xdigit > > {};
struct int_num : plus< digit > {};

struct number : seq< minus, sor< float_num, hex_num, int_num > > {};

// key paths
struct key_path : list< seq< sor< alpha, one< '_' > >, star< sor< alnum, one< '_', '-' > > > >, one< '.' > > {};

// expressions and operators
struct expr : sor< string, key_path, number > {};

struct eq : sor< two< '=' >, one< '=' > > {};
struct noteq : pegtl::string< '!', '=' > {};
struct lteq : pegtl::string< '<', '=' > {};
struct lt : one< '<' > {};
struct gteq : pegtl::string< '>', '=' > {};
struct gt : one< '<' > {};
struct begins : istring< 'b', 'e', 'g', 'i', 'n', 's', 'w', 'i', 't', 'h' > {};
struct ends : istring< 'e', 'n', 'd', 's', 'w', 'i', 't', 'h' > {};
struct contains : istring< 'c', 'o', 'n', 't', 'a', 'i', 'n', 's' > {};
struct oper : sor< eq, noteq, lteq, lt, gteq, gt, begins, ends, contains > {};

// predicates
struct comparison_pred : seq< expr, pad< oper, blank >, expr > {};

struct pred;
struct group_pred : if_must< one< '(' >, pad< pred, blank >, one< ')' > > {};

struct not_pre : sor< seq< one< '!' >, star< blank > >, seq< istring< 'N', 'O', 'T' >, plus< blank > > > {};
struct atom_pred : seq< opt< not_pre >, pad< sor< group_pred, comparison_pred >, blank > > {};

struct and_op : sor< two< '&' >, istring< 'A', 'N', 'D' > > {};
struct or_op : sor< two< '|' >, istring< 'O', 'R' > > {};

struct or_ext : seq< pad< or_op, blank >, pred > {};
struct and_ext : seq< pad< and_op, blank >, pred > {};
struct and_pred : seq< atom_pred, star< and_ext > > {};

struct pred : seq< and_pred, star< or_ext > > {};

// state
struct ParserState
{
    std::vector<Predicate *> predicate_stack;
    Predicate *current() { return predicate_stack.back(); }

    void addExpression(Expression exp)
    {
        if (current()->type == Predicate::Type::Comparison) {
            current()->sub_expressions.emplace_back(std::move(exp));
            predicate_stack.pop_back();
        }
        else {
            Predicate p;
            p.type = Predicate::Type::Comparison;
            p.sub_expressions.emplace_back(std::move(exp));
            current()->sub_predicates.emplace_back(std::move(p));
            predicate_stack.push_back(&current()->sub_predicates.back());
        }
    }
};

// rules
template< typename Rule >
struct action : nothing< Rule > {};

template<> struct action< and_ext >
{
    static void apply( const input & in, ParserState & state )
    {
        std::cout << "<and>" << in.string() << std::endl;

        // if we were put into an OR group we need to rearrange
        auto current = state.current();
        if (current->type == Predicate::Type::Or) {
            auto &sub_preds = state.current()->sub_predicates;
            auto second_last = sub_preds[sub_preds.size()-2];
            if (second_last.type == Predicate::Type::And) {
                // if we are in an OR group and second to last predicate group is
                // an AND group then move the last predicate inside
                second_last.sub_predicates.push_back(sub_preds.back());
                sub_preds.pop_back();
            }
            else {
                // otherwise combine last two into a new AND group
                Predicate pred;
                pred.type = Predicate::Type::And;
                pred.sub_predicates = { second_last, sub_preds.back() };
                sub_preds.pop_back();
                sub_preds.pop_back();
                sub_preds.push_back(pred);
            }
        }
    }
};

template<> struct action< or_ext >
{
    static void apply( const input & in, ParserState & state )
    {
        std::cout << "<or>" << in.string() << std::endl;

        // if already an OR group do nothing
        auto current = state.current();
        if (current->type == Predicate::Type::Or) {
            return;
        }

        // if only two predicates in the group, then convert to OR
        auto &sub_preds = state.current()->sub_predicates;
        if (sub_preds.size()) {
            current->type = Predicate::Type::Or;
            return;
        }

        // split the current group into to groups which are ORed together
        Predicate pred1, pred2;
        pred1.type = Predicate::Type::And;
        pred2.type = Predicate::Type::And;
        pred1.sub_predicates.insert(sub_preds.begin(), sub_preds.back());
        pred2.sub_predicates.push_back(sub_preds.back());

        current->type = Predicate::Type::Or;
        sub_preds = { pred1, pred2 };
{'fraction_non_alphanumeric': 0.20749740981444853, 'fraction_numerical': 0.004050108316850334, 'mean_word_length': 2.61156462585034, 'pattern_counts': {'":': 0, '<': 235, '<?xml version=': 0, '>': 221, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3259094', 'n_tokens_mistral': 3472, 'n_tokens_neox': 3412, 'n_words': 1010}