<commit_before>#include "queries.h"
#include <unordered_map>

using namespace Trinity;

namespace
{
        static constexpr bool traceParser{false};
};

static constexpr uint8_t OpPrio(const Operator op) noexcept
{
        switch (op)
        {
                case Operator::STRICT_AND:
                        return 8;

                case Operator::AND:
                        return 8;

                case Operator::NOT:
                        return 8;

                case Operator::OR:
                        return 7;

                default:
                        return 0;
        }
}

static std::pair<str32_t, range_base<uint16_t, uint16_t>> parse_term(ast_parser &ctx, const bool in_phrase)
{
        // Will strip characters that are not part of a valid token
        // and will pay attention to special group termination characters
        for (;;)
        {
                if (const auto pair = ctx.token_parser(ctx.content, ctx.lastParsedToken, in_phrase); pair.second)
                {
                        const auto o = ctx.content.data() - ctx.contentBase;

                        ctx.content.strip_prefix(pair.first);

                        // for e.g san francisco-based
                        // after we have parsed 'francisco' we don't want to
                        // consider '-' as a NOT operator
                        while (ctx.content.size() && ctx.content.front() == '-')
                                ctx.content.strip_prefix(1);

                        if (unlikely(pair.second > Trinity::Limits::MaxTermLength))
                        {
                                // TODO: what's the right thing to do here?
                                return {{}, {}};
                        }
                        else
                        {
                                return {{ctx.lastParsedToken, pair.second}, {uint16_t(o), uint16_t(pair.first)}};
                        }
                }
                else if (pair.first)
                {
                        // We may still have skipped past some content  if we didn't consume any
                        ctx.content.strip_prefix(pair.first);
                        continue;
                }

                if (ctx.content.empty() || (ctx.groupTerm.size() && ctx.groupTerm.back() == ctx.content.front()))
                {
                        return {{}, {}};
                }
                else if (ctx.content)
                {
                        // whitespace or other content here
                        ctx.content.strip_prefix(1);
                }
        }
}

static ast_node *parse_phrase_or_token(ast_parser &ctx)
{
        ctx.skip_ws();
        if (ctx.content && ctx.content.StripPrefix(_S("\"")))
        {
                auto &terms = ctx.terms;
                uint8_t n{0};
                term t;
                range_base<uint16_t, uint16_t> range;
                const auto b = ctx.content.data();

                range.offset = b - ctx.contentBase;
                for (;;)
                {
                        ctx.skip_ws();
                        range.len = ctx.content.data() - b;
                        if (!ctx.content || ctx.content.StripPrefix(_S("\"")))
                                break;

                        if (const auto pair = parse_term(ctx, true); const auto token = pair.first)
                        {
                                if (unlikely(token.size() > Limits::MaxTermLength))
                                        return ctx.alloc_node(ast_node::Type::ConstFalse);

                                t.token.Set(token.data(), uint8_t(token.size()));

                                if (n != sizeof_array(terms))
                                {
                                        // silently ignore the rest
                                        // Not sure this is ideal though
                                        ctx.track_term(t);
                                        terms[n++] = t;
                                }
                        }
                        else if (ctx.content)
                                ctx.content.strip_prefix(1);
                }

                if (!n)
                        return nullptr;
                else
                {
                        auto node = ctx.alloc_node(ast_node::Type::Phrase);
                        auto p = (phrase *)ctx.allocator.Alloc(sizeof(phrase) + sizeof(term) * n);

                        p->size = n;
                        std::copy(terms, terms + n, p->terms);
                        p->rep = 1;
                        p->toNextSpan = n;
                        p->rewrite_ctx.range.reset();
                        p->rewrite_ctx.srcSeqSize = 0;
                        p->rewrite_ctx.translationCoefficient = 1.0;<commit_msg>parse_phrase_or_token() now properly sets toNextSpan of new phrase to 0<commit_after>#include "queries.h"
#include <unordered_map>

using namespace Trinity;

namespace
{
        static constexpr bool traceParser{false};
};

static constexpr uint8_t OpPrio(const Operator op) noexcept
{
        switch (op)
        {
                case Operator::STRICT_AND:
                        return 8;

                case Operator::AND:
                        return 8;

                case Operator::NOT:
                        return 8;

                case Operator::OR:
                        return 7;

                default:
                        return 0;
        }
}

static std::pair<str32_t, range_base<uint16_t, uint16_t>> parse_term(ast_parser &ctx, const bool in_phrase)
{
        // Will strip characters that are not part of a valid token
        // and will pay attention to special group termination characters
        for (;;)
        {
                if (const auto pair = ctx.token_parser(ctx.content, ctx.lastParsedToken, in_phrase); pair.second)
                {
                        const auto o = ctx.content.data() - ctx.contentBase;

                        ctx.content.strip_prefix(pair.first);

                        // for e.g san francisco-based
                        // after we have parsed 'francisco' we don't want to
                        // consider '-' as a NOT operator
                        while (ctx.content.size() && ctx.content.front() == '-')
                                ctx.content.strip_prefix(1);

                        if (unlikely(pair.second > Trinity::Limits::MaxTermLength))
                        {
                                // TODO: what's the right thing to do here?
                                return {{}, {}};
                        }
                        else
                        {
                                return {{ctx.lastParsedToken, pair.second}, {uint16_t(o), uint16_t(pair.first)}};
                        }
                }
                else if (pair.first)
                {
                        // We may still have skipped past some content  if we didn't consume any
                        ctx.content.strip_prefix(pair.first);
                        continue;
                }

                if (ctx.content.empty() || (ctx.groupTerm.size() && ctx.groupTerm.back() == ctx.content.front()))
                {
                        return {{}, {}};
                }
                else if (ctx.content)
                {
                        // whitespace or other content here
                        ctx.content.strip_prefix(1);
                }
        }
}

static ast_node *parse_phrase_or_token(ast_parser &ctx)
{
        ctx.skip_ws();
        if (ctx.content && ctx.content.StripPrefix(_S("\"")))
        {
                auto &terms = ctx.terms;
                uint8_t n{0};
                term t;
                range_base<uint16_t, uint16_t> range;
                const auto b = ctx.content.data();

                range.offset = b - ctx.contentBase;
                for (;;)
                {
                        ctx.skip_ws();
                        range.len = ctx.content.data() - b;
                        if (!ctx.content || ctx.content.StripPrefix(_S("\"")))
                                break;

                        if (const auto pair = parse_term(ctx, true); const auto token = pair.first)
                        {
                                if (unlikely(token.size() > Limits::MaxTermLength))
                                        return ctx.alloc_node(ast_node::Type::ConstFalse);

                                t.token.Set(token.data(), uint8_t(token.size()));

                                if (n != sizeof_array(terms))
                                {
                                        // silently ignore the rest
                                        // Not sure this is ideal though
                                        ctx.track_term(t);
                                        terms[n++] = t;
                                }
                        }
                        else if (ctx.content)
                                ctx.content.strip_prefix(1);
                }

                if (!n)
                        return nullptr;
                else
                {
                        auto node = ctx.alloc_node(ast_node::Type::Phrase);
                        auto p = (phrase *)ctx.allocator.Alloc(sizeof(phrase) + sizeof(term) * n);

                        p->size = n;
                        std::copy(terms, terms + n, p->terms);
                        p->rep = 1;
                        p->toNextSpan = DefaultToNextSpan;
                        p->rewrite_ctx.range.reset();
                        p->rewrite_ctx.srcSeqSize = 0;
                        p->rewrite_ctx.translationCoefficient = 1.0;
{'fraction_non_alphanumeric': 0.09471320677689324, 'fraction_numerical': 0.006225760359256991, 'mean_word_length': 0.9281778827233372, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '593460', 'n_tokens_mistral': 2463, 'n_tokens_neox': 2208, 'n_words': 522}