<commit_before><?php

namespace QueryTranslator\Languages\Galach;

use QueryTranslator\Languages\Galach\Values\Node\Group;
use QueryTranslator\Languages\Galach\Values\Node\LogicalAnd;
use QueryTranslator\Languages\Galach\Values\Node\LogicalNot;
use QueryTranslator\Languages\Galach\Values\Node\LogicalOr;
use QueryTranslator\Languages\Galach\Values\Node\Mandatory;
use QueryTranslator\Languages\Galach\Values\Node\Prohibited;
use QueryTranslator\Languages\Galach\Values\Node\Query;
use QueryTranslator\Languages\Galach\Values\Node\Term;
use QueryTranslator\Languages\Galach\Values\Token\GroupBegin;
use QueryTranslator\Parsing;
use QueryTranslator\Values\Correction;
use QueryTranslator\Values\Node;
use QueryTranslator\Values\SyntaxTree;
use QueryTranslator\Values\Token;
use QueryTranslator\Values\TokenSequence;
use SplStack;

/**
 * Galach implementation of the Parsing interface.
 */
final class Parser implements Parsing
{
    /**
     * Parser ignored unary operator preceding another operator.
     */
    const CORRECTION_ADJACENT_UNARY_OPERATOR_PRECEDING_OPERATOR_IGNORED = 0;

    /**
     * Parser ignored unary operator missing operand.
     */
    const CORRECTION_UNARY_OPERATOR_MISSING_OPERAND_IGNORED = 1;

    /**
     * Parser ignored binary operator missing left side operand.
     */
    const CORRECTION_BINARY_OPERATOR_MISSING_LEFT_OPERAND_IGNORED = 2;

    /**
     * Parser ignored binary operator missing right side operand.
     */
    const CORRECTION_BINARY_OPERATOR_MISSING_RIGHT_OPERAND_IGNORED = 3;

    /**
     * Parser ignored binary operator following another operator and connecting operators.
     */
    const CORRECTION_BINARY_OPERATOR_FOLLOWING_OPERATOR_IGNORED = 4;

    /**
     * Parser ignored logical not operators preceding preference (mandatory/prohibited) operator.
     */
    const CORRECTION_LOGICAL_NOT_OPERATORS_PRECEDING_PREFERENCE_IGNORED = 5;

    /**
     * Parser ignored empty group and connecting operators.
     */
    const CORRECTION_EMPTY_GROUP_IGNORED = 6;

    /**
     * Parser ignored unmatched left side group delimiter.
     */
    const CORRECTION_UNMATCHED_GROUP_LEFT_DELIMITER_IGNORED = 7;

    /**
     * Parser ignored unmatched right side group delimiter.
     */
    const CORRECTION_UNMATCHED_GROUP_RIGHT_DELIMITER_IGNORED = 8;

    /**
     * Parser ignored bailout type token.
     *
     * @see \QueryTranslator\Languages\Galach\Tokenizer::TOKEN_BAILOUT
     */
    const CORRECTION_BAILOUT_TOKEN_IGNORED = 9;

    private static $tokenShortcuts = [
        'operatorNot' => Tokenizer::TOKEN_LOGICAL_NOT | Tokenizer::TOKEN_LOGICAL_NOT_2,
        'operatorPreference' => Tokenizer::TOKEN_MANDATORY | Tokenizer::TOKEN_PROHIBITED,
        'operatorPrefix' => Tokenizer::TOKEN_MANDATORY | Tokenizer::TOKEN_PROHIBITED | Tokenizer::TOKEN_LOGICAL_NOT_2,
        'operatorUnary' => Tokenizer::TOKEN_MANDATORY | Tokenizer::TOKEN_PROHIBITED | Tokenizer::TOKEN_LOGICAL_NOT | Tokenizer::TOKEN_LOGICAL_NOT_2,
        'operatorBinary' => Tokenizer::TOKEN_LOGICAL_AND | Tokenizer::TOKEN_LOGICAL_OR,
        'operator' => Tokenizer::TOKEN_LOGICAL_AND | Tokenizer::TOKEN_LOGICAL_OR | Tokenizer::TOKEN_MANDATORY | Tokenizer::TOKEN_PROHIBITED | Tokenizer::TOKEN_LOGICAL_NOT | Tokenizer::TOKEN_LOGICAL_NOT_2,
        'groupDelimiter' => Tokenizer::TOKEN_GROUP_BEGIN | Tokenizer::TOKEN_GROUP_END,
        'binaryOperatorAndWhitespace' => Tokenizer::TOKEN_LOGICAL_AND | Tokenizer::TOKEN_LOGICAL_OR | Tokenizer::TOKEN_WHITESPACE,
    ];

    private static $shifts = [
        Tokenizer::TOKEN_WHITESPACE => 'shiftWhitespace',
        Tokenizer::TOKEN_TERM => 'shiftTerm',
        Tokenizer::TOKEN_GROUP_BEGIN => 'shiftGroupBegin',
        Tokenizer::TOKEN_GROUP_END => 'shiftGroupEnd',
        Tokenizer::TOKEN_LOGICAL_AND => 'shiftBinaryOperator',
        Tokenizer::TOKEN_LOGICAL_OR => 'shiftBinaryOperator',
        Tokenizer::TOKEN_LOGICAL_NOT => 'shiftLogicalNot',
        Tokenizer::TOKEN_LOGICAL_NOT_2 => 'shiftLogicalNot2',
        Tokenizer::TOKEN_MANDATORY => 'shiftPreference',
        Tokenizer::TOKEN_PROHIBITED => 'shiftPreference',
        Tokenizer::TOKEN_BAILOUT => 'shiftBailout',
    ];

    private static $nodeToReductionGroup = [
        Group::class => 'group',
        LogicalAnd::class => 'logicalAnd',
        LogicalOr::class => 'logicalOr',
        LogicalNot::class => 'unaryOperator',
        Mandatory::class => 'unaryOperator',
        Prohibited::class => 'unaryOperator',
        Term::class => 'term',
    ];

    private static $reductionGroups = [
        'group' => [
            'reduceGroup',
            'reducePreference',
            'reduceLogicalNot',
            'reduceLogicalAnd',
            'reduceLogicalOr',
        ],
        'unaryOperator' => [
            'reduceLogicalNot',
            'reduceLogicalAnd',
            'reduceLogicalOr',
        ],
        'logicalOr' => [],
        'logicalAnd' => [
            'reduceLogicalOr',
        ],
        'term' => [
            'reducePreference',
            'reduceLogicalNot',
            'reduceLogicalAnd',
            'reduceLogicalOr',
        ],
    ];

    /**
     * Input tokens.
     *
     * @var \QueryTranslator\Values\Token[]
     */
    private $tokens;

    /**
     * Query stack.
     *
     * @var \SplStack
     */
    private $stack;

    /**
     * An array of applied corrections.
     *
     * @var \QueryTranslator\Values\Correction[]
     */
    private $corrections = [];

    public function parse(TokenSequence $tokenSequence)
    {
        $this->init($tokenSequence->tokens);

        while (!empty($this->tokens)) {
            $node = $this->shift();

            if ($node instanceof Node) {
                $this->reduce($node);
            }
        }

        $this->reduceQuery();

        return new SyntaxTree($this->stack->top(), $tokenSequence, $this->corrections);
    }

    protected function shift()
    {
        $token = array_shift($this->tokens);
        $shift = self::$shifts[$token->type];

        return $this->{$shift}($token);
    }

    protected function reduce(Node $node)
    {
        $previousNode = null;
        $reductionIndex = null;

        while ($node instanceof Node) {
            // Reset reduction index on first iteration or on Node change
            if ($node !== $previousNode) {
                $reductionIndex = 0;
            }

            // If there are no reductions to try, put the Node on the stack
            // and continue shifting
            $reduction = $this->getReduction($node, $reductionIndex);
            if ($reduction === null) {
                $this->stack->push($node);
                break;
            }

            $previousNode = $node;
            $node = $this->{$reduction}($node);
            ++$reductionIndex;
        }
    }
<commit_msg>Make shift() and reduce() private<commit_after><?php

namespace QueryTranslator\Languages\Galach;

use QueryTranslator\Languages\Galach\Values\Node\Group;
use QueryTranslator\Languages\Galach\Values\Node\LogicalAnd;
use QueryTranslator\Languages\Galach\Values\Node\LogicalNot;
use QueryTranslator\Languages\Galach\Values\Node\LogicalOr;
use QueryTranslator\Languages\Galach\Values\Node\Mandatory;
use QueryTranslator\Languages\Galach\Values\Node\Prohibited;
use QueryTranslator\Languages\Galach\Values\Node\Query;
use QueryTranslator\Languages\Galach\Values\Node\Term;
use QueryTranslator\Languages\Galach\Values\Token\GroupBegin;
use QueryTranslator\Parsing;
use QueryTranslator\Values\Correction;
use QueryTranslator\Values\Node;
use QueryTranslator\Values\SyntaxTree;
use QueryTranslator\Values\Token;
use QueryTranslator\Values\TokenSequence;
use SplStack;

/**
 * Galach implementation of the Parsing interface.
 */
final class Parser implements Parsing
{
    /**
     * Parser ignored unary operator preceding another operator.
     */
    const CORRECTION_ADJACENT_UNARY_OPERATOR_PRECEDING_OPERATOR_IGNORED = 0;

    /**
     * Parser ignored unary operator missing operand.
     */
    const CORRECTION_UNARY_OPERATOR_MISSING_OPERAND_IGNORED = 1;

    /**
     * Parser ignored binary operator missing left side operand.
     */
    const CORRECTION_BINARY_OPERATOR_MISSING_LEFT_OPERAND_IGNORED = 2;

    /**
     * Parser ignored binary operator missing right side operand.
     */
    const CORRECTION_BINARY_OPERATOR_MISSING_RIGHT_OPERAND_IGNORED = 3;

    /**
     * Parser ignored binary operator following another operator and connecting operators.
     */
    const CORRECTION_BINARY_OPERATOR_FOLLOWING_OPERATOR_IGNORED = 4;

    /**
     * Parser ignored logical not operators preceding preference (mandatory/prohibited) operator.
     */
    const CORRECTION_LOGICAL_NOT_OPERATORS_PRECEDING_PREFERENCE_IGNORED = 5;

    /**
     * Parser ignored empty group and connecting operators.
     */
    const CORRECTION_EMPTY_GROUP_IGNORED = 6;

    /**
     * Parser ignored unmatched left side group delimiter.
     */
    const CORRECTION_UNMATCHED_GROUP_LEFT_DELIMITER_IGNORED = 7;

    /**
     * Parser ignored unmatched right side group delimiter.
     */
    const CORRECTION_UNMATCHED_GROUP_RIGHT_DELIMITER_IGNORED = 8;

    /**
     * Parser ignored bailout type token.
     *
     * @see \QueryTranslator\Languages\Galach\Tokenizer::TOKEN_BAILOUT
     */
    const CORRECTION_BAILOUT_TOKEN_IGNORED = 9;

    private static $tokenShortcuts = [
        'operatorNot' => Tokenizer::TOKEN_LOGICAL_NOT | Tokenizer::TOKEN_LOGICAL_NOT_2,
        'operatorPreference' => Tokenizer::TOKEN_MANDATORY | Tokenizer::TOKEN_PROHIBITED,
        'operatorPrefix' => Tokenizer::TOKEN_MANDATORY | Tokenizer::TOKEN_PROHIBITED | Tokenizer::TOKEN_LOGICAL_NOT_2,
        'operatorUnary' => Tokenizer::TOKEN_MANDATORY | Tokenizer::TOKEN_PROHIBITED | Tokenizer::TOKEN_LOGICAL_NOT | Tokenizer::TOKEN_LOGICAL_NOT_2,
        'operatorBinary' => Tokenizer::TOKEN_LOGICAL_AND | Tokenizer::TOKEN_LOGICAL_OR,
        'operator' => Tokenizer::TOKEN_LOGICAL_AND | Tokenizer::TOKEN_LOGICAL_OR | Tokenizer::TOKEN_MANDATORY | Tokenizer::TOKEN_PROHIBITED | Tokenizer::TOKEN_LOGICAL_NOT | Tokenizer::TOKEN_LOGICAL_NOT_2,
        'groupDelimiter' => Tokenizer::TOKEN_GROUP_BEGIN | Tokenizer::TOKEN_GROUP_END,
        'binaryOperatorAndWhitespace' => Tokenizer::TOKEN_LOGICAL_AND | Tokenizer::TOKEN_LOGICAL_OR | Tokenizer::TOKEN_WHITESPACE,
    ];

    private static $shifts = [
        Tokenizer::TOKEN_WHITESPACE => 'shiftWhitespace',
        Tokenizer::TOKEN_TERM => 'shiftTerm',
        Tokenizer::TOKEN_GROUP_BEGIN => 'shiftGroupBegin',
        Tokenizer::TOKEN_GROUP_END => 'shiftGroupEnd',
        Tokenizer::TOKEN_LOGICAL_AND => 'shiftBinaryOperator',
        Tokenizer::TOKEN_LOGICAL_OR => 'shiftBinaryOperator',
        Tokenizer::TOKEN_LOGICAL_NOT => 'shiftLogicalNot',
        Tokenizer::TOKEN_LOGICAL_NOT_2 => 'shiftLogicalNot2',
        Tokenizer::TOKEN_MANDATORY => 'shiftPreference',
        Tokenizer::TOKEN_PROHIBITED => 'shiftPreference',
        Tokenizer::TOKEN_BAILOUT => 'shiftBailout',
    ];

    private static $nodeToReductionGroup = [
        Group::class => 'group',
        LogicalAnd::class => 'logicalAnd',
        LogicalOr::class => 'logicalOr',
        LogicalNot::class => 'unaryOperator',
        Mandatory::class => 'unaryOperator',
        Prohibited::class => 'unaryOperator',
        Term::class => 'term',
    ];

    private static $reductionGroups = [
        'group' => [
            'reduceGroup',
            'reducePreference',
            'reduceLogicalNot',
            'reduceLogicalAnd',
            'reduceLogicalOr',
        ],
        'unaryOperator' => [
            'reduceLogicalNot',
            'reduceLogicalAnd',
            'reduceLogicalOr',
        ],
        'logicalOr' => [],
        'logicalAnd' => [
            'reduceLogicalOr',
        ],
        'term' => [
            'reducePreference',
            'reduceLogicalNot',
            'reduceLogicalAnd',
            'reduceLogicalOr',
        ],
    ];

    /**
     * Input tokens.
     *
     * @var \QueryTranslator\Values\Token[]
     */
    private $tokens;

    /**
     * Query stack.
     *
     * @var \SplStack
     */
    private $stack;

    /**
     * An array of applied corrections.
     *
     * @var \QueryTranslator\Values\Correction[]
     */
    private $corrections = [];

    public function parse(TokenSequence $tokenSequence)
    {
        $this->init($tokenSequence->tokens);

        while (!empty($this->tokens)) {
            $node = $this->shift();

            if ($node instanceof Node) {
                $this->reduce($node);
            }
        }

        $this->reduceQuery();

        return new SyntaxTree($this->stack->top(), $tokenSequence, $this->corrections);
    }

    private function shift()
    {
        $token = array_shift($this->tokens);
        $shift = self::$shifts[$token->type];

        return $this->{$shift}($token);
    }

    private function reduce(Node $node)
    {
        $previousNode = null;
        $reductionIndex = null;

        while ($node instanceof Node) {
            // Reset reduction index on first iteration or on Node change
            if ($node !== $previousNode) {
                $reductionIndex = 0;
            }

            // If there are no reductions to try, put the Node on the stack
            // and continue shifting
            $reduction = $this->getReduction($node, $reductionIndex);
            if ($reduction === null) {
                $this->stack->push($node);
                break;
            }

            $previousNode = $node;
            $node = $this->{$reduction}($node);
            ++$reductionIndex;
        }
    }

{'fraction_non_alphanumeric': 0.10964719660371834, 'fraction_numerical': 0.002488654662567706, 'mean_word_length': 2.8132849567401617, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2245801', 'n_tokens_mistral': 4223, 'n_tokens_neox': 4140, 'n_words': 716}