<commit_before>package org.nwapw.abacus.parsing;

import org.nwapw.abacus.Abacus;
import org.nwapw.abacus.function.Operator;
import org.nwapw.abacus.function.OperatorAssociativity;
import org.nwapw.abacus.function.OperatorType;
import org.nwapw.abacus.lexing.pattern.Match;
import org.nwapw.abacus.plugin.PluginListener;
import org.nwapw.abacus.plugin.PluginManager;
import org.nwapw.abacus.tree.*;

import java.util.*;

/**
 * A parser that uses shunting yard to rearranged matches into postfix
 * and then convert them into a parse tree.
 */
public class ShuntingYardParser implements Parser<Match<TokenType>>, PluginListener {

    /**
     * The Abacus instance used to create number instances.
     */
    private Abacus abacus;
    /**
     * Map of operator precedences, loaded from the plugin operators.
     */
    private Map<String, Integer> precedenceMap;
    /**
     * Map of operator associativity, loaded from the plugin operators.
     */
    private Map<String, OperatorAssociativity> associativityMap;
    /**
     * Map of operator types, loaded from plugin operators.
     */
    private Map<String, OperatorType> typeMap;

    /**
     * Creates a new Shunting Yard parser with the given Abacus instance.
     *
     * @param abacus the abacus instance.
     */
    public ShuntingYardParser(Abacus abacus) {
        this.abacus = abacus;
        precedenceMap = new HashMap<>();
        associativityMap = new HashMap<>();
        typeMap = new HashMap<>();
    }

    /**
     * Rearranges tokens into a postfix list, using Shunting Yard.
     *
     * @param from the tokens to be rearranged.
     * @return the resulting list of rearranged tokens.
     */
    public List<Match<TokenType>> intoPostfix(List<Match<TokenType>> from) {
        ArrayList<Match<TokenType>> output = new ArrayList<>();
        Stack<Match<TokenType>> tokenStack = new Stack<>();
        TokenType previousType;
        TokenType matchType = null;
        while (!from.isEmpty()) {
            Match<TokenType> match = from.remove(0);
            previousType = matchType;
            matchType = match.getType();
            if (matchType == TokenType.NUM) {
                output.add(match);
            } else if (matchType == TokenType.FUNCTION) {
                output.add(new Match<>("", TokenType.INTERNAL_FUNCTION_END));
                tokenStack.push(match);
            } else if (matchType == TokenType.OP) {
                String tokenString = match.getContent();
                OperatorType type = typeMap.get(tokenString);
                int precedence = precedenceMap.get(tokenString);
                OperatorAssociativity associativity = associativityMap.get(tokenString);

                if (type == OperatorType.UNARY_POSTFIX) {
                    output.add(match);
                    continue;
                }

                if(tokenString.equals("-") && (previousType == null || previousType == TokenType.OP ||
                        previousType == TokenType.OPEN_PARENTH)){
                    from.add(0, new Match<>("`", TokenType.OP));
                    continue;
                }

                while (!tokenStack.empty() && type == OperatorType.BINARY_INFIX) {
                    Match<TokenType> otherMatch = tokenStack.peek();
                    TokenType otherMatchType = otherMatch.getType();
                    if (!(otherMatchType == TokenType.OP || otherMatchType == TokenType.FUNCTION)) break;

                    if (otherMatchType == TokenType.OP) {
                        int otherPrecedence = precedenceMap.get(match.getContent());
                        if (otherPrecedence < precedence ||
                                (associativity == OperatorAssociativity.RIGHT && otherPrecedence == precedence)) {
                            break;
                        }
                    }
                    output.add(tokenStack.pop());
                }
                tokenStack.push(match);
            } else if (matchType == TokenType.OPEN_PARENTH) {
                tokenStack.push(match);
            } else if (matchType == TokenType.CLOSE_PARENTH || matchType == TokenType.COMMA) {
                while (!tokenStack.empty() && tokenStack.peek().getType() != TokenType.OPEN_PARENTH) {
                    output.add(tokenStack.pop());
                }
                if (tokenStack.empty()) return null;
                if (matchType == TokenType.CLOSE_PARENTH) {
                    tokenStack.pop();<commit_msg>Fix order of operations bug.<commit_after>package org.nwapw.abacus.parsing;

import org.nwapw.abacus.Abacus;
import org.nwapw.abacus.function.Operator;
import org.nwapw.abacus.function.OperatorAssociativity;
import org.nwapw.abacus.function.OperatorType;
import org.nwapw.abacus.lexing.pattern.Match;
import org.nwapw.abacus.plugin.PluginListener;
import org.nwapw.abacus.plugin.PluginManager;
import org.nwapw.abacus.tree.*;

import java.util.*;

/**
 * A parser that uses shunting yard to rearranged matches into postfix
 * and then convert them into a parse tree.
 */
public class ShuntingYardParser implements Parser<Match<TokenType>>, PluginListener {

    /**
     * The Abacus instance used to create number instances.
     */
    private Abacus abacus;
    /**
     * Map of operator precedences, loaded from the plugin operators.
     */
    private Map<String, Integer> precedenceMap;
    /**
     * Map of operator associativity, loaded from the plugin operators.
     */
    private Map<String, OperatorAssociativity> associativityMap;
    /**
     * Map of operator types, loaded from plugin operators.
     */
    private Map<String, OperatorType> typeMap;

    /**
     * Creates a new Shunting Yard parser with the given Abacus instance.
     *
     * @param abacus the abacus instance.
     */
    public ShuntingYardParser(Abacus abacus) {
        this.abacus = abacus;
        precedenceMap = new HashMap<>();
        associativityMap = new HashMap<>();
        typeMap = new HashMap<>();
    }

    /**
     * Rearranges tokens into a postfix list, using Shunting Yard.
     *
     * @param from the tokens to be rearranged.
     * @return the resulting list of rearranged tokens.
     */
    public List<Match<TokenType>> intoPostfix(List<Match<TokenType>> from) {
        ArrayList<Match<TokenType>> output = new ArrayList<>();
        Stack<Match<TokenType>> tokenStack = new Stack<>();
        TokenType previousType;
        TokenType matchType = null;
        while (!from.isEmpty()) {
            Match<TokenType> match = from.remove(0);
            previousType = matchType;
            matchType = match.getType();
            if (matchType == TokenType.NUM) {
                output.add(match);
            } else if (matchType == TokenType.FUNCTION) {
                output.add(new Match<>("", TokenType.INTERNAL_FUNCTION_END));
                tokenStack.push(match);
            } else if (matchType == TokenType.OP) {
                String tokenString = match.getContent();
                OperatorType type = typeMap.get(tokenString);
                int precedence = precedenceMap.get(tokenString);
                OperatorAssociativity associativity = associativityMap.get(tokenString);

                if (type == OperatorType.UNARY_POSTFIX) {
                    output.add(match);
                    continue;
                }

                if(tokenString.equals("-") && (previousType == null || previousType == TokenType.OP ||
                        previousType == TokenType.OPEN_PARENTH)){
                    from.add(0, new Match<>("`", TokenType.OP));
                    continue;
                }

                while (!tokenStack.empty() && type == OperatorType.BINARY_INFIX) {
                    Match<TokenType> otherMatch = tokenStack.peek();
                    TokenType otherMatchType = otherMatch.getType();
                    if (!(otherMatchType == TokenType.OP || otherMatchType == TokenType.FUNCTION)) break;

                    if (otherMatchType == TokenType.OP) {
                        int otherPrecedence = precedenceMap.get(otherMatch.getContent());
                        if (otherPrecedence < precedence ||
                                (associativity == OperatorAssociativity.RIGHT && otherPrecedence == precedence)) {
                            break;
                        }
                    }
                    output.add(tokenStack.pop());
                }
                tokenStack.push(match);
            } else if (matchType == TokenType.OPEN_PARENTH) {
                tokenStack.push(match);
            } else if (matchType == TokenType.CLOSE_PARENTH || matchType == TokenType.COMMA) {
                while (!tokenStack.empty() && tokenStack.peek().getType() != TokenType.OPEN_PARENTH) {
                    output.add(tokenStack.pop());
                }
                if (tokenStack.empty()) return null;
                if (matchType == TokenType.CLOSE_PARENTH) {
                    tokenStack.pop();
{'fraction_non_alphanumeric': 0.10574789169995562, 'fraction_numerical': 0.0004438526409232135, 'mean_word_length': 2.0003328894806924, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4331942', 'n_tokens_mistral': 2464, 'n_tokens_neox': 2319, 'n_words': 551}