<commit_before>﻿using Irony.Parsing;
using System;
using System.Collections.Generic;

namespace XLParser
{
    /// <summary>
    /// Contains the XLParser grammar
    /// </summary>
    [Language("Excel Formulas", "1.1.3", "Grammar for Excel Formulas")]
    public class ExcelFormulaGrammar : Grammar
    {
        public ExcelFormulaGrammar() : base(false)
        {
            #region 1-Terminals

            #region Symbols and operators
            var comma = ToTerm(",");
            var colon = ToTerm(":");
            var semicolon = ToTerm(";");
            var OpenParen = ToTerm("(");
            var CloseParen = ToTerm(")");
            var CloseSquareParen = ToTerm("]");
            var OpenSquareParen = ToTerm("[");
            var exclamationMark = ToTerm("!");
            var CloseCurlyParen = ToTerm("}");
            var OpenCurlyParen = ToTerm("{");

            var mulop = ToTerm("*");
            var plusop = ToTerm("+");
            var divop = ToTerm("/");
            var minop = ToTerm("-");
            var concatop = ToTerm("&");
            var expop = ToTerm("^");
            // Intersect op is a single space, which cannot be parsed normally so we need an ImpliedSymbolTerminal
            // Attention: ImpliedSymbolTerminal seems to break if you assign it a priority, and it's default priority is low
            var intersectop = new ImpliedSymbolTerminal(GrammarNames.TokenIntersect);

            var percentop = ToTerm("%");

            var gtop = ToTerm(">");
            var eqop = ToTerm("=");
            var ltop = ToTerm("<");
            var neqop = ToTerm("<>");
            var gteop = ToTerm(">=");
            var lteop = ToTerm("<=");
            #endregion

            #region Literals
            var BoolToken = new RegexBasedTerminal(GrammarNames.TokenBool, "TRUE|FALSE");
            BoolToken.Priority = TerminalPriority.Bool;

            var NumberToken = new NumberLiteral(GrammarNames.TokenNumber, NumberOptions.None);
            NumberToken.DefaultIntTypes = new TypeCode[] { TypeCode.Int32, TypeCode.Int64, NumberLiteral.TypeCodeBigInt };

            var TextToken = new StringLiteral(GrammarNames.TokenText, "\"", StringOptions.AllowsDoubledQuote | StringOptions.AllowsLineBreak);

            var ErrorToken = new RegexBasedTerminal(GrammarNames.TokenError, "#NULL!|#DIV/0!|#VALUE!|#NAME\\?|#NUM!|#N/A");
            var RefErrorToken = ToTerm("#REF!", GrammarNames.TokenRefError);
            #endregion

            #region Functions

            var UDFToken = new RegexBasedTerminal(GrammarNames.TokenUDF, @"(_xll\.)?[\w\\.]+\(");
            UDFToken.Priority = TerminalPriority.UDF;

            var ExcelRefFunctionToken = new RegexBasedTerminal(GrammarNames.TokenExcelRefFunction, "(INDEX|OFFSET|INDIRECT)\\(");
            ExcelRefFunctionToken.Priority = TerminalPriority.ExcelRefFunction;
            
            var ExcelConditionalRefFunctionToken = new RegexBasedTerminal(GrammarNames.TokenExcelConditionalRefFunction, "(IF|CHOOSE)\\(");
            ExcelConditionalRefFunctionToken.Priority = TerminalPriority.ExcelRefFunction;

            var ExcelFunction = new RegexBasedTerminal(GrammarNames.ExcelFunction, "(" + String.Join("|", excelFunctionList)  +")\\(");
            ExcelFunction.Priority = TerminalPriority.ExcelFunction;

            // Using this instead of Empty allows a more accurate trees
            var EmptyArgumentToken = new ImpliedSymbolTerminal(GrammarNames.TokenEmptyArgument);

            #endregion

            #region References and names

            var VRangeToken = new RegexBasedTerminal(GrammarNames.TokenVRange, "[$]?[A-Z]{1,4}:[$]?[A-Z]{1,4}");
            var HRangeToken = new RegexBasedTerminal(GrammarNames.TokenHRange, "[$]?[1-9][0-9]*:[$]?[1-9][0-9]*");
            
            const string CellTokenRegex = "[$]?[A-Z]{1,4}[$]?[1-9][0-9]*";
            var CellToken = new RegexBasedTerminal(GrammarNames.TokenCell, CellTokenRegex);
            CellToken.Priority = TerminalPriority.CellToken;

            const string NamedRangeRegex = @"[A-Za-z\\_][\w\.]*";
            var NamedRangeToken = new RegexBasedTerminal(GrammarNames.TokenNamedRange, NamedRangeRegex);
            NamedRangeToken.Priority = TerminalPriority.NamedRange;

            // To prevent e.g. "A1A1" being parsed as 2 celltokens
            var NamedRangeCombinationToken = new RegexBasedTerminal(GrammarNames.TokenNamedRangeCombination, "(TRUE|FALSE|" + CellTokenRegex + ")" + NamedRangeRegex);
            NamedRangeCombinationToken.Priority = TerminalPriority.NamedRangeCombination;

            const string mustBeQuotedInSheetName = @"\(\);{}#""=<>&+\-*/\^%, :";
            const string notSheetNameChars = @"'*\[\]\\:/?";
            //const string singleQuotedContent = @"\w !@#$%^&*()\-\+={}|:;<>,\./\?" + "\\\"";
            //const string sheetRegEx = @"(([\w\.]+)|('([" + singleQuotedContent + @"]|'')+'))!";
            const string normalSheetName = "[^" + notSheetNameChars + mustBeQuotedInSheetName + "]+";
            const string quotedSheetName = "([^" + notSheetNameChars +  "]|'')+";
            const string sheetRegEx = "((" + normalSheetName + ")|('" + quotedSheetName + "'))!";

            var SheetToken = new RegexBasedTerminal(GrammarNames.TokenSheet, sheetRegEx);
            SheetToken.Priority = TerminalPriority.SheetToken;

            var multiSheetRegex = String.Format("(({0}:{0})|('{1}:{1}'))!", normalSheetName, quotedSheetName);
            var MultipleSheetsToken = new RegexBasedTerminal(GrammarNames.TokenMultipleSheets, multiSheetRegex);
            MultipleSheetsToken.Priority = TerminalPriority.MultipleSheetsToken;

            var FileToken = new RegexBasedTerminal(GrammarNames.TokenFileNameNumeric, "[0-9]+");<commit_msg>Fixed allowing of : in sheet names<commit_after>﻿using Irony.Parsing;
using System;
using System.Collections.Generic;

namespace XLParser
{
    /// <summary>
    /// Contains the XLParser grammar
    /// </summary>
    [Language("Excel Formulas", "1.1.3", "Grammar for Excel Formulas")]
    public class ExcelFormulaGrammar : Grammar
    {
        public ExcelFormulaGrammar() : base(false)
        {
            #region 1-Terminals

            #region Symbols and operators
            var comma = ToTerm(",");
            var colon = ToTerm(":");
            var semicolon = ToTerm(";");
            var OpenParen = ToTerm("(");
            var CloseParen = ToTerm(")");
            var CloseSquareParen = ToTerm("]");
            var OpenSquareParen = ToTerm("[");
            var exclamationMark = ToTerm("!");
            var CloseCurlyParen = ToTerm("}");
            var OpenCurlyParen = ToTerm("{");

            var mulop = ToTerm("*");
            var plusop = ToTerm("+");
            var divop = ToTerm("/");
            var minop = ToTerm("-");
            var concatop = ToTerm("&");
            var expop = ToTerm("^");
            // Intersect op is a single space, which cannot be parsed normally so we need an ImpliedSymbolTerminal
            // Attention: ImpliedSymbolTerminal seems to break if you assign it a priority, and it's default priority is low
            var intersectop = new ImpliedSymbolTerminal(GrammarNames.TokenIntersect);

            var percentop = ToTerm("%");

            var gtop = ToTerm(">");
            var eqop = ToTerm("=");
            var ltop = ToTerm("<");
            var neqop = ToTerm("<>");
            var gteop = ToTerm(">=");
            var lteop = ToTerm("<=");
            #endregion

            #region Literals
            var BoolToken = new RegexBasedTerminal(GrammarNames.TokenBool, "TRUE|FALSE");
            BoolToken.Priority = TerminalPriority.Bool;

            var NumberToken = new NumberLiteral(GrammarNames.TokenNumber, NumberOptions.None);
            NumberToken.DefaultIntTypes = new TypeCode[] { TypeCode.Int32, TypeCode.Int64, NumberLiteral.TypeCodeBigInt };

            var TextToken = new StringLiteral(GrammarNames.TokenText, "\"", StringOptions.AllowsDoubledQuote | StringOptions.AllowsLineBreak);

            var ErrorToken = new RegexBasedTerminal(GrammarNames.TokenError, "#NULL!|#DIV/0!|#VALUE!|#NAME\\?|#NUM!|#N/A");
            var RefErrorToken = ToTerm("#REF!", GrammarNames.TokenRefError);
            #endregion

            #region Functions

            var UDFToken = new RegexBasedTerminal(GrammarNames.TokenUDF, @"(_xll\.)?[\w\\.]+\(");
            UDFToken.Priority = TerminalPriority.UDF;

            var ExcelRefFunctionToken = new RegexBasedTerminal(GrammarNames.TokenExcelRefFunction, "(INDEX|OFFSET|INDIRECT)\\(");
            ExcelRefFunctionToken.Priority = TerminalPriority.ExcelRefFunction;
            
            var ExcelConditionalRefFunctionToken = new RegexBasedTerminal(GrammarNames.TokenExcelConditionalRefFunction, "(IF|CHOOSE)\\(");
            ExcelConditionalRefFunctionToken.Priority = TerminalPriority.ExcelRefFunction;

            var ExcelFunction = new RegexBasedTerminal(GrammarNames.ExcelFunction, "(" + String.Join("|", excelFunctionList)  +")\\(");
            ExcelFunction.Priority = TerminalPriority.ExcelFunction;

            // Using this instead of Empty allows a more accurate trees
            var EmptyArgumentToken = new ImpliedSymbolTerminal(GrammarNames.TokenEmptyArgument);

            #endregion

            #region References and names

            var VRangeToken = new RegexBasedTerminal(GrammarNames.TokenVRange, "[$]?[A-Z]{1,4}:[$]?[A-Z]{1,4}");
            var HRangeToken = new RegexBasedTerminal(GrammarNames.TokenHRange, "[$]?[1-9][0-9]*:[$]?[1-9][0-9]*");
            
            const string CellTokenRegex = "[$]?[A-Z]{1,4}[$]?[1-9][0-9]*";
            var CellToken = new RegexBasedTerminal(GrammarNames.TokenCell, CellTokenRegex);
            CellToken.Priority = TerminalPriority.CellToken;

            const string NamedRangeRegex = @"[A-Za-z\\_][\w\.]*";
            var NamedRangeToken = new RegexBasedTerminal(GrammarNames.TokenNamedRange, NamedRangeRegex);
            NamedRangeToken.Priority = TerminalPriority.NamedRange;

            // To prevent e.g. "A1A1" being parsed as 2 celltokens
            var NamedRangeCombinationToken = new RegexBasedTerminal(GrammarNames.TokenNamedRangeCombination, "(TRUE|FALSE|" + CellTokenRegex + ")" + NamedRangeRegex);
            NamedRangeCombinationToken.Priority = TerminalPriority.NamedRangeCombination;

            const string mustBeQuotedInSheetName = @"\(\);{}#""=<>&+\-*/\^%, ";
            const string notSheetNameChars = @"'*\[\]\\:/?";
            //const string singleQuotedContent = @"\w !@#$%^&*()\-\+={}|:;<>,\./\?" + "\\\"";
            //const string sheetRegEx = @"(([\w\.]+)|('([" + singleQuotedContent + @"]|'')+'))!";
            const string normalSheetName = "[^" + notSheetNameChars + mustBeQuotedInSheetName + "]+";
            const string quotedSheetName = "([^" + notSheetNameChars +  "]|'')+";
            const string sheetRegEx = "((" + normalSheetName + ")|('" + quotedSheetName + "'))!";

            var SheetToken = new RegexBasedTerminal(GrammarNames.TokenSheet, sheetRegEx);
            SheetToken.Priority = TerminalPriority.SheetToken;

            var multiSheetRegex = String.Format("(({0}:{0})|('{1}:{1}'))!", normalSheetName, quotedSheetName);
            var MultipleSheetsToken = new RegexBasedTerminal(GrammarNames.TokenMultipleSheets, multiSheetRegex);
            MultipleSheetsToken.Priority = TerminalPriority.MultipleSheetsToken;

            var FileToken = new RegexBasedTerminal(GrammarNames.TokenFileNameNumeric, "[0-9]+");
{'fraction_non_alphanumeric': 0.1384429483859831, 'fraction_numerical': 0.006214396685655101, 'mean_word_length': 2.679580819307717, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2630459', 'n_tokens_mistral': 3440, 'n_tokens_neox': 3293, 'n_words': 666}