<commit_before>﻿module ExcelParser
    open FParsec
    open AST
    open Microsoft.Office.Interop.Excel
    open System.Text.RegularExpressions

    type Workbook = Microsoft.Office.Interop.Excel.Workbook
    type Worksheet = Microsoft.Office.Interop.Excel.Worksheet
    type Name = Microsoft.Office.Interop.Excel.Name
    type UserState = unit
    type Parser<'t> = Parser<'t, UserState>
    
    // custom character classes
    let isWSChar(c: char) : bool =
        isDigit(c) || isLetter(c) || c = '-' || c = ' '

    // Special breakpoint-friendly parser
    let BP (p: Parser<_,_>) stream =
        p stream // set a breakpoint here

    let (<!>) (p: Parser<_,_>) label : Parser<_,_> =
        fun stream ->
//            printfn "%A: Entering %s" stream.Position label
            let reply = p stream
//            printfn "%A: Leaving %s (%A)" stream.Position label reply.Status
            reply

    // Grammar forward references
    let ArgumentList, ArgumentListImpl = createParserForwardedToRef()
    let ExpressionSimple, ExpressionSimpleImpl = createParserForwardedToRef()
    let (ExpressionDecl: Parser<Expression,unit>, ExpressionDeclImpl) = createParserForwardedToRef()

    // Addresses
    // We treat relative and absolute addresses the same-- they behave
    // exactly the same way unless you copy and paste them.
    let AddrR = pstring "R" >>. pint32
    let AddrC = pstring "C" >>. pint32
    let AddrR1C1 = pipe2 AddrR AddrC (fun r c -> Address.NewFromR1C1(r,c,None,None,None))
    let AddrA = many1Satisfy isAsciiUpper
    let AddrAAbs = (pstring "$" <|> pstring "") >>. AddrA
    let Addr1 = pint32
    let Addr1Abs = (pstring "$" <|> pstring "") >>. Addr1
    let AddrA1 = (pipe2 AddrAAbs Addr1Abs (fun col row -> Address.NewFromA1(row,col,None,None,None))) <!> "AddrA1"
    let AnyAddr = ((attempt AddrR1C1) <|> AddrA1) <!> "AnyAddr"

    // Ranges
    let MoreAddrR1C1 = pstring ":" >>. AddrR1C1
    let RangeR1C1 = pipe2 AddrR1C1 MoreAddrR1C1 (fun r1 r2 -> Range(r1, r2))
    let MoreAddrA1 = pstring ":" >>. AddrA1
    let RangeA1 = pipe2 AddrA1 MoreAddrA1 (fun r1 r2 -> Range(r1, r2))
    let RangeAny = ((attempt RangeR1C1) <|> RangeA1) <!> "RangeAny"

    // Worksheet Names
    let WorksheetNameQuoted = (between (pstring "'") (pstring "'") (many1Satisfy ((<>) '\''))) <!> "WorksheetNameQuoted"
    let WorksheetNameUnquoted = (many1Satisfy (fun c -> isWSChar(c))) <!> "WorksheetNameUnquoted"
    let WorksheetName = (WorksheetNameQuoted <|> WorksheetNameUnquoted) <!> "WorksheetName"

    // Workbook Names (this may be too restrictive)
    let Path = many1Satisfy ((<>) '[') <!> "Path"
    let WorkbookName = between (pstring "[") (pstring "]") (many1Satisfy (fun c -> c <> '[' && c <> ']')) <!> "WorkbookName"
    let Workbook = ((Path |>> Some) <|> ((pstring "") >>% None)) .>>. WorkbookName <!> "Workbook"

    // References
    // References consist of the following parts:
    //   An optional workbook name prefix
    //   An optional worksheet name prefix
    //   A single-cell ("Address") or multi-cell address ("Range")
    let RRWQuoted = (between (pstring "'") (pstring "'") (Workbook .>>. WorksheetNameUnquoted)) <!> "RRWQuoted"
    let RangeReferenceWorkbook = (pipe2 (RRWQuoted .>> pstring "!") RangeAny (fun ((wbpath, wbname), wsname) rng -> ReferenceRange(wbpath, Some(wbname), Some(wsname), rng) :> Reference)) <!> "RangeReferenceWorkbook"
    let RangeReferenceWorksheet = pipe2 (WorksheetName .>> pstring "!") RangeAny (fun wsname rng -> ReferenceRange(None, None, Some(wsname), rng) :> Reference)
    let RangeReferenceNoWorksheet = RangeAny |>> (fun rng -> ReferenceRange(None, None, None, rng) :> Reference)
    let RangeReference = (attempt RangeReferenceWorkbook) <|> (attempt RangeReferenceWorksheet) <|> RangeReferenceNoWorksheet

    let ARWQuoted = (between (pstring "'") (pstring "'") (Workbook .>>. WorksheetNameUnquoted)) <!> "ARWQuoted"
    let AddressReferenceWorkbook = (pipe2 (ARWQuoted .>> pstring "!") AnyAddr (fun ((wbpath, wbname), wsname) addr ->  ReferenceAddress(wbpath, Some(wbname), Some(wsname), addr) :> Reference)) <!> "AddressReferenceWorkbook"
    let AddressReferenceWorksheet = pipe2 (WorksheetName .>> pstring "!") AnyAddr (fun wsname addr -> ReferenceAddress(None, None, Some(wsname), addr) :> Reference)
    let AddressReferenceNoWorksheet = AnyAddr |>> (fun addr -> ReferenceAddress(None, None, None, addr) :> Reference)
    let AddressReference = (attempt AddressReferenceWorkbook) <|> (attempt AddressReferenceWorksheet) <|> AddressReferenceNoWorksheet

    let NamedReferenceFirstChar = satisfy (fun c -> c = '_' || isLetter(c))
    let NamedReferenceLastChars = manySatisfy (fun c -> c = '_' || isLetter(c) || isDigit(c))
    let NamedReference = pipe2 NamedReferenceFirstChar NamedReferenceLastChars (fun c s -> ReferenceNamed(None, c.ToString() + s) :> Reference)

    let StringReference = between (pstring "\"") (pstring "\"") (many1Satisfy ((<>) '"')) |>> (fun s -> ReferenceString(None, s) :> Reference)

    let ConstantReference = pint32 |>> (fun r -> ReferenceConstant(None, r) :> Reference)

    let Reference = (attempt RangeReference) <|> (attempt AddressReference) <|> (attempt ConstantReference) <|> (attempt StringReference) <|> NamedReference

    // Functions
    let FunctionName = many1Satisfy (fun c -> isLetter(c))
    let Function = pipe2 (FunctionName .>> pstring "(") (ArgumentList .>> pstring ")") (fun fname arglist -> ReferenceFunction(None, fname, arglist) :> Reference)
    do ArgumentListImpl := sepBy ExpressionDecl (spaces >>. pstring "," .>> spaces)

    // Binary arithmetic operators
    let BinOpChar = spaces >>. satisfy (fun c -> c = '+' || c = '-' || c = '/' || c = '*' || c = '<' || c = '>' || c = '=') .>> spaces
    let BinOp2Char = spaces >>. regex "<=" .>> spaces
    let BinOpLong: Parser<string*Expression,unit> = pipe2 BinOp2Char ExpressionDecl (fun op rhs -> (op, rhs))
    let BinOpShort: Parser<string*Expression,unit> = pipe2 BinOpChar ExpressionDecl (fun op rhs -> (op.ToString(), rhs))
    let BinOp: Parser<string*Expression,unit> = (attempt BinOpLong) <|> BinOpShort

    // Unary operators
    let UnaryOpChar = satisfy (fun c -> c = '+' || c = '-')
<commit_msg>Unary operators may also be padded with whitespace.<commit_after>﻿module ExcelParser
    open FParsec
    open AST
    open Microsoft.Office.Interop.Excel
    open System.Text.RegularExpressions

    type Workbook = Microsoft.Office.Interop.Excel.Workbook
    type Worksheet = Microsoft.Office.Interop.Excel.Worksheet
    type Name = Microsoft.Office.Interop.Excel.Name
    type UserState = unit
    type Parser<'t> = Parser<'t, UserState>
    
    // custom character classes
    let isWSChar(c: char) : bool =
        isDigit(c) || isLetter(c) || c = '-' || c = ' '

    // Special breakpoint-friendly parser
    let BP (p: Parser<_,_>) stream =
        p stream // set a breakpoint here

    let (<!>) (p: Parser<_,_>) label : Parser<_,_> =
        fun stream ->
//            printfn "%A: Entering %s" stream.Position label
            let reply = p stream
//            printfn "%A: Leaving %s (%A)" stream.Position label reply.Status
            reply

    // Grammar forward references
    let ArgumentList, ArgumentListImpl = createParserForwardedToRef()
    let ExpressionSimple, ExpressionSimpleImpl = createParserForwardedToRef()
    let (ExpressionDecl: Parser<Expression,unit>, ExpressionDeclImpl) = createParserForwardedToRef()

    // Addresses
    // We treat relative and absolute addresses the same-- they behave
    // exactly the same way unless you copy and paste them.
    let AddrR = pstring "R" >>. pint32
    let AddrC = pstring "C" >>. pint32
    let AddrR1C1 = pipe2 AddrR AddrC (fun r c -> Address.NewFromR1C1(r,c,None,None,None))
    let AddrA = many1Satisfy isAsciiUpper
    let AddrAAbs = (pstring "$" <|> pstring "") >>. AddrA
    let Addr1 = pint32
    let Addr1Abs = (pstring "$" <|> pstring "") >>. Addr1
    let AddrA1 = (pipe2 AddrAAbs Addr1Abs (fun col row -> Address.NewFromA1(row,col,None,None,None))) <!> "AddrA1"
    let AnyAddr = ((attempt AddrR1C1) <|> AddrA1) <!> "AnyAddr"

    // Ranges
    let MoreAddrR1C1 = pstring ":" >>. AddrR1C1
    let RangeR1C1 = pipe2 AddrR1C1 MoreAddrR1C1 (fun r1 r2 -> Range(r1, r2))
    let MoreAddrA1 = pstring ":" >>. AddrA1
    let RangeA1 = pipe2 AddrA1 MoreAddrA1 (fun r1 r2 -> Range(r1, r2))
    let RangeAny = ((attempt RangeR1C1) <|> RangeA1) <!> "RangeAny"

    // Worksheet Names
    let WorksheetNameQuoted = (between (pstring "'") (pstring "'") (many1Satisfy ((<>) '\''))) <!> "WorksheetNameQuoted"
    let WorksheetNameUnquoted = (many1Satisfy (fun c -> isWSChar(c))) <!> "WorksheetNameUnquoted"
    let WorksheetName = (WorksheetNameQuoted <|> WorksheetNameUnquoted) <!> "WorksheetName"

    // Workbook Names (this may be too restrictive)
    let Path = many1Satisfy ((<>) '[') <!> "Path"
    let WorkbookName = between (pstring "[") (pstring "]") (many1Satisfy (fun c -> c <> '[' && c <> ']')) <!> "WorkbookName"
    let Workbook = ((Path |>> Some) <|> ((pstring "") >>% None)) .>>. WorkbookName <!> "Workbook"

    // References
    // References consist of the following parts:
    //   An optional workbook name prefix
    //   An optional worksheet name prefix
    //   A single-cell ("Address") or multi-cell address ("Range")
    let RRWQuoted = (between (pstring "'") (pstring "'") (Workbook .>>. WorksheetNameUnquoted)) <!> "RRWQuoted"
    let RangeReferenceWorkbook = (pipe2 (RRWQuoted .>> pstring "!") RangeAny (fun ((wbpath, wbname), wsname) rng -> ReferenceRange(wbpath, Some(wbname), Some(wsname), rng) :> Reference)) <!> "RangeReferenceWorkbook"
    let RangeReferenceWorksheet = pipe2 (WorksheetName .>> pstring "!") RangeAny (fun wsname rng -> ReferenceRange(None, None, Some(wsname), rng) :> Reference)
    let RangeReferenceNoWorksheet = RangeAny |>> (fun rng -> ReferenceRange(None, None, None, rng) :> Reference)
    let RangeReference = (attempt RangeReferenceWorkbook) <|> (attempt RangeReferenceWorksheet) <|> RangeReferenceNoWorksheet

    let ARWQuoted = (between (pstring "'") (pstring "'") (Workbook .>>. WorksheetNameUnquoted)) <!> "ARWQuoted"
    let AddressReferenceWorkbook = (pipe2 (ARWQuoted .>> pstring "!") AnyAddr (fun ((wbpath, wbname), wsname) addr ->  ReferenceAddress(wbpath, Some(wbname), Some(wsname), addr) :> Reference)) <!> "AddressReferenceWorkbook"
    let AddressReferenceWorksheet = pipe2 (WorksheetName .>> pstring "!") AnyAddr (fun wsname addr -> ReferenceAddress(None, None, Some(wsname), addr) :> Reference)
    let AddressReferenceNoWorksheet = AnyAddr |>> (fun addr -> ReferenceAddress(None, None, None, addr) :> Reference)
    let AddressReference = (attempt AddressReferenceWorkbook) <|> (attempt AddressReferenceWorksheet) <|> AddressReferenceNoWorksheet

    let NamedReferenceFirstChar = satisfy (fun c -> c = '_' || isLetter(c))
    let NamedReferenceLastChars = manySatisfy (fun c -> c = '_' || isLetter(c) || isDigit(c))
    let NamedReference = pipe2 NamedReferenceFirstChar NamedReferenceLastChars (fun c s -> ReferenceNamed(None, c.ToString() + s) :> Reference)

    let StringReference = between (pstring "\"") (pstring "\"") (many1Satisfy ((<>) '"')) |>> (fun s -> ReferenceString(None, s) :> Reference)

    let ConstantReference = pint32 |>> (fun r -> ReferenceConstant(None, r) :> Reference)

    let Reference = (attempt RangeReference) <|> (attempt AddressReference) <|> (attempt ConstantReference) <|> (attempt StringReference) <|> NamedReference

    // Functions
    let FunctionName = many1Satisfy (fun c -> isLetter(c))
    let Function = pipe2 (FunctionName .>> pstring "(") (ArgumentList .>> pstring ")") (fun fname arglist -> ReferenceFunction(None, fname, arglist) :> Reference)
    do ArgumentListImpl := sepBy ExpressionDecl (spaces >>. pstring "," .>> spaces)

    // Binary arithmetic operators
    let BinOpChar = spaces >>. satisfy (fun c -> c = '+' || c = '-' || c = '/' || c = '*' || c = '<' || c = '>' || c = '=') .>> spaces
    let BinOp2Char = spaces >>. regex "<=" .>> spaces
    let BinOpLong: Parser<string*Expression,unit> = pipe2 BinOp2Char ExpressionDecl (fun op rhs -> (op, rhs))
    let BinOpShort: Parser<string*Expression,unit> = pipe2 BinOpChar ExpressionDecl (fun op rhs -> (op.ToString(), rhs))
    let BinOp: Parser<string*Expression,unit> = (attempt BinOpLong) <|> BinOpShort

    // Unary operators
    let UnaryOpChar = spaces >>. satisfy (fun c -> c = '+' || c = '-') .>> spaces

{'fraction_non_alphanumeric': 0.14899232245681382, 'fraction_numerical': 0.011036468330134356, 'mean_word_length': 3.74573055028463, 'pattern_counts': {'":': 4, '<': 93, '<?xml version=': 0, '>': 271, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1949851', 'n_tokens_mistral': 4135, 'n_tokens_neox': 3938, 'n_words': 1073}