<commit_before>﻿module Grammar
    open FParsec
    open AST
    open System.Text.RegularExpressions

    let DEBUG_MODE = false
    let NO_OPTIMIZATIONS = false

    //#region OPTIMIZED_IMPLEMENTATIONS
    // this is an optimized "choice" parser that disables
    // error message logging when not in debug mode
    let (<||>) (p1: Parser<'a,'u>) (p2: Parser<'a,'u>) : Parser<'a,'u> =
        if NO_OPTIMIZATIONS then
            fun stream ->
                let mutable stateTag = stream.StateTag
                let mutable reply = p1 stream
                if reply.Status = Error && stateTag = stream.StateTag then
                    let error = reply.Error
                    reply <- p2 stream
                    if stateTag = stream.StateTag then
                        reply.Error <- mergeErrors reply.Error error
                reply
        else
            fun stream ->
                let mutable stateTag = stream.StateTag
                let mutable reply = p1 stream
                if reply.Status = Error && stateTag = stream.StateTag then
                    reply <- p2 stream
                reply

    // this is an optimized "attempt" parser that disables
    // error message logging when not in debug mode
    let attempt_opt (p: Parser<'a,'u>) : Parser<'a,'u> =
        if NO_OPTIMIZATIONS then
            fun stream ->
                // state is only declared mutable so it can be passed by ref, it won't be mutated
                let mutable state = CharStreamState(stream) // = stream.State (manually inlined)
                let mutable reply = p stream
                if reply.Status <> Ok then
                    if state.Tag <> stream.StateTag then
                        reply.Error  <- nestedError stream reply.Error
                        reply.Status <- Error // turns FatalErrors into Errors
                        stream.BacktrackTo(&state) // passed by ref as a (slight) optimization
                    elif reply.Status = FatalError then
                        reply.Status <- Error
                reply
        else
            fun stream ->
                // state is only declared mutable so it can be passed by ref, it won't be mutated
                let mutable state = CharStreamState(stream) // = stream.State (manually inlined)
                let mutable reply = p stream
                if reply.Status <> Ok then
                    if state.Tag <> stream.StateTag then
                        reply.Status <- Error // turns FatalErrors into Errors
                        stream.BacktrackTo(&state) // passed by ref as a (slight) optimization
                    elif reply.Status = FatalError then
                        reply.Status <- Error
                reply

    // this is an optimized "choice" parser that disables
    // error message logging when not in debug mode
    let choice_opt (ps: seq<Parser<'a,'u>>)  =
        if NO_OPTIMIZATIONS then
            match ps with
            | :? (Parser<'a,'u>[]) as ps ->
                if ps.Length = 0 then pzero
                else
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable error = NoErrorMessages
                        let mutable reply = ps.[0] stream
                        let mutable i = 1
                        while reply.Status = Error && stateTag = stream.StateTag && i < ps.Length do
                            error <- mergeErrors error reply.Error
                            reply <- ps.[i] stream
                            i <- i + 1
                        if stateTag = stream.StateTag then
                            error <- mergeErrors error reply.Error
                            reply.Error <- error
                        reply
            | :? (Parser<'a,'u> list) as ps ->
                match ps with
                | [] -> pzero
                | hd::tl ->
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable error = NoErrorMessages
                        let mutable hd, tl = hd, tl
                        let mutable reply = hd stream
                        while reply.Status = Error && stateTag = stream.StateTag
                              && (match tl with
                                  | h::t -> hd <- h; tl <- t; true
                                  | _ -> false)
                           do
                            error <- mergeErrors error reply.Error
                            reply <- hd stream
                        if stateTag = stream.StateTag then
                            error <- mergeErrors error reply.Error
                            reply.Error <- error
                        reply
            | _ -> fun stream ->
                       use iter = ps.GetEnumerator()
                       if iter.MoveNext() then
                           let stateTag = stream.StateTag
                           let mutable error = NoErrorMessages
                           let mutable reply = iter.Current stream
                           while reply.Status = Error && stateTag = stream.StateTag && iter.MoveNext() do
                               error <- mergeErrors error reply.Error
                               reply <- iter.Current stream
                           if stateTag = stream.StateTag then
                               error <- mergeErrors error reply.Error
                               reply.Error <- error
                           reply
                       else
                           Reply()
        else
            match ps with
            | :? (Parser<'a,'u>[]) as ps ->
                if ps.Length = 0 then pzero
                else
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable reply = ps.[0] stream
                        let mutable i = 1
                        while reply.Status = Error && stateTag = stream.StateTag && i < ps.Length do
                            reply <- ps.[i] stream
                            i <- i + 1
                        reply
            | :? (Parser<'a,'u> list) as ps ->
                match ps with
                | [] -> pzero
                | hd::tl ->
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable hd, tl = hd, tl
                        let mutable reply = hd stream
                        while reply.Status = Error && stateTag = stream.StateTag
                              && (match tl with
                                  | h::t -> hd <- h; tl <- t; true
                                  | _ -> false)
                           do
                            reply <- hd stream
                        reply
            | _ -> fun stream ->
                       use iter = ps.GetEnumerator()
                       if iter.MoveNext() then
                           let stateTag = stream.StateTag
                           let mutable reply = iter.Current stream
                           while reply.Status = Error && stateTag = stream.StateTag && iter.MoveNext() do<commit_msg>Eliminate unnecessary copies in very hot parsing method.<commit_after>﻿module Grammar
    open FParsec
    open AST
    open System.Text.RegularExpressions

    let DEBUG_MODE = false
    let NO_OPTIMIZATIONS = false

    //#region OPTIMIZED_IMPLEMENTATIONS
    // this is an optimized "choice" parser that disables
    // error message logging when not in debug mode
    let (<||>) (p1: Parser<'a,'u>) (p2: Parser<'a,'u>) : Parser<'a,'u> =
        if NO_OPTIMIZATIONS then
            fun stream ->
                let mutable stateTag = stream.StateTag
                let mutable reply = p1 stream
                if reply.Status = Error && stateTag = stream.StateTag then
                    let error = reply.Error
                    reply <- p2 stream
                    if stateTag = stream.StateTag then
                        reply.Error <- mergeErrors reply.Error error
                reply
        else
            fun stream ->
                let mutable stateTag = stream.StateTag
                let mutable reply = p1 stream
                if reply.Status = Error && stateTag = stream.StateTag then
                    reply <- p2 stream
                reply

    // this is an optimized "attempt" parser that disables
    // error message logging when not in debug mode
    let attempt_opt (p: Parser<'a,'u>) : Parser<'a,'u> =
        if NO_OPTIMIZATIONS then
            fun stream ->
                // state is only declared mutable so it can be passed by ref, it won't be mutated
                let mutable state = CharStreamState(stream) // = stream.State (manually inlined)
                let mutable reply = p stream
                if reply.Status <> Ok then
                    if state.Tag <> stream.StateTag then
                        reply.Error  <- nestedError stream reply.Error
                        reply.Status <- Error // turns FatalErrors into Errors
                        stream.BacktrackTo(&state) // passed by ref as a (slight) optimization
                    elif reply.Status = FatalError then
                        reply.Status <- Error
                reply
        else
            fun stream ->
                // state is only declared mutable so it can be passed by ref, it won't be mutated
                let mutable state = CharStreamState(stream) // = stream.State (manually inlined)
                let mutable reply = p stream
                if reply.Status <> Ok then
                    if state.Tag <> stream.StateTag then
                        reply.Status <- Error // turns FatalErrors into Errors
                        stream.BacktrackTo(&state) // passed by ref as a (slight) optimization
                    elif reply.Status = FatalError then
                        reply.Status <- Error
                reply

    // this is an optimized "choice" parser that disables
    // error message logging when not in debug mode
    let choice_opt (ps: seq<Parser<'a,'u>>)  =
        if NO_OPTIMIZATIONS then
            match ps with
            | :? (Parser<'a,'u>[]) as ps ->
                if ps.Length = 0 then pzero
                else
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable error = NoErrorMessages
                        let mutable reply = ps.[0] stream
                        let mutable i = 1
                        while reply.Status = Error && stateTag = stream.StateTag && i < ps.Length do
                            error <- mergeErrors error reply.Error
                            reply <- ps.[i] stream
                            i <- i + 1
                        if stateTag = stream.StateTag then
                            error <- mergeErrors error reply.Error
                            reply.Error <- error
                        reply
            | :? (Parser<'a,'u> list) as ps ->
                match ps with
                | [] -> pzero
                | hd::tl ->
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable error = NoErrorMessages
                        let mutable hd, tl = hd, tl
                        let mutable reply = hd stream
                        while reply.Status = Error && stateTag = stream.StateTag
                              && (match tl with
                                  | h::t -> hd <- h; tl <- t; true
                                  | _ -> false)
                           do
                            error <- mergeErrors error reply.Error
                            reply <- hd stream
                        if stateTag = stream.StateTag then
                            error <- mergeErrors error reply.Error
                            reply.Error <- error
                        reply
            | _ -> fun stream ->
                       use iter = ps.GetEnumerator()
                       if iter.MoveNext() then
                           let stateTag = stream.StateTag
                           let mutable error = NoErrorMessages
                           let mutable reply = iter.Current stream
                           while reply.Status = Error && stateTag = stream.StateTag && iter.MoveNext() do
                               error <- mergeErrors error reply.Error
                               reply <- iter.Current stream
                           if stateTag = stream.StateTag then
                               error <- mergeErrors error reply.Error
                               reply.Error <- error
                           reply
                       else
                           Reply()
        else
            match ps with
            | :? (Parser<'a,'u>[]) as ps ->
                if ps.Length = 0 then pzero
                else
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable reply = ps.[0] stream
                        let mutable i = 1
                        while reply.Status = Error && stateTag = stream.StateTag && i < ps.Length do
                            reply <- ps.[i] stream
                            i <- i + 1
                        reply
            | :? (Parser<'a,'u> list) as ps ->
                match ps with
                | [] -> pzero
                | hd::tl ->
                    fun stream ->
                        let mutable hd, tl = hd, tl
                        let mutable reply = hd stream
                        while reply.Status = Error && stream.StateTag = stream.StateTag
                              && (match tl with
                                  | h::t -> hd <- h; tl <- t; true
                                  | _ -> false)
                           do
                            reply <- hd stream
                        reply
            | _ -> fun stream ->
                       use iter = ps.GetEnumerator()
                       if iter.MoveNext() then
                           let mutable reply = iter.Current stream
                           while reply.Status = Error && stream.StateTag = stream.StateTag && iter.MoveNext() do
{'fraction_non_alphanumeric': 0.06911191536748329, 'fraction_numerical': 0.0019487750556792874, 'mean_word_length': 0.9619060622610596, 'pattern_counts': {'":': 0, '<': 95, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2593127', 'n_tokens_mistral': 3347, 'n_tokens_neox': 3082, 'n_words': 1088}