<commit_before>// Copyright (c) 2018 <NAME>
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT

namespace Farkle

open Farkle.Builder
open Farkle.Grammar
open Farkle.IO
open Farkle.Parser
open System
open System.IO
open System.Reflection
open System.Runtime.CompilerServices
open System.Runtime.InteropServices
open System.Text

/// A type signifying an error during the parsing process.
[<RequireQualifiedAccess>]
type FarkleError =
    /// There was a parsing error.
    | ParseError of ParserError
    /// There had been an error while building the grammar .
    | BuildError of BuildError
    override x.ToString() =
        match x with
        | ParseError x -> x.ToString()
        | BuildError x ->
            let sb = StringBuilder("Error while building the grammar:")
            let msg = x.ToString()
            // We will add a line break before the error message if it is multiline.
            if msg.Contains(Environment.NewLine) then
                Environment.NewLine
            else
                " "
            |> sb.Append |> ignore
            sb.Append(msg) |> ignore
            sb.ToString()

/// <summary>A reusable parser and post-processor,
/// created for a specific grammar, and returning
/// a specific type of object that best describes
/// an expression of the language of this grammar.</summary>
/// <remarks><para>Its parsing methods return an F# result
/// type containing either the post-processed return
/// type, or a type describing what did wrong and where.</para>
/// <para>Exceptions during post-processing (apart from
/// <see cref="ParserApplicationException"/>) are thrown
/// after being wrapped in a <see cref="PostProcessorException"/>.</para></remarks>
[<NoComparison; ReferenceEquality>]
type RuntimeFarkle<'TResult> = internal {
    Grammar: Result<Grammar,BuildError>
    PostProcessor: PostProcessor<'TResult>
    TokenizerFactory: TokenizerFactory
}
with
    static member internal CreateMaybe postProcessor grammarMaybe =
        {
            Grammar = grammarMaybe
            PostProcessor = postProcessor
            TokenizerFactory = TokenizerFactory.Default
        }
    /// <summary>Creates a <see cref="RuntimeFarkle{TResult}"/> from the given
    /// <see cref="Grammar"/> and <see cref="PostProcessor{TResult}"/>.</summary>
    static member Create(grammar, postProcessor) =
        grammar
        |> Ok
        |> RuntimeFarkle<_>.CreateMaybe postProcessor

    /// <summary>Creates a <see cref="RuntimeFarkle{TResult}"/> from the given
    /// EGT file and <see cref="PostProcessor{TResult}"/>.</summary>
    static member Create(fileName, postProcessor) =
        fileName
        |> EGT.ofFile
        |> Ok
        |> RuntimeFarkle<_>.CreateMaybe postProcessor

    /// <summary>Creates a <see cref="RuntimeFarkle{TResult}"/> from the given
    /// EGT file encoded in Base64 and <see cref="PostProcessor{TResult}"/>.</summary>
    static member CreateFromBase64String(str, postProcessor) =
        str
        |> EGT.ofBase64String
        |> Ok
        |> RuntimeFarkle<_>.CreateMaybe postProcessor

    /// Returns whether building was successful.
    /// If loaded from an EGT file, it will always return true.
    member this.IsBuildSuccessful =
        match this.Grammar with
        | Ok _ -> true
        | Error _ -> false

    /// Returns a domain-specific error object that
    /// describes what had gone wrong while building,
    /// or raises an exception if building had been successful.
    member this.GetBuildError() =
        match this.Grammar with
        | Ok _ -> invalidOp "Building the grammar did not fail."
        | Error msg -> msg

    /// Returns a user-friendly error message that
    /// describes what had gone wrong while building,
    /// or an empty string if building had been successful.
    member this.GetBuildErrorMessage() =
        match this.Grammar with
        | Ok _ -> String.Empty
        | Error msg -> string msg

    /// <summary>Gets the <see cref="Farkle.Grammar.Grammar"/>
    /// behind the <see cref="RuntimeFarkle{TResult}"/>.</summary>
    /// <exception cref="InvalidOperationException">Building the grammar
    /// had failed. The exception's message contains further details.</exception>
    member this.GetGrammar() =
        match this.Grammar with
        | Ok grammar -> grammar
        | Error msg -> msg |> string |> invalidOp
    /// <summary>Parses and post-processes a <see cref="Farkle.IO.CharStream"/>.</summary>
    /// <param name="charStream">The character stream to parse.</param>
    /// <param name="tokenizer">An optional custom <see cref="Tokenizer"/>.</param>
    /// <returns>An F# result type containing either the
    /// post-processed return type, or a type describing
    /// what did wrong and where.</returns>
    member this.Parse(input: CharStream) =
        let mkError msg = msg |> FarkleError.ParseError |> Error
        match this.Grammar with
        | Ok grammar ->
            let tokenizer = this.TokenizerFactory.CreateTokenizer grammar
            try
                LALRParser.parse grammar this.PostProcessor tokenizer input |> Ok
            with
            | :? ParserException as e -> mkError e.Error
            | :? ParserApplicationException as e ->
                ParserError(input.CurrentPosition, ParseErrorType.UserError e.Message)
                |> mkError
        | Error x -> Error <| FarkleError.BuildError x
    /// <summary>Changes the <see cref="TokenizerFactory"/> of this runtime Farkle.</summary>
    /// <param name="tokenizerFactory">The new tokenizer factory</param>
    /// <returns>A new runtime Farkle that will parse text with the tokenizer
    /// <paramref name="tokenizerFactory"/> will create.</returns>
    /// <remarks>A new tokenizer will be created with each parse operation.</remarks>
    member this.ChangeTokenizer tokenizerFactory =
        {this with TokenizerFactory = tokenizerFactory}
    /// <summary>Changes the <see cref="Tokenizer"/> type this runtime Farkle will use.</summary>
    /// <typeparam name="TTokenizer">The type of the new tokenizer. It must have
    /// a public constructor that accepts a <see cref="Grammar"/></typeparam>
    /// <returns>A new runtime Farkle that will parse text with
    /// tokenizers of type <typeparamref name="TTokenizer"/>.</returns>
    /// <exception cref="MissingMethodEsception"><typeparamref name="TTokenizer"/>
    /// does not have a public constructor that accepts a <see cref="Grammar"/>.</exception>
    /// <remarks>A new tokenizer will be created with each parse operation.</remarks>
    member this.ChangeTokenizer<'TTokenizer when 'TTokenizer :> Tokenizer>() =
        this.ChangeTokenizer(TokenizerFactoryOfType typeof<'TTokenizer>)
    interface IGrammarProvider with
        member this.IsBuildSuccessful = this.IsBuildSuccessful
        member this.GetGrammar() = this.GetGrammar()
        member this.GetBuildErrorMessage() = this.GetBuildErrorMessage()

/// Functions to create and use `RuntimeFarkle`s.
module RuntimeFarkle =

    /// Changes the post-processor of a `RuntimeFarkle`.
    [<CompiledName("ChangePostProcessor")>]
    let changePostProcessor pp rf = {
        Grammar = rf.Grammar
        PostProcessor = pp
        TokenizerFactory = rf.TokenizerFactory
    }

    /// Changes the tokenizer that will be used by this runtime Farkle.
    /// This function accepts a function that takes a `Grammar` and returns a `Tokenizer`.
    let changeTokenizer fTokenizer (rf: RuntimeFarkle<'TResult>) =
        rf.ChangeTokenizer
            {new TokenizerFactory() with member _.CreateTokenizer g = fTokenizer g}

    /// Changes the post-processor of a runtime Farkle to a
    /// dummy one suitable for syntax-checking, not parsing.
    let syntaxCheck rf = changePostProcessor PostProcessors.syntaxCheck rf

    /// Creates a `RuntimeFarkle` from the given grammar and post-processor.
    [<CompiledName("Create")>]
    let create postProcessor (grammar: Grammar) =
        RuntimeFarkle<_>.Create(grammar, postProcessor)
<commit_msg>Use flexible types in RuntimeFarkle.changeTokenizer.<commit_after>// Copyright (c) 2018 <NAME>
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT

namespace Farkle

open Farkle.Builder
open Farkle.Grammar
open Farkle.IO
open Farkle.Parser
open System
open System.IO
open System.Reflection
open System.Runtime.CompilerServices
open System.Runtime.InteropServices
open System.Text

/// A type signifying an error during the parsing process.
[<RequireQualifiedAccess>]
type FarkleError =
    /// There was a parsing error.
    | ParseError of ParserError
    /// There had been an error while building the grammar .
    | BuildError of BuildError
    override x.ToString() =
        match x with
        | ParseError x -> x.ToString()
        | BuildError x ->
            let sb = StringBuilder("Error while building the grammar:")
            let msg = x.ToString()
            // We will add a line break before the error message if it is multiline.
            if msg.Contains(Environment.NewLine) then
                Environment.NewLine
            else
                " "
            |> sb.Append |> ignore
            sb.Append(msg) |> ignore
            sb.ToString()

/// <summary>A reusable parser and post-processor,
/// created for a specific grammar, and returning
/// a specific type of object that best describes
/// an expression of the language of this grammar.</summary>
/// <remarks><para>Its parsing methods return an F# result
/// type containing either the post-processed return
/// type, or a type describing what did wrong and where.</para>
/// <para>Exceptions during post-processing (apart from
/// <see cref="ParserApplicationException"/>) are thrown
/// after being wrapped in a <see cref="PostProcessorException"/>.</para></remarks>
[<NoComparison; ReferenceEquality>]
type RuntimeFarkle<'TResult> = internal {
    Grammar: Result<Grammar,BuildError>
    PostProcessor: PostProcessor<'TResult>
    TokenizerFactory: TokenizerFactory
}
with
    static member internal CreateMaybe postProcessor grammarMaybe =
        {
            Grammar = grammarMaybe
            PostProcessor = postProcessor
            TokenizerFactory = TokenizerFactory.Default
        }
    /// <summary>Creates a <see cref="RuntimeFarkle{TResult}"/> from the given
    /// <see cref="Grammar"/> and <see cref="PostProcessor{TResult}"/>.</summary>
    static member Create(grammar, postProcessor) =
        grammar
        |> Ok
        |> RuntimeFarkle<_>.CreateMaybe postProcessor

    /// <summary>Creates a <see cref="RuntimeFarkle{TResult}"/> from the given
    /// EGT file and <see cref="PostProcessor{TResult}"/>.</summary>
    static member Create(fileName, postProcessor) =
        fileName
        |> EGT.ofFile
        |> Ok
        |> RuntimeFarkle<_>.CreateMaybe postProcessor

    /// <summary>Creates a <see cref="RuntimeFarkle{TResult}"/> from the given
    /// EGT file encoded in Base64 and <see cref="PostProcessor{TResult}"/>.</summary>
    static member CreateFromBase64String(str, postProcessor) =
        str
        |> EGT.ofBase64String
        |> Ok
        |> RuntimeFarkle<_>.CreateMaybe postProcessor

    /// Returns whether building was successful.
    /// If loaded from an EGT file, it will always return true.
    member this.IsBuildSuccessful =
        match this.Grammar with
        | Ok _ -> true
        | Error _ -> false

    /// Returns a domain-specific error object that
    /// describes what had gone wrong while building,
    /// or raises an exception if building had been successful.
    member this.GetBuildError() =
        match this.Grammar with
        | Ok _ -> invalidOp "Building the grammar did not fail."
        | Error msg -> msg

    /// Returns a user-friendly error message that
    /// describes what had gone wrong while building,
    /// or an empty string if building had been successful.
    member this.GetBuildErrorMessage() =
        match this.Grammar with
        | Ok _ -> String.Empty
        | Error msg -> string msg

    /// <summary>Gets the <see cref="Farkle.Grammar.Grammar"/>
    /// behind the <see cref="RuntimeFarkle{TResult}"/>.</summary>
    /// <exception cref="InvalidOperationException">Building the grammar
    /// had failed. The exception's message contains further details.</exception>
    member this.GetGrammar() =
        match this.Grammar with
        | Ok grammar -> grammar
        | Error msg -> msg |> string |> invalidOp
    /// <summary>Parses and post-processes a <see cref="Farkle.IO.CharStream"/>.</summary>
    /// <param name="charStream">The character stream to parse.</param>
    /// <param name="tokenizer">An optional custom <see cref="Tokenizer"/>.</param>
    /// <returns>An F# result type containing either the
    /// post-processed return type, or a type describing
    /// what did wrong and where.</returns>
    member this.Parse(input: CharStream) =
        let mkError msg = msg |> FarkleError.ParseError |> Error
        match this.Grammar with
        | Ok grammar ->
            let tokenizer = this.TokenizerFactory.CreateTokenizer grammar
            try
                LALRParser.parse grammar this.PostProcessor tokenizer input |> Ok
            with
            | :? ParserException as e -> mkError e.Error
            | :? ParserApplicationException as e ->
                ParserError(input.CurrentPosition, ParseErrorType.UserError e.Message)
                |> mkError
        | Error x -> Error <| FarkleError.BuildError x
    /// <summary>Changes the <see cref="TokenizerFactory"/> of this runtime Farkle.</summary>
    /// <param name="tokenizerFactory">The new tokenizer factory</param>
    /// <returns>A new runtime Farkle that will parse text with the tokenizer
    /// <paramref name="tokenizerFactory"/> will create.</returns>
    /// <remarks>A new tokenizer will be created with each parse operation.</remarks>
    member this.ChangeTokenizer tokenizerFactory =
        {this with TokenizerFactory = tokenizerFactory}
    /// <summary>Changes the <see cref="Tokenizer"/> type this runtime Farkle will use.</summary>
    /// <typeparam name="TTokenizer">The type of the new tokenizer. It must have
    /// a public constructor that accepts a <see cref="Grammar"/></typeparam>
    /// <returns>A new runtime Farkle that will parse text with
    /// tokenizers of type <typeparamref name="TTokenizer"/>.</returns>
    /// <exception cref="MissingMethodEsception"><typeparamref name="TTokenizer"/>
    /// does not have a public constructor that accepts a <see cref="Grammar"/>.</exception>
    /// <remarks>A new tokenizer will be created with each parse operation.</remarks>
    member this.ChangeTokenizer<'TTokenizer when 'TTokenizer :> Tokenizer>() =
        this.ChangeTokenizer(TokenizerFactoryOfType typeof<'TTokenizer>)
    interface IGrammarProvider with
        member this.IsBuildSuccessful = this.IsBuildSuccessful
        member this.GetGrammar() = this.GetGrammar()
        member this.GetBuildErrorMessage() = this.GetBuildErrorMessage()

/// Functions to create and use `RuntimeFarkle`s.
module RuntimeFarkle =

    /// Changes the post-processor of a `RuntimeFarkle`.
    [<CompiledName("ChangePostProcessor")>]
    let changePostProcessor pp rf = {
        Grammar = rf.Grammar
        PostProcessor = pp
        TokenizerFactory = rf.TokenizerFactory
    }

    /// Changes the tokenizer that will be used by this runtime Farkle.
    /// This function accepts a function that takes a `Grammar` and returns a `Tokenizer`.
    let changeTokenizer (fTokenizer: _ -> #Tokenizer) (rf: RuntimeFarkle<'TResult>) =
        rf.ChangeTokenizer
            {new TokenizerFactory() with member _.CreateTokenizer g = fTokenizer g :> _}

    /// Changes the post-processor of a runtime Farkle to a
    /// dummy one suitable for syntax-checking, not parsing.
    let syntaxCheck rf = changePostProcessor PostProcessors.syntaxCheck rf

    /// Creates a `RuntimeFarkle` from the given grammar and post-processor.
    [<CompiledName("Create")>]
    let create postProcessor (grammar: Grammar) =
        RuntimeFarkle<_>.Create(grammar, postProcessor)

{'fraction_non_alphanumeric': 0.10450428245732947, 'fraction_numerical': 0.001232361821430772, 'mean_word_length': 2.7823351200186437, 'pattern_counts': {'":': 0, '<': 163, '<?xml version=': 0, '>': 227, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '213538', 'n_tokens_mistral': 4369, 'n_tokens_neox': 4266, 'n_words': 1423}