<commit_before>// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.

namespace FSharp.Test.Utilities

open FSharp.Compiler.SourceCodeServices
open FSharp.Test.Utilities
open FSharp.Test.Utilities.Assert
open FSharp.Test.Utilities.Utilities
open Microsoft.CodeAnalysis
open Microsoft.CodeAnalysis.CSharp
open NUnit.Framework
open System
open System.Collections.Immutable
open System.IO

module rec Compiler =

    type TestType =
        | Text of string
        | Path of string
        | Baseline of (string * string)

    type CompilationUnit =
        | FS  of FSharpCompilationSource
        | CS  of CSharpCompilationSource
        | IL  of ILCompilationSource

    type FSharpCompilationSource =
        { Source:         TestType
          Options:        string list
          OutputType:     CompileOutput
          SourceKind:     SourceKind
          Name:           string option
          IgnoreWarnings: bool
          References:     CompilationUnit list }

    type CSharpCompilationSource =
        { Source:          TestType
          LangVersion:     CSharpLanguageVersion
          TargetFramework: TargetFramework
          Name:            string option
          References:      CompilationUnit list }

    type ILCompilationSource =
        { Source:     TestType
          References: CompilationUnit list  }

    type ErrorType = Error of int | Warning of int

    type Line = Line of int
    type Col = Col of int

    type Range =
          { StartLine:   int
            StartColumn: int
            EndLine:     int
            EndColumn:   int }

    type ErrorInfo =
        { Error:   ErrorType
          Range:   Range
          Message: string }

    type ExecutionOutput =
        { ExitCode: int
          StdOut:   string
          StdErr:   string }

    type Output =
        { OutputPath:   string option
          Dependencies: string list
          Adjust:       int
          Errors:       ErrorInfo list
          Warnings:     ErrorInfo list
          Output:       ExecutionOutput option }

    type TestResult =
        | Success of Output
        | Failure of Output

    let private defaultOptions : string list = []

    // Not very safe version of reading stuff from file, but we want to fail fast for now if anything goes wrong.
    let private getSource (src: TestType) : string =
        match src with
        | Text t -> t
        | Path p -> System.IO.File.ReadAllText p
        | Baseline (d, f) -> System.IO.File.ReadAllText (System.IO.Path.Combine(d, f))

    let private fsFromString (source: string) (kind: SourceKind) : FSharpCompilationSource =
        match source with
        | null -> failwith "Source cannot be null"
        | _ ->
            { Source         = Text source
              Options        = defaultOptions
              OutputType     = Library
              SourceKind     = kind
              Name           = None
              IgnoreWarnings = false
              References     = [] }

    let private csFromString (source: string) : CSharpCompilationSource =
        match source with
        | null -> failwith "Source cannot be null"
        | _ ->
            { Source          = Text source
              LangVersion     = CSharpLanguageVersion.CSharp8
              TargetFramework = TargetFramework.NetCoreApp30
              Name            = None
              References      = [] }

    let private fromFSharpErrorInfo (errors: FSharpErrorInfo[]) : (ErrorInfo list * ErrorInfo list) =
        let toErrorInfo (e: FSharpErrorInfo) : ErrorInfo =
            let errorNumber = e.ErrorNumber
            let severity = e.Severity

            let error = if severity = FSharpErrorSeverity.Warning then Warning errorNumber else Error errorNumber

            { Error   = error
              Range   =
                  { StartLine   = e.StartLineAlternate
                    StartColumn = e.StartColumn
                    EndLine     = e.EndLineAlternate
                    EndColumn   = e.EndColumn }
              Message = e.Message }

        errors
        |> List.ofArray
        |> List.distinctBy (fun e -> e.Severity, e.ErrorNumber, e.StartLineAlternate, e.StartColumn, e.EndLineAlternate, e.EndColumn, e.Message)
        |> List.map toErrorInfo
        |> List.partition  (fun e -> match e.Error with Error _ -> true | _ -> false)

    let private adjustRange (range: Range) (adjust: int) : Range =
        { range with
                StartLine   = range.StartLine   - adjust
                StartColumn = range.StartColumn + 1
                EndLine     = range.EndLine     - adjust
                EndColumn   = range.EndColumn   + 1 }

    let Fsx (source: string) : CompilationUnit =
        fsFromString source SourceKind.Fsx |> FS

    let FSharp (source: string) : CompilationUnit =
        fsFromString source SourceKind.Fs |> FS

    let baseline (dir: string, file: string) : CompilationUnit =
        match (dir, file) with
        | dir, _ when String.IsNullOrWhiteSpace dir -> failwith "Baseline tests directory cannot be null or empty."
        | _, file when String.IsNullOrWhiteSpace file -> failwith "Baseline source file name cannot be null or empty."
        | _ ->
            { Source         = Baseline (dir, file)
              Options        = defaultOptions
              OutputType     = Library
              SourceKind     = Fs
              Name           = None
              IgnoreWarnings = false
              References     = [] } |> FS

    let CSharp (source: string) : CompilationUnit =
        csFromString source |> CS

    let withName (name: string) (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS src -> FS { src with Name = Some name }
        | CS src -> CS { src with Name = Some name }
        | IL _ -> failwith "IL Compilation cannot be named."

    let withReferences (references: CompilationUnit list) (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with References = fs.References @ references }
        | CS cs -> CS { cs with References = cs.References @ references }
        | IL _ -> failwith "References are not supported in IL"

    let withOptions (options: string list) (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with Options = options }
        | _ -> failwith "withOptions is only supported n F#"

    let withPreview (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with Options = [ "--langversion:preview" ] }
        | _ -> failwith "withPreview is only supported in F#"

    let asLibrary (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with OutputType = CompileOutput.Library }
        | _ -> failwith "TODO: Implement where applicable."

    let asExe (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with OutputType = CompileOutput.Exe }
        | _ -> failwith "TODO: Implement where applicable."

    let ignoreWarnings (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with IgnoreWarnings = true }
        | _ -> failwith "TODO: Implement ignorewarnings for the rest."

    let rec private asMetadataReference reference =
        match reference with
        | CompilationReference (cmpl, _) ->
            let result = compileFSharpCompilation cmpl false
            match result with
            | Failure f ->
                let message = sprintf "Operation failed (expected to succeed).\n All errors:\n%A" (f.Errors @ f.Warnings)
                failwith message
            | Success s ->
                match s.OutputPath with
                    | None -> failwith "Operation didn't produce any output!"
                    | Some p -> p |> MetadataReference.CreateFromFile
        | _ -> failwith "Conversion isn't possible"<commit_msg>Oops - I should have reviewed the original pr better ... sorry (#9881)<commit_after>// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.

namespace FSharp.Test.Utilities

open FSharp.Compiler.SourceCodeServices
open FSharp.Test.Utilities
open FSharp.Test.Utilities.Assert
open FSharp.Test.Utilities.Utilities
open Microsoft.CodeAnalysis
open Microsoft.CodeAnalysis.CSharp
open NUnit.Framework
open System
open System.Collections.Immutable
open System.IO

module rec Compiler =

    type TestType =
        | Text of string
        | Path of string
        | Baseline of (string * string)

    type CompilationUnit =
        | FS  of FSharpCompilationSource
        | CS  of CSharpCompilationSource
        | IL  of ILCompilationSource

    type FSharpCompilationSource =
        { Source:         TestType
          Options:        string list
          OutputType:     CompileOutput
          SourceKind:     SourceKind
          Name:           string option
          IgnoreWarnings: bool
          References:     CompilationUnit list }

    type CSharpCompilationSource =
        { Source:          TestType
          LangVersion:     CSharpLanguageVersion
          TargetFramework: TargetFramework
          Name:            string option
          References:      CompilationUnit list }

    type ILCompilationSource =
        { Source:     TestType
          References: CompilationUnit list  }

    type ErrorType = Error of int | Warning of int

    type Line = Line of int
    type Col = Col of int

    type Range =
          { StartLine:   int
            StartColumn: int
            EndLine:     int
            EndColumn:   int }

    type ErrorInfo =
        { Error:   ErrorType
          Range:   Range
          Message: string }

    type ExecutionOutput =
        { ExitCode: int
          StdOut:   string
          StdErr:   string }

    type Output =
        { OutputPath:   string option
          Dependencies: string list
          Adjust:       int
          Errors:       ErrorInfo list
          Warnings:     ErrorInfo list
          Output:       ExecutionOutput option }

    type TestResult =
        | Success of Output
        | Failure of Output

    let private defaultOptions : string list = []

    // Not very safe version of reading stuff from file, but we want to fail fast for now if anything goes wrong.
    let private getSource (src: TestType) : string =
        match src with
        | Text t -> t
        | Path p -> System.IO.File.ReadAllText p
        | Baseline (d, f) -> System.IO.File.ReadAllText (System.IO.Path.Combine(d, f))

    let private fsFromString (source: string) (kind: SourceKind) : FSharpCompilationSource =
        match source with
        | null -> failwith "Source cannot be null"
        | _ ->
            { Source         = Text source
              Options        = defaultOptions
              OutputType     = Library
              SourceKind     = kind
              Name           = None
              IgnoreWarnings = false
              References     = [] }

    let private csFromString (source: string) : CSharpCompilationSource =
        match source with
        | null -> failwith "Source cannot be null"
        | _ ->
            { Source          = Text source
              LangVersion     = CSharpLanguageVersion.CSharp8
              TargetFramework = TargetFramework.NetCoreApp30
              Name            = None
              References      = [] }

    let private fromFSharpErrorInfo (errors: FSharpErrorInfo[]) : (ErrorInfo list * ErrorInfo list) =
        let toErrorInfo (e: FSharpErrorInfo) : ErrorInfo =
            let errorNumber = e.ErrorNumber
            let severity = e.Severity

            let error = if severity = FSharpErrorSeverity.Warning then Warning errorNumber else Error errorNumber

            { Error   = error
              Range   =
                  { StartLine   = e.StartLineAlternate
                    StartColumn = e.StartColumn
                    EndLine     = e.EndLineAlternate
                    EndColumn   = e.EndColumn }
              Message = e.Message }

        errors
        |> List.ofArray
        |> List.distinctBy (fun e -> e.Severity, e.ErrorNumber, e.StartLineAlternate, e.StartColumn, e.EndLineAlternate, e.EndColumn, e.Message)
        |> List.map toErrorInfo
        |> List.partition  (fun e -> match e.Error with Error _ -> true | _ -> false)

    let private adjustRange (range: Range) (adjust: int) : Range =
        { range with
                StartLine   = range.StartLine   - adjust
                StartColumn = range.StartColumn + 1
                EndLine     = range.EndLine     - adjust
                EndColumn   = range.EndColumn   + 1 }

    let Fsx (source: string) : CompilationUnit =
        fsFromString source SourceKind.Fsx |> FS

    let FSharp (source: string) : CompilationUnit =
        fsFromString source SourceKind.Fs |> FS

    let baseline (dir: string, file: string) : CompilationUnit =
        match (dir, file) with
        | dir, _ when String.IsNullOrWhiteSpace dir -> failwith "Baseline tests directory cannot be null or empty."
        | _, file when String.IsNullOrWhiteSpace file -> failwith "Baseline source file name cannot be null or empty."
        | _ ->
            { Source         = Baseline (dir, file)
              Options        = defaultOptions
              OutputType     = Library
              SourceKind     = Fs
              Name           = None
              IgnoreWarnings = false
              References     = [] } |> FS

    let CSharp (source: string) : CompilationUnit =
        csFromString source |> CS

    let withName (name: string) (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS src -> FS { src with Name = Some name }
        | CS src -> CS { src with Name = Some name }
        | IL _ -> failwith "IL Compilation cannot be named."

    let withReferences (references: CompilationUnit list) (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with References = fs.References @ references }
        | CS cs -> CS { cs with References = cs.References @ references }
        | IL _ -> failwith "References are not supported in IL"

    let withOptions (options: string list) (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with Options = fs.Options @ options }
        | _ -> failwith "withOptions is only supported n F#"

    let withPreview (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with Options = fs.Options @ [ "--langversion:preview" ] }
        | _ -> failwith "withPreview is only supported in F#"

    let asLibrary (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with OutputType = CompileOutput.Library }
        | _ -> failwith "TODO: Implement where applicable."

    let asExe (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with OutputType = CompileOutput.Exe }
        | _ -> failwith "TODO: Implement where applicable."

    let ignoreWarnings (cUnit: CompilationUnit) : CompilationUnit =
        match cUnit with
        | FS fs -> FS { fs with IgnoreWarnings = true }
        | _ -> failwith "TODO: Implement ignorewarnings for the rest."

    let rec private asMetadataReference reference =
        match reference with
        | CompilationReference (cmpl, _) ->
            let result = compileFSharpCompilation cmpl false
            match result with
            | Failure f ->
                let message = sprintf "Operation failed (expected to succeed).\n All errors:\n%A" (f.Errors @ f.Warnings)
                failwith message
            | Success s ->
                match s.OutputPath with
                    | None -> failwith "Operation didn't produce any output!"
                    | Some p -> p |> MetadataReference.CreateFromFile
        | _ -> failwith "Conversion isn't possible"
{'fraction_non_alphanumeric': 0.07017652524001239, 'fraction_numerical': 0.0008671415298854134, 'mean_word_length': 1.8251968503937008, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6188215', 'n_tokens_mistral': 4254, 'n_tokens_neox': 3995, 'n_words': 1381}