<commit_before>module Fable.Main

open System
open System.IO
open Microsoft.FSharp.Compiler
open Microsoft.FSharp.Compiler.Ast
open Microsoft.FSharp.Compiler.SourceCodeServices
open Newtonsoft.Json
open Fable

let readOptions argv =
    let def opts key defArg f =
        defaultArg (Map.tryFind key opts |> Option.map f) defArg
    let rec readOpts opts = function
        | [] -> opts
        | (opt: string)::rest ->
            let k = opt.Substring(2)
            match Map.tryFind k opts with
            | None -> Map.add k rest.Head opts
            | Some v -> Map.add k (v + "," + rest.Head) opts
            |> readOpts <| rest.Tail
    let opts = readOpts Map.empty<_,_> (List.ofArray argv)
    {
        code = def opts "code" null id
        outDir = def opts "outDir" "." id
        lib = def opts "lib" "." id
        projFile = def opts "projFile" null id
        watch = def opts "watch" false bool.Parse
        symbols = def opts "symbols" [||] (fun x -> x.Split(','))
    }

let parseFSharpProject (com: ICompiler) (checker: FSharpChecker) (projCode: string option) =
    let checkProjectResults =
        let projCode =
            match projCode with
            | None -> File.ReadAllText com.Options.projFile
            | Some projCode ->
                File.WriteAllText(com.Options.projFile, projCode)
                projCode
        match (Path.GetExtension com.Options.projFile).ToLower() with
        | ".fsx" ->
            let otherFlags =
                com.Options.symbols |> Array.map (sprintf "--define:%s")
            let comOptionsprojFile =
              try
                let test = Path.GetDirectoryName( com.Options.projFile )
                if test <> "" then
                  com.Options.projFile
                else
                  "." + Path.DirectorySeparatorChar.ToString() + com.Options.projFile
              with
              | _ -> "." + Path.DirectorySeparatorChar.ToString() + com.Options.projFile

            checker.GetProjectOptionsFromScript(
                comOptionsprojFile, projCode, DateTime.UtcNow, otherFlags=otherFlags)
            |> Async.RunSynchronously
        | _ ->
            let properties =
                [("DefineConstants", String.concat ";" com.Options.symbols)]
            ProjectCracker.GetProjectOptionsFromProjectFile(
                com.Options.projFile, properties, DateTime.UtcNow)
        |> checker.ParseAndCheckProject
        |> Async.RunSynchronously
    let errors =
        checkProjectResults.Errors
        |> Array.filter (fun x -> x.Severity = FSharpErrorSeverity.Error)
    if errors.Length = 0
    then checkProjectResults
    else errors
        |> Seq.map (fun e -> "> " + e.Message)<commit_msg>revert back to before #32<commit_after>module Fable.Main

open System
open System.IO
open Microsoft.FSharp.Compiler
open Microsoft.FSharp.Compiler.Ast
open Microsoft.FSharp.Compiler.SourceCodeServices
open Newtonsoft.Json
open Fable

let readOptions argv =
    let def opts key defArg f =
        defaultArg (Map.tryFind key opts |> Option.map f) defArg
    let rec readOpts opts = function
        | [] -> opts
        | (opt: string)::rest ->
            let k = opt.Substring(2)
            match Map.tryFind k opts with
            | None -> Map.add k rest.Head opts
            | Some v -> Map.add k (v + "," + rest.Head) opts
            |> readOpts <| rest.Tail
    let opts = readOpts Map.empty<_,_> (List.ofArray argv)
    {
        code = def opts "code" null id
        outDir = def opts "outDir" "." id
        lib = def opts "lib" "." id
        projFile = def opts "projFile" null id
        watch = def opts "watch" false bool.Parse
        symbols = def opts "symbols" [||] (fun x -> x.Split(','))
    }

let parseFSharpProject (com: ICompiler) (checker: FSharpChecker) (projCode: string option) =
    let checkProjectResults =
        let projCode =
            match projCode with
            | None -> File.ReadAllText com.Options.projFile
            | Some projCode ->
                File.WriteAllText(com.Options.projFile, projCode)
                projCode
        match (Path.GetExtension com.Options.projFile).ToLower() with
        | ".fsx" ->
            let otherFlags =
                com.Options.symbols |> Array.map (sprintf "--define:%s")
            checker.GetProjectOptionsFromScript(
                com.Options.projFile, projCode, DateTime.UtcNow, otherFlags=otherFlags)
            |> Async.RunSynchronously
        | _ ->
            let properties =
                [("DefineConstants", String.concat ";" com.Options.symbols)]
            ProjectCracker.GetProjectOptionsFromProjectFile(
                com.Options.projFile, properties, DateTime.UtcNow)
        |> checker.ParseAndCheckProject
        |> Async.RunSynchronously
    let errors =
        checkProjectResults.Errors
        |> Array.filter (fun x -> x.Severity = FSharpErrorSeverity.Error)
    if errors.Length = 0
    then checkProjectResults
    else errors
        |> Seq.map (fun e -> "> " + e.Message)
{'fraction_non_alphanumeric': 0.10159045725646124, 'fraction_numerical': 0.0011928429423459245, 'mean_word_length': 1.9893048128342246, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4767236', 'n_tokens_mistral': 1473, 'n_tokens_neox': 1402, 'n_words': 368}