<commit_before>﻿open System
open Utilities
open ParserMonad
open BasicExpression
open ConcreteExpressionParserPrelude
open ConcreteExpressionParser
open CodeGenerator
open Microsoft.CSharp
open System.CodeDom.Compiler
open System.IO
open System.Runtime.Serialization
open System.Xml.Serialization

do System.Threading.Thread.CurrentThread.CurrentCulture <- System.Globalization.CultureInfo.GetCultureInfo("EN-US")

let flushCSFileOnError = true
let numSteps = 0

let runDeduction path =
  let originalFilePath = System.IO.Path.Combine(path, "transform.mc")
  let rules = System.IO.File.ReadAllText(originalFilePath)
  let title = System.IO.Path.GetFileName path
  let timer = System.Diagnostics.Stopwatch()
  let output = ref ""
  let addOutput s = output := sprintf "%s\n%s" (output.Value) s
  match (program()).Parse (rules |> Seq.toList) ConcreteExpressionContext.Empty Position.Zero with
  | (x,_,ctxt,pos)::xs,[] -> 
    fun (input:string) ->
      let input = input.Trim([|'\r'; '\n'|]) + "\n"
//      do debug_expr <- true
      match expr().Parse (input |> Seq.toList) ctxt Position.Zero with
      | (y,_,ctxt',pos')::ys,[] ->
        try
          let generatedPath = generateCode originalFilePath title x y ctxt
          let args = new System.Collections.Generic.Dictionary<string, string>()
          do args.Add("CompilerVersion", "v4.5")
          let csc = new CSharpCodeProvider()
          let parameters = new CompilerParameters([| "mscorlib.dll"; "System.dll"; "System.Runtime.dll"; "System.Core.dll"; "System.Collections.Immutable.dll" |], sprintf "%s.dll" title, true)
          do parameters.GenerateInMemory <- true
          do parameters.CompilerOptions <- @"/optimize+"
          let results = csc.CompileAssemblyFromFile(parameters, [|generatedPath|])
          if results.Errors.HasErrors then
            for error in results.Errors do
              if error.IsWarning |> not then
                do sprintf "%s at %d: %s" error.FileName error.Line error.ErrorText |> addOutput 
            if flushCSFileOnError then
              do System.IO.File.WriteAllText(generatedPath, "")
          else
            let types = results.CompiledAssembly.GetTypes()
            let entryPoint = types |> Seq.find (fun t -> t.Name = "EntryPoint")
            let run = entryPoint.GetMethod("Run")
            let results = run.Invoke(null, [|false|]) :?> seq<obj> |> Seq.toList
            do timer.Start()
            for i = 1 to numSteps do
              do run.Invoke(null, [|false|]) :?> seq<obj> |> Seq.toList |> ignore<commit_msg>Filenames now show up in generated code ( used to be empty).<commit_after>﻿open System
open Utilities
open ParserMonad
open BasicExpression
open ConcreteExpressionParserPrelude
open ConcreteExpressionParser
open CodeGenerator
open Microsoft.CSharp
open System.CodeDom.Compiler
open System.IO
open System.Runtime.Serialization
open System.Xml.Serialization

do System.Threading.Thread.CurrentThread.CurrentCulture <- System.Globalization.CultureInfo.GetCultureInfo("EN-US")

let flushCSFileOnError = true
let numSteps = 0

let runDeduction path =
  let originalFilePath = System.IO.Path.Combine(path, "transform.mc")
  let rules = System.IO.File.ReadAllText(originalFilePath)
  let title = System.IO.Path.GetFileName path
  let timer = System.Diagnostics.Stopwatch()
  let output = ref ""
  let addOutput s = output := sprintf "%s\n%s" (output.Value) s
  match (program()).Parse (rules |> Seq.toList) ConcreteExpressionContext.Empty (Position.Create path) with
  | (x,_,ctxt,pos)::xs,[] -> 
    fun (input:string) ->
      let input = input.Trim([|'\r'; '\n'|]) + "\n"
//      do debug_expr <- true
      match expr().Parse (input |> Seq.toList) ctxt Position.Zero with
      | (y,_,ctxt',pos')::ys,[] ->
        try
          let generatedPath = generateCode originalFilePath title x y ctxt
          let args = new System.Collections.Generic.Dictionary<string, string>()
          do args.Add("CompilerVersion", "v4.5")
          let csc = new CSharpCodeProvider()
          let parameters = new CompilerParameters([| "mscorlib.dll"; "System.dll"; "System.Runtime.dll"; "System.Core.dll"; "System.Collections.Immutable.dll" |], sprintf "%s.dll" title, true)
          do parameters.GenerateInMemory <- true
          do parameters.CompilerOptions <- @"/optimize+"
          let results = csc.CompileAssemblyFromFile(parameters, [|generatedPath|])
          if results.Errors.HasErrors then
            for error in results.Errors do
              if error.IsWarning |> not then
                do sprintf "%s at %d: %s" error.FileName error.Line error.ErrorText |> addOutput 
            if flushCSFileOnError then
              do System.IO.File.WriteAllText(generatedPath, "")
          else
            let types = results.CompiledAssembly.GetTypes()
            let entryPoint = types |> Seq.find (fun t -> t.Name = "EntryPoint")
            let run = entryPoint.GetMethod("Run")
            let results = run.Invoke(null, [|false|]) :?> seq<obj> |> Seq.toList
            do timer.Start()
            for i = 1 to numSteps do
              do run.Invoke(null, [|false|]) :?> seq<obj> |> Seq.toList |> ignore
{'fraction_non_alphanumeric': 0.12088974854932302, 'fraction_numerical': 0.0015473887814313346, 'mean_word_length': 3.0023219814241484, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5589528', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1501, 'n_words': 396}