<commit_before>(*
   Copyright 2008-2014 <NAME> and Microsoft Research

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*)
// Using light syntax in this file because of object-oriented F# constructs
// (c) Microsoft Corporation. All rights reserved
module FStar.Util

open System
open System.Text
open System.Diagnostics
open System.IO
open System.IO.Compression
open System.Security.Cryptography
open System.Runtime.Serialization
open System.Runtime.Serialization.Json

let return_all x = x

type time = System.DateTime
let now () = System.DateTime.Now
let time_diff (t1:time) (t2:time) : float * int =
    let ts = t2 - t1 in
    ts.TotalSeconds, int32 ts.TotalMilliseconds
let record_time f =
    let start = now () in
    let res = f () in
    let _, elapsed = time_diff start (now()) in
    res, elapsed
let get_file_last_modification_time f = System.IO.File.GetLastWriteTime f
let is_before t1 t2 = System.DateTime.Compare (t1, t2) < 0
let string_of_time (t:time) = t.ToString "MM-dd-yyyy"

exception Impos
exception NYI of string
exception Failure of string
let max_int: int = System.Int32.MaxValue

type proc = {m:Object;
             outbuf:StringBuilder;
             proc:Process;
             killed:ref<bool>;
             id:string}
let all_procs : ref<list<proc>> = ref []
open System.Threading
let global_lock = new Object()
let monitor_enter m = System.Threading.Monitor.Enter(m)
let monitor_exit m = System.Threading.Monitor.Exit(m)
let monitor_wait m = ignore <| System.Threading.Monitor.Wait(m)
let monitor_pulse m = System.Threading.Monitor.Pulse(m)
let current_tid () = System.Threading.Thread.CurrentThread.ManagedThreadId
let sleep n = System.Threading.Thread.Sleep(0+n)
let atomically (f:unit -> 'a) =
    System.Threading.Monitor.Enter(global_lock);
    let result = f () in
    System.Threading.Monitor.Exit(global_lock);
    result
let spawn (f:unit -> unit) = let t = new Thread(f) in t.Start()
let ctr = ref 0

let launch_process (id:string) (prog:string) (args:string) (input:string) (cond:string -> string -> bool) : string =
  let proc = start_process id prog args cond in
  ask_process proc input

let start_process (id:string) (prog:string) (args:string) (cond:string -> string -> bool) : proc =
    let signal = new Object() in
    let startInfo = new ProcessStartInfo() in
    let driverOutput = new StringBuilder() in
    let killed = ref false in
    let proc = new Process() in
    incr ctr;
    let proc_wrapper = {m=signal;
                        outbuf=new StringBuilder();
                        proc=proc;
                        killed=killed;
                        id=prog ^ ":" ^id^ "-" ^ (string_of_int !ctr)} in

    startInfo.FileName <- prog;
    startInfo.Arguments <- args;
    startInfo.UseShellExecute <- false;
    startInfo.RedirectStandardOutput <- true;
    startInfo.RedirectStandardInput <- true;
    proc.EnableRaisingEvents <- true;
    proc.OutputDataReceived.AddHandler(
            DataReceivedEventHandler(
                fun _ args ->
                    if !killed then ()
                    else
                        ignore <| driverOutput.Append(args.Data);
                        ignore <| driverOutput.Append("\n");
                        if null = args.Data
                            then (Printf.printf "Unexpected output from %s\n%s\n" prog (driverOutput.ToString()));
                        if null = args.Data || cond id args.Data
                        then
                            System.Threading.Monitor.Enter(signal);
                            ignore (proc_wrapper.outbuf.Clear());
                            ignore (proc_wrapper.outbuf.Append(driverOutput.ToString()));
                            ignore (driverOutput.Clear());
                            System.Threading.Monitor.Pulse(signal);
                            System.Threading.Monitor.Exit(signal)));
    proc.Exited.AddHandler(
            EventHandler(fun _ _ ->
            if !killed then ()
            else
                System.Threading.Monitor.Enter(signal);
                killed := true;
                Printf.fprintf stdout "%s exited inadvertently\n%s\n" prog (driverOutput.ToString());
                stdout.Flush();
                System.Threading.Monitor.Exit(signal);
                exit(1)));
    proc.StartInfo <- startInfo;
    proc.Start() |> ignore;
    proc.BeginOutputReadLine();
    all_procs := proc_wrapper::!all_procs;
//        Printf.printf "Started process %s\n" (proc.id);
    proc_wrapper
let tid () = System.Threading.Thread.CurrentThread.ManagedThreadId |> string_of_int

let ask_process (p:proc) (input:string) : string =
    System.Threading.Monitor.Enter(p.m);
    //Printf.printf "Thread %s is asking process %s\n" (tid()) p.id;
    //Printf.printf "Thread %s is writing to process %s ... responding?=%A\n" (tid()) p.id p.proc.Responding;
    //Printf.fprintf stderr "Thread %s is writing to process %s:\n%s\n" (tid()) p.id input;
//    if p.id = "z3.exe:bg"
//    then begin
//        Printf.printf "Thread BG break\n"
//    end;
    p.proc.StandardInput.WriteLine(input);
//    Printf.printf "Thread %s is waiting for process to reply\n" (tid());
//    flush(stdout);
    ignore <| System.Threading.Monitor.Wait(p.m);
//    Printf.printf "Thread %s is continuing with reply from process %s\n" (tid()) p.id;
    let x = p.outbuf.ToString() in
    System.Threading.Monitor.Exit(p.m);
    x

let kill_process (p:proc) =
//    Printf.printf "Killing process %s\n" (p.id);
    p.killed := true;
    System.Threading.Monitor.Enter(p.m);
    p.proc.StandardInput.Close();
    System.Threading.Monitor.Exit(p.m);
    p.proc.WaitForExit()

let kill_all () = !all_procs |> List.iter (fun p -> if not !p.killed then kill_process p)

let run_proc (name:string) (args:string) (stdin:string) : bool * string * string =
  let pinfo = new ProcessStartInfo(name, args) in
  pinfo.RedirectStandardOutput <- true;
  pinfo.RedirectStandardError <- true;<commit_msg>ocaml mostly working with multi-core, F# multi-core not working, single-core same as before<commit_after>(*
   Copyright 2008-2014 <NAME> and Microsoft Research

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*)
// Using light syntax in this file because of object-oriented F# constructs
// (c) Microsoft Corporation. All rights reserved
module FStar.Util

open System
open System.Text
open System.Diagnostics
open System.IO
open System.IO.Compression
open System.Security.Cryptography
open System.Runtime.Serialization
open System.Runtime.Serialization.Json

let return_all x = x

type time = System.DateTime
let now () = System.DateTime.Now
let time_diff (t1:time) (t2:time) : float * int =
    let ts = t2 - t1 in
    ts.TotalSeconds, int32 ts.TotalMilliseconds
let record_time f =
    let start = now () in
    let res = f () in
    let _, elapsed = time_diff start (now()) in
    res, elapsed
let get_file_last_modification_time f = System.IO.File.GetLastWriteTime f
let is_before t1 t2 = System.DateTime.Compare (t1, t2) < 0
let string_of_time (t:time) = t.ToString "MM-dd-yyyy"

exception Impos
exception NYI of string
exception Failure of string
let max_int: int = System.Int32.MaxValue

type proc = {m:Object;
             outbuf:StringBuilder;
             proc:Process;
             killed:ref<bool>;
             id:string}
let all_procs : ref<list<proc>> = ref []
open System.Threading
let global_lock = new Object()
let monitor_enter m = System.Threading.Monitor.Enter(m)
let monitor_exit m = System.Threading.Monitor.Exit(m)
let monitor_wait m = ignore <| System.Threading.Monitor.Wait(m)
let monitor_pulse m = System.Threading.Monitor.Pulse(m)
let current_tid () = System.Threading.Thread.CurrentThread.ManagedThreadId
let sleep n = System.Threading.Thread.Sleep(0+n)
let atomically (f:unit -> 'a) =
    System.Threading.Monitor.Enter(global_lock);
    let result = f () in
    System.Threading.Monitor.Exit(global_lock);
    result
let spawn (f:unit -> unit) = let t = new Thread(f) in t.Start()
let ctr = ref 0

let start_process (id:string) (prog:string) (args:string) (cond:string -> string -> bool) : proc =
    let signal = new Object() in
    let startInfo = new ProcessStartInfo() in
    let driverOutput = new StringBuilder() in
    let killed = ref false in
    let proc = new Process() in
    incr ctr;
    let proc_wrapper = {m=signal;
                        outbuf=new StringBuilder();
                        proc=proc;
                        killed=killed;
                        id=prog ^ ":" ^id^ "-" ^ (string_of_int !ctr)} in

    startInfo.FileName <- prog;
    startInfo.Arguments <- args;
    startInfo.UseShellExecute <- false;
    startInfo.RedirectStandardOutput <- true;
    startInfo.RedirectStandardInput <- true;
    proc.EnableRaisingEvents <- true;
    proc.OutputDataReceived.AddHandler(
            DataReceivedEventHandler(
                fun _ args ->
                    if !killed then ()
                    else
                        ignore <| driverOutput.Append(args.Data);
                        ignore <| driverOutput.Append("\n");
                        if null = args.Data
                            then (Printf.printf "Unexpected output from %s\n%s\n" prog (driverOutput.ToString()));
                        if null = args.Data || cond id args.Data
                        then
                            System.Threading.Monitor.Enter(signal);
                            ignore (proc_wrapper.outbuf.Clear());
                            ignore (proc_wrapper.outbuf.Append(driverOutput.ToString()));
                            ignore (driverOutput.Clear());
                            System.Threading.Monitor.Pulse(signal);
                            System.Threading.Monitor.Exit(signal)));
    proc.Exited.AddHandler(
            EventHandler(fun _ _ ->
            if !killed then ()
            else
                System.Threading.Monitor.Enter(signal);
                killed := true;
                Printf.fprintf stdout "%s exited inadvertently\n%s\n" prog (driverOutput.ToString());
                stdout.Flush();
                System.Threading.Monitor.Exit(signal);
                exit(1)));
    proc.StartInfo <- startInfo;
    proc.Start() |> ignore;
    proc.BeginOutputReadLine();
    all_procs := proc_wrapper::!all_procs;
//        Printf.printf "Started process %s\n" (proc.id);
    proc_wrapper
let tid () = System.Threading.Thread.CurrentThread.ManagedThreadId |> string_of_int

let ask_process (p:proc) (input:string) : string =
    System.Threading.Monitor.Enter(p.m);
    //Printf.printf "Thread %s is asking process %s\n" (tid()) p.id;
    //Printf.printf "Thread %s is writing to process %s ... responding?=%A\n" (tid()) p.id p.proc.Responding;
    //Printf.fprintf stderr "Thread %s is writing to process %s:\n%s\n" (tid()) p.id input;
//    if p.id = "z3.exe:bg"
//    then begin
//        Printf.printf "Thread BG break\n"
//    end;
    p.proc.StandardInput.WriteLine(input);
//    Printf.printf "Thread %s is waiting for process to reply\n" (tid());
//    flush(stdout);
    ignore <| System.Threading.Monitor.Wait(p.m);
//    Printf.printf "Thread %s is continuing with reply from process %s\n" (tid()) p.id;
    let x = p.outbuf.ToString() in
    System.Threading.Monitor.Exit(p.m);
    x

let launch_process (id:string) (prog:string) (args:string) (input:string) (cond:string -> string -> bool) : string =
  let proc = start_process id prog args cond in
  ask_process proc input

let kill_process (p:proc) =
//    Printf.printf "Killing process %s\n" (p.id);
    p.killed := true;
    System.Threading.Monitor.Enter(p.m);
    p.proc.StandardInput.Close();
    System.Threading.Monitor.Exit(p.m);
    p.proc.WaitForExit()

let kill_all () = !all_procs |> List.iter (fun p -> if not !p.killed then kill_process p)

let run_proc (name:string) (args:string) (stdin:string) : bool * string * string =
  let pinfo = new ProcessStartInfo(name, args) in
  pinfo.RedirectStandardOutput <- true;
  pinfo.RedirectStandardError <- true;
{'fraction_non_alphanumeric': 0.11174300645557947, 'fraction_numerical': 0.004457423916384875, 'mean_word_length': 2.5419161676646707, 'pattern_counts': {'":': 2, '<': 39, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4501463', 'n_tokens_mistral': 3847, 'n_tokens_neox': 3785, 'n_words': 1095}