<commit_before>﻿namespace FSharp.Enterprise

module Json = 
    
    open System
    open Microsoft.FSharp.Reflection
    open Newtonsoft.Json
    open Newtonsoft.Json.Linq
    open Newtonsoft.Json.Converters
    
    type UnionTypeConverter() =
        inherit JsonConverter()
    
        let doRead pos (reader: JsonReader) = 
            reader.Read() |> ignore
    
        override x.CanConvert(typ:Type) =
            let result = 
                ((typ.GetInterface(typeof<System.Collections.IEnumerable>.FullName) = null) 
                && FSharpType.IsUnion typ)
            result
    
        override x.WriteJson(writer: JsonWriter, value: obj, serializer: JsonSerializer) =
            let t = value.GetType()
            let write (name : string) (fields : obj []) = 
                writer.WriteStartObject()
                writer.WritePropertyName("case")
                writer.WriteValue(name)
                if fields.Length > 0 || serializer.NullValueHandling = NullValueHandling.Include
                then  
                    writer.WritePropertyName("values")
                    serializer.Serialize(writer, fields)
                writer.WriteEndObject()   
    
            let (info, fields) = FSharpValue.GetUnionFields(value, t)
            write info.Name fields
    
        override x.ReadJson(reader: JsonReader, objectType: Type, existingValue: obj, serializer: JsonSerializer) =      
             let cases = FSharpType.GetUnionCases(objectType)
             
             let createUnion (reader:JsonReader) = 
                 doRead "1" reader
                 doRead "2" reader
                 let case = cases |> Array.find(fun x -> x.Name = if reader.Value = null then "None" else reader.Value.ToString())
                 printfn "%A" case
                 doRead "3" reader
                 let fields =
                    if reader.Value <> null
                    then
                        doRead "4" reader
                        doRead "5" reader
                        [| 
                               for field in case.GetFields() do
                                   let result = serializer.Deserialize(reader, field.PropertyType)
                                   reader.Read() |> ignore
                                   yield result
                         |]<commit_msg>Removed printf debugging in Json<commit_after>﻿namespace FSharp.Enterprise

module Json = 
    
    open System
    open Microsoft.FSharp.Reflection
    open Newtonsoft.Json
    open Newtonsoft.Json.Linq
    open Newtonsoft.Json.Converters
    
    type UnionTypeConverter() =
        inherit JsonConverter()
    
        let doRead pos (reader: JsonReader) = 
            reader.Read() |> ignore
    
        override x.CanConvert(typ:Type) =
            let result = 
                ((typ.GetInterface(typeof<System.Collections.IEnumerable>.FullName) = null) 
                && FSharpType.IsUnion typ)
            result
    
        override x.WriteJson(writer: JsonWriter, value: obj, serializer: JsonSerializer) =
            let t = value.GetType()
            let write (name : string) (fields : obj []) = 
                writer.WriteStartObject()
                writer.WritePropertyName("case")
                writer.WriteValue(name)
                if fields.Length > 0 || serializer.NullValueHandling = NullValueHandling.Include
                then  
                    writer.WritePropertyName("values")
                    serializer.Serialize(writer, fields)
                writer.WriteEndObject()   
    
            let (info, fields) = FSharpValue.GetUnionFields(value, t)
            write info.Name fields
    
        override x.ReadJson(reader: JsonReader, objectType: Type, existingValue: obj, serializer: JsonSerializer) =      
             let cases = FSharpType.GetUnionCases(objectType)
             
             let createUnion (reader:JsonReader) = 
                 doRead "1" reader
                 doRead "2" reader
                 let case = cases |> Array.find(fun x -> x.Name = if reader.Value = null then "None" else reader.Value.ToString())
                 doRead "3" reader
                 let fields =
                    if reader.Value <> null
                    then
                        doRead "4" reader
                        doRead "5" reader
                        [| 
                               for field in case.GetFields() do
                                   let result = serializer.Deserialize(reader, field.PropertyType)
                                   reader.Read() |> ignore
                                   yield result
                         |]
{'fraction_non_alphanumeric': 0.07716973065412569, 'fraction_numerical': 0.0025651988029072254, 'mean_word_length': 1.373921867072552, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7633626', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1086, 'n_words': 273}