<commit_before>﻿namespace XPlot.GoogleCharts

open Google.DataTable.Net.Wrapper.Extension
open Microsoft.FSharp.Reflection
open Newtonsoft.Json
open System
open System.Data
open System.Globalization
open System.IO

type key = IConvertible
type value = IConvertible

[<AutoOpen>]
module Data =

    type Datum =
        {
            X : key
            Y1 : value
            Y2 : value option
            Y3 : value option
            Y4 : value option
        }

        static member New(x, y) = {X = x; Y1 = y; Y2 = None; Y3 = None; Y4 = None}

        static member New(x, y1, y2) = {X = x; Y1 = y1; Y2 = Some y2; Y3 = None; Y4 = None}

        static member New(x, y1, y2, y3) = {X = x; Y1 = y1; Y2 = Some y2; Y3 = Some y3; Y4 = None}

        static member New(x, y1, y2, y3, y4) = {X = x; Y1 = y1; Y2 = Some y2; Y3 = Some y3; Y4 = Some y4}

    type Series =
        {
            DataPoints : seq<Datum>
        }

        static member New dps = {DataPoints = dps}

        static member New (data: seq<#key * #value>) =
            data
            |> Seq.map Datum.New
            |> Series.New

        static member New (data: seq<#key * #value * #value>) =
            data
            |> Seq.map Datum.New
            |> Series.New

        static member New (data: seq<#key * #value * #value * #value>) =
            data
            |> Seq.map Datum.New
            |> Series.New

        static member New (data: seq<#key * #value * #value * #value * #value>) =
            data
            |> Seq.map Datum.New
            |> Series.New

    let makeDataTable series labels =

        let rows =
            match Seq.length series with
            | 1 ->
                series
                |> Seq.item 0
                |> fun x ->
                    x.DataPoints
                    |> Seq.map (fun dp ->
                        let key = dp.X
                        let fields =
                            [
                                    yield Some dp.Y1
                                    if dp.Y2.IsSome then yield dp.Y2
                                    if dp.Y3.IsSome then yield dp.Y3
                                    if dp.Y4.IsSome then yield dp.Y4
                            ]
                            |> List.mapi (fun idx x -> x, idx + 1)
                        key, fields)
            | _ ->
                let dps =
                    series
                    |> Seq.map (fun x -> x.DataPoints)
                    |> Seq.concat

                let keys =
                    dps
                    |> Seq.map (fun x -> x.X)
                    |> Seq.distinct

                keys
                |> Seq.map (fun key ->
                    let datapoints =
                        series
                        |> Seq.map (fun x -> Seq.tryFind (fun dp -> dp.X = key) x.DataPoints)
                    let fields' =
                        [
                            for x in datapoints do
                                if x.IsSome then
                                    let x' = Option.get x
                                    yield Some x'.Y1
                                    if x'.Y2.IsSome then yield x'.Y2
                                    if x'.Y3.IsSome then yield x'.Y3
                                    if x'.Y4.IsSome then yield x'.Y4
                                else yield None
                        ]
                        |> List.mapi (fun idx x -> x, idx + 1)
                    key, fields'
                )
            
        let sysDt = new System.Data.DataTable()
        sysDt.Locale <- CultureInfo.InvariantCulture

        let longestRow = Seq.maxBy (snd >> Seq.length) rows

        let labels' =
            let length = snd longestRow |> Seq.length
            match labels with
            | None -> seq {for x in 1 .. (length + 1) -> "Column " + string x}
            | Some labelsSeq ->
                match (Seq.length labelsSeq) = length with
                | false -> Seq.append ["Column 1"] labelsSeq
                | true -> labelsSeq

        sysDt.Columns.Add(Seq.item 0 labels', (fst longestRow).GetType())
        |> ignore

        let values = Seq.map snd rows 

        (snd longestRow)
        |> List.iteri (fun idx _ ->
            let columnType =
                values
                |> Seq.tryPick (fun x ->
                    match x.Length with
                    | l when l < idx + 1 -> None
                    | _ -> fst x.[idx])
                |> Option.get
                |> fun x -> x.GetType()
            sysDt.Columns.Add(Seq.item (idx + 1) labels', columnType)
            |> ignore
        )

        rows
        |> Seq.iter (fun (key, values) -> 
            let row = sysDt.NewRow()
            row.SetField(0, key)
            values
            |> Seq.iter (fun (value, column) ->
                match value with
                | Some v -> row.SetField(column, v)
                | _ -> ()
            )
            sysDt.Rows.Add(row)
        )

        sysDt

module Html =
<commit_msg>NET45 compatibility<commit_after>﻿namespace XPlot.GoogleCharts

open Google.DataTable.Net.Wrapper.Extension
open Microsoft.FSharp.Reflection
open Newtonsoft.Json
open System
open System.Data

open System.Globalization
open System.IO

type key = IConvertible
type value = IConvertible

[<AutoOpen>]
module Data =

    type Datum =
        {
            X : key
            Y1 : value
            Y2 : value option
            Y3 : value option
            Y4 : value option
        }

        static member New(x, y) = {X = x; Y1 = y; Y2 = None; Y3 = None; Y4 = None}

        static member New(x, y1, y2) = {X = x; Y1 = y1; Y2 = Some y2; Y3 = None; Y4 = None}

        static member New(x, y1, y2, y3) = {X = x; Y1 = y1; Y2 = Some y2; Y3 = Some y3; Y4 = None}

        static member New(x, y1, y2, y3, y4) = {X = x; Y1 = y1; Y2 = Some y2; Y3 = Some y3; Y4 = Some y4}

    type Series =
        {
            DataPoints : seq<Datum>
        }

        static member New dps = {DataPoints = dps}

        static member New (data: seq<#key * #value>) =
            data
            |> Seq.map Datum.New
            |> Series.New

        static member New (data: seq<#key * #value * #value>) =
            data
            |> Seq.map Datum.New
            |> Series.New

        static member New (data: seq<#key * #value * #value * #value>) =
            data
            |> Seq.map Datum.New
            |> Series.New

        static member New (data: seq<#key * #value * #value * #value * #value>) =
            data
            |> Seq.map Datum.New
            |> Series.New

    let makeDataTable series labels =

        let rows =
            match Seq.length series with
            | 1 ->
                series
                |> Seq.item 0
                |> fun x ->
                    x.DataPoints
                    |> Seq.map (fun dp ->
                        let key = dp.X
                        let fields =
                            [
                                    yield Some dp.Y1
                                    if dp.Y2.IsSome then yield dp.Y2
                                    if dp.Y3.IsSome then yield dp.Y3
                                    if dp.Y4.IsSome then yield dp.Y4
                            ]
                            |> List.mapi (fun idx x -> x, idx + 1)
                        key, fields)
            | _ ->
                let dps =
                    series
                    |> Seq.map (fun x -> x.DataPoints)
                    |> Seq.concat

                let keys =
                    dps
                    |> Seq.map (fun x -> x.X)
                    |> Seq.distinct

                keys
                |> Seq.map (fun key ->
                    let datapoints =
                        series
                        |> Seq.map (fun x -> Seq.tryFind (fun dp -> dp.X = key) x.DataPoints)
                    let fields' =
                        [
                            for x in datapoints do
                                if x.IsSome then
                                    let x' = Option.get x
                                    yield Some x'.Y1
                                    if x'.Y2.IsSome then yield x'.Y2
                                    if x'.Y3.IsSome then yield x'.Y3
                                    if x'.Y4.IsSome then yield x'.Y4
                                else yield None
                        ]
                        |> List.mapi (fun idx x -> x, idx + 1)
                    key, fields'
                )
            
        let sysDt = new System.Data.DataTable()
        sysDt.Locale <- CultureInfo.InvariantCulture

        let longestRow = Seq.maxBy (snd >> Seq.length) rows

        let labels' =
            let length = snd longestRow |> Seq.length
            match labels with
            | None -> seq {for x in 1 .. (length + 1) -> "Column " + string x}
            | Some labelsSeq ->
                match (Seq.length labelsSeq) = length with
                | false -> Seq.append ["Column 1"] labelsSeq
                | true -> labelsSeq

        sysDt.Columns.Add(Seq.item 0 labels', (fst longestRow).GetType())
        |> ignore

        let values = Seq.map snd rows 

        (snd longestRow)
        |> List.iteri (fun idx _ ->
            let columnType =
                values
                |> Seq.tryPick (fun x ->
                    match x.Length with
                    | l when l < idx + 1 -> None
                    | _ -> fst x.[idx])
                |> Option.get
                |> fun x -> x.GetType()
            sysDt.Columns.Add(Seq.item (idx + 1) labels', columnType)
            |> ignore
        )

        rows
        |> Seq.iter (fun (key, values) -> 
            let row = sysDt.NewRow()
            row.[0] <- key // SetField(0, key)
            values
            |> Seq.iter (fun (value, column) ->
                match value with
                | Some v -> row.[column]  <- v // SetField(column, v)
                | _ -> ()
            )
            sysDt.Rows.Add(row)
        )

        sysDt

module Html =

{'fraction_non_alphanumeric': 0.0959322367773072, 'fraction_numerical': 0.012705604254900029, 'mean_word_length': 0.920196671709531, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 125, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5913247', 'n_tokens_mistral': 2966, 'n_tokens_neox': 2724, 'n_words': 821}