<commit_before>﻿// Copyright (C) by <NAME>

module Toys.PU

open System.Collections.Generic
open System.IO
open Infers
open Infers.Rep
open Toys.Basic
open Toys.Rec

type State = Writing | Cyclic | Acyclic
type Info = {Pos: int64; mutable State: State}
type PU<'x> =
  {Pickle: Dictionary<obj, Info> -> BinaryWriter -> 'x -> unit
   Unpickle: Dictionary<int64, obj> -> BinaryReader -> 'x}
type PUE<'x> = {PU: PU<'x>}
type PUP<'e, 'r, 'o, 't> = P of PU<'e>
type PUS<'p, 'o, 't> = S of list<PU<'t>>

type [<InferenceRules>] PU () =
  member t.Entry (_: Rep, _: Basic, _: Rec, xP) = {PU = xP}
  member t.Int : PU<int> =
    {Pickle = fun _ w -> w.Write
     Unpickle = fun _ r -> r.ReadInt32 ()}
  member t.Float : PU<float> =
    {Pickle = fun _ w -> w.Write
     Unpickle = fun _ r -> r.ReadDouble ()}
  member t.String : PU<string> =
    {Pickle = fun _ w -> w.Write
     Unpickle = fun _ r -> r.ReadString ()}
  member t.Elem (_: Elem<'e, 'r, 'o, 't>, eP: PU<'e>) : PUP<'e, 'r, 'o, 't> =
    P eP
  member t.Pair (P eP: PUP<     'e     , Pair<'e, 'r>, 'o, 't>,
                 P rP: PUP<         'r ,          'r , 'o, 't>)
                     : PUP<Pair<'e, 'r>, Pair<'e, 'r>, 'o, 't> =
    P {Pickle = fun d w (Pair (e, r)) -> eP.Pickle d w e; rP.Pickle d w r
       Unpickle = fun d r -> Pair (eP.Unpickle d r, rP.Unpickle d r)}
  member t.Product (_: Choice<Union<'t>, Tuple<'t>>,
                    asPairs: AsPairs<'p, 'o, 't>,
                    P pP: PUP<'p, 'p, 'o, 't>) : PU<'t> =
    {Pickle = fun d w -> asPairs.ToPairs >> pP.Pickle d w<commit_msg>Unions already handled so restrict to tuples.<commit_after>﻿// Copyright (C) by <NAME>

module Toys.PU

open System.Collections.Generic
open System.IO
open Infers
open Infers.Rep
open Toys.Basic
open Toys.Rec

type State = Writing | Cyclic | Acyclic
type Info = {Pos: int64; mutable State: State}
type PU<'x> =
  {Pickle: Dictionary<obj, Info> -> BinaryWriter -> 'x -> unit
   Unpickle: Dictionary<int64, obj> -> BinaryReader -> 'x}
type PUE<'x> = {PU: PU<'x>}
type PUP<'e, 'r, 'o, 't> = P of PU<'e>
type PUS<'p, 'o, 't> = S of list<PU<'t>>

type [<InferenceRules>] PU () =
  member t.Entry (_: Rep, _: Basic, _: Rec, xP) = {PU = xP}
  member t.Int : PU<int> =
    {Pickle = fun _ w -> w.Write
     Unpickle = fun _ r -> r.ReadInt32 ()}
  member t.Float : PU<float> =
    {Pickle = fun _ w -> w.Write
     Unpickle = fun _ r -> r.ReadDouble ()}
  member t.String : PU<string> =
    {Pickle = fun _ w -> w.Write
     Unpickle = fun _ r -> r.ReadString ()}
  member t.Elem (_: Elem<'e, 'r, 'o, 't>, eP: PU<'e>) : PUP<'e, 'r, 'o, 't> =
    P eP
  member t.Pair (P eP: PUP<     'e     , Pair<'e, 'r>, 'o, 't>,
                 P rP: PUP<         'r ,          'r , 'o, 't>)
                     : PUP<Pair<'e, 'r>, Pair<'e, 'r>, 'o, 't> =
    P {Pickle = fun d w (Pair (e, r)) -> eP.Pickle d w e; rP.Pickle d w r
       Unpickle = fun d r -> Pair (eP.Unpickle d r, rP.Unpickle d r)}
  member t.Tuple (_: Tuple<'t>,
                  asPairs: AsPairs<'p, 'o, 't>,
                  P pP: PUP<'p, 'p, 'o, 't>) : PU<'t> =
    {Pickle = fun d w -> asPairs.ToPairs >> pP.Pickle d w
{'fraction_non_alphanumeric': 0.1909467644246095, 'fraction_numerical': 0.003825310806503028, 'mean_word_length': 2.3136219640971487, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '189379', 'n_tokens_mistral': 1356, 'n_tokens_neox': 1326, 'n_words': 392}