<commit_before>﻿namespace BitReaderWriterFs

open System
open System.Collections
open System.Collections.Generic
open System.IO

[<AutoOpen>]
module internal Helpes =
    /// Converts a BitArray to a byte array of length n
    let toByteArray n (bitArr : BitArray) =
        let bytes = Array.zeroCreate<byte> n
        for i = 0 to bitArr.Count-1 do
            if bitArr.[i] 
            then bytes.[i/8] <- bytes.[i/8] <<< 1 ||| 1uy 
            else bytes.[i/8] <- bytes.[i/8] <<< 1
        bytes

    let convertBytes f byteArray    = f(byteArray, 0)

    let convertToInt16              = toByteArray 2 >> (convertBytes BitConverter.ToInt16)
    let convertToUint16             = toByteArray 2 >> (convertBytes BitConverter.ToUInt16)
    let convertToInt32              = toByteArray 4 >> (convertBytes BitConverter.ToInt32)
    let convertToUint32             = toByteArray 4 >> (convertBytes BitConverter.ToUInt32)
    let convertToInt64              = toByteArray 8 >> (convertBytes BitConverter.ToInt64)
    let convertToUint64             = toByteArray 8 >> (convertBytes BitConverter.ToUInt64)

    let convertToFloat              = toByteArray 4 >> (convertBytes BitConverter.ToSingle)
    let convertToDouble             = toByteArray 8 >> (convertBytes BitConverter.ToDouble)

    let convertToBool (bitArr : BitArray)  = bitArr.[0]
    let convertToBytes (bitArr : BitArray) = 
        let len = bitArr.Count / 8 + (if bitArr.Count % 8 <> 0 then 1 else 0)
        toByteArray len bitArr
    let convertToChar                      = toByteArray 1  >> Text.Encoding.Default.GetChars >> (fun arr -> arr.[0])
    let convertToString                    = convertToBytes >> Text.Encoding.Default.GetString

type BitsCount = | N of int | Rest

type BitReaderBuilder (enumerator : IEnumerator<bool>) =
    let mutable bitArrays = []
    let next () = if enumerator.MoveNext() then enumerator.Current else failwithf "Insufficient bits"
    let rest () = [| while enumerator.MoveNext() do yield enumerator.Current |]

    member this.Bind((count, f : BitArray -> 'a), cont : 'a -> 'b) =
        let bitArr = match count with
                     | N n  -> BitArray([| for i = 1 to n do yield next() |])
                     | Rest -> BitArray(rest())
        cont(f bitArr)
        
    member this.Return x = x

/// Container for helper functions
type Bits =
    static member Skip n            = N n, (fun _ -> ())
                                 
    static member ReadInt16 (?n)    = N(defaultArg n 16), convertToInt16
    static member ReadUint16 (?n)   = N(defaultArg n 16), convertToUint16
    static member ReadInt32 (?n)    = N(defaultArg n 32), convertToInt32
    static member ReadUint32 (?n)   = N(defaultArg n 32), convertToUint32    
    static member ReadInt64 (?n)    = N(defaultArg n 64), convertToInt64
    static member ReadUint64 (?n)   = N(defaultArg n 64), convertToUint64

    static member ReadBool ()       = N 1, convertToBool
    static member ReadBytes n       = N n, convertToBytes
    static member ReadChar (?n)     = N(defaultArg n 8), convertToChar
    static member ReadString n      = N n, convertToString

    static member ReadAs n f        = N n, convertToBytes >> f
    static member ReadRest ()       = Rest, convertToBytes

[<AutoOpen>]
module ReaderWorkflow = 
    let bitReader (stream : Stream) = 
        stream.Position <- 0L
        BitReaderBuilder(stream.ToBitSequence().GetEnumerator())<commit_msg>- wrap the type for the Bind function into a BitReader type instead of tuple<commit_after>﻿namespace BitReaderWriterFs

open System
open System.Collections
open System.Collections.Generic
open System.IO

[<AutoOpen>]
module internal Helpes =
    /// Converts a BitArray to a byte array of length n
    let toByteArray n (bitArr : BitArray) =
        let bytes = Array.zeroCreate<byte> n
        for i = 0 to bitArr.Count-1 do
            if bitArr.[i] 
            then bytes.[i/8] <- bytes.[i/8] <<< 1 ||| 1uy 
            else bytes.[i/8] <- bytes.[i/8] <<< 1
        bytes

    let convertBytes f byteArray    = f(byteArray, 0)

    let convertToInt16              = toByteArray 2 >> (convertBytes BitConverter.ToInt16)
    let convertToUint16             = toByteArray 2 >> (convertBytes BitConverter.ToUInt16)
    let convertToInt32              = toByteArray 4 >> (convertBytes BitConverter.ToInt32)
    let convertToUint32             = toByteArray 4 >> (convertBytes BitConverter.ToUInt32)
    let convertToInt64              = toByteArray 8 >> (convertBytes BitConverter.ToInt64)
    let convertToUint64             = toByteArray 8 >> (convertBytes BitConverter.ToUInt64)

    let convertToFloat              = toByteArray 4 >> (convertBytes BitConverter.ToSingle)
    let convertToDouble             = toByteArray 8 >> (convertBytes BitConverter.ToDouble)

    let convertToBool (bitArr : BitArray)  = bitArr.[0]
    let convertToBytes (bitArr : BitArray) = 
        let len = bitArr.Count / 8 + (if bitArr.Count % 8 <> 0 then 1 else 0)
        toByteArray len bitArr
    let convertToChar                      = toByteArray 1  >> Text.Encoding.Default.GetChars >> (fun arr -> arr.[0])
    let convertToString                    = convertToBytes >> Text.Encoding.Default.GetString

type BitsCount = | N of int | Rest
type BitReader<'a> = BitReader of BitsCount * (BitArray -> 'a)

type BitReaderBuilder (enumerator : IEnumerator<bool>) =
    let mutable bitArrays = []
    let next () = if enumerator.MoveNext() then enumerator.Current else failwithf "Insufficient bits"
    let rest () = [| while enumerator.MoveNext() do yield enumerator.Current |]

    member this.Bind(BitReader(count, f), cont : 'a -> 'b) =
        let bitArr = match count with
                     | N n  -> BitArray([| for i = 1 to n do yield next() |])
                     | Rest -> BitArray(rest())
        cont(f bitArr)
        
    member this.Return x = x

/// Container for helper functions
type Bits =
    static member Skip n            = BitReader <| (N n, (fun _ -> ()))

    static member ReadInt16 (?n)    = BitReader <| (N(defaultArg n 16), convertToInt16)
    static member ReadUint16 (?n)   = BitReader <| (N(defaultArg n 16), convertToUint16)
    static member ReadInt32 (?n)    = BitReader <| (N(defaultArg n 32), convertToInt32)
    static member ReadUint32 (?n)   = BitReader <| (N(defaultArg n 32), convertToUint32)    
    static member ReadInt64 (?n)    = BitReader <| (N(defaultArg n 64), convertToInt64)
    static member ReadUint64 (?n)   = BitReader <| (N(defaultArg n 64), convertToUint64)

    static member ReadBool ()       = BitReader <|(N 1, convertToBool)
    static member ReadBytes n       = BitReader <|(N n, convertToBytes)
    static member ReadChar (?n)     = BitReader <|(N(defaultArg n 8), convertToChar)
    static member ReadString n      = BitReader <|(N n, convertToString)

    static member ReadAs n f        = BitReader <| (N n, convertToBytes >> f)
    static member ReadRest ()       = BitReader <| (Rest, convertToBytes)

[<AutoOpen>]
module ReaderWorkflow = 
    let bitReader (stream : Stream) = 
        stream.Position <- 0L
        BitReaderBuilder(stream.ToBitSequence().GetEnumerator())
{'fraction_non_alphanumeric': 0.098107431116059, 'fraction_numerical': 0.025048705816866128, 'mean_word_length': 2.487142163998059, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5619394', 'n_tokens_mistral': 2238, 'n_tokens_neox': 2079, 'n_words': 663}