<commit_before>﻿module DHDB

open System.IO

open Bytes

type Key   = bytes * bytes
type IntKey = cbytes * cbytes
type Value = bytes * bool

type dhdb = {
    filename: string;
}

(* ------------------------------------------------------------------------------- *)
let intkey_to_key (a,b) = abytes(a),abytes(b)
let key_to_intkey (a,b) = cbytes(a),cbytes(b)

(* ------------------------------------------------------------------------------- *)
let create (filename:string) =
    let self = {
        filename = filename;
    }
    DB.closedb (DB.opendb self.filename)
    self

(* ------------------------------------------------------------------------------- *)
let remove self key =
    let key = DB.serialize<IntKey> (key_to_intkey key) in
  
    let db  = DB.opendb self.filename in

    try
        DB.tx db (fun db -> ignore (DB.remove db key));
        self
    finally
        DB.closedb db

(* ------------------------------------------------------------------------------- *)
let select self key =
    let key = DB.serialize<IntKey> (key_to_intkey key) in

    let select (db : DB.db) =
        DB.get db key
            |> Option.map DB.deserialize<Value>
          
    let db = DB.opendb self.filename in

    try
        DB.tx db select
    finally
        DB.closedb db

(* ------------------------------------------------------------------------------- *)
let insert self key v =
    let key = DB.serialize<IntKey> (key_to_intkey key) in
  
    let insert (db : DB.db) =
        match DB.get db key with
        | Some _ -> ()
        | None   -> DB.put db key (DB.serialize<Value> v) in

    let db = DB.opendb self.filename in

    try
        DB.tx db insert; self
    finally
        DB.closedb db

(* ------------------------------------------------------------------------------- *)
let keys self =
    let aout =
        let db = DB.opendb self.filename in
    
        try
            DB.tx db (fun db -> DB.keys db)
        finally
            DB.closedb db
    in
        List.map DB.deserialize<IntKey> aout
        |> List.map intkey_to_key

(* ------------------------------------------------------------------------------- *)
let merge self db1 =
    let db = DB.opendb self.filename in
    
    try
        let db = DB.attach db db1 "db" in
        DB.tx db (fun db -> DB.merge db "db"); self
    finally
        DB.closedb db
<commit_msg>Fixed problem with serialization of abstract types<commit_after>﻿module DHDB

open System.IO

open Bytes

type Key   = bytes * bytes
type IntKey = cbytes * cbytes
type Value = bytes * bool
type IntValue = cbytes * bool

type dhdb = {
    filename: string;
}

(* ------------------------------------------------------------------------------- *)
let intkey_to_key (a,b) = abytes(a),abytes(b)
let key_to_intkey (a,b) = cbytes(a),cbytes(b)

(* ------------------------------------------------------------------------------- *)
let intvalue_to_value (a,b) = abytes(a),b
let value_to_intvalue (a,b) = cbytes(a),b

(* ------------------------------------------------------------------------------- *)
let create (filename:string) =
    let self = {
        filename = filename;
    }
    DB.closedb (DB.opendb self.filename)
    self

(* ------------------------------------------------------------------------------- *)
let remove self key =
    let key = DB.serialize<IntKey> (key_to_intkey key) in
  
    let db  = DB.opendb self.filename in

    try
        DB.tx db (fun db -> ignore (DB.remove db key));
        self
    finally
        DB.closedb db

(* ------------------------------------------------------------------------------- *)
let select self key =
    let key = DB.serialize<IntKey> (key_to_intkey key) in

    let select (db : DB.db) =
        DB.get db key
            |> Option.map (DB.deserialize<IntValue> >> intvalue_to_value)
          
    let db = DB.opendb self.filename in

    try
        DB.tx db select
    finally
        DB.closedb db

(* ------------------------------------------------------------------------------- *)
let insert self key v =
    let key = DB.serialize<IntKey> (key_to_intkey key) in
    let v   = DB.serialize<IntValue> (value_to_intvalue v) in
  
    let insert (db : DB.db) =
        match DB.get db key with
        | Some _ -> ()
        | None   -> DB.put db key v in

    let db = DB.opendb self.filename in

    try
        DB.tx db insert; self
    finally
        DB.closedb db

(* ------------------------------------------------------------------------------- *)
let keys self =
    let aout =
        let db = DB.opendb self.filename in
    
        try
            DB.tx db (fun db -> DB.keys db)
        finally
            DB.closedb db
    in
        List.map (DB.deserialize<IntKey> >> intkey_to_key) aout
     
(* ------------------------------------------------------------------------------- *)
let merge self db1 =
    let db = DB.opendb self.filename in
    
    try
        let db = DB.attach db db1 "db" in
        DB.tx db (fun db -> DB.merge db "db"); self
    finally
        DB.closedb db

{'fraction_non_alphanumeric': 0.31579992060341405, 'fraction_numerical': 0.0007939658594680428, 'mean_word_length': 1.0003969829297341, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3781040', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1366, 'n_words': 413}