<commit_before>﻿namespace Eventful.Raven

open System
open System.Runtime.Caching
open Raven.Json.Linq
open Raven.Client
open Raven.Abstractions.Data

type GetDocRequest = string * Type 
type GetDocResponse = string * Type * (obj * RavenJObject * Etag) option

module RavenOperations =
    let log = Common.Logging.LogManager.GetLogger(typeof<BatchWrite>)

    let cacheHitCounter = Metrics.Metric.Counter("RavenOperations Cache Hit", Metrics.Unit.Items)

    let serializeDocument<'T> (documentStore : IDocumentStore) (doc : 'T) =
        let serializer = documentStore.Conventions.CreateSerializer()
        RavenJObject.FromObject(doc, serializer)

    let deserializeToType (documentStore : IDocumentStore) (toType : Type) ravenJObject =
        let serializer = documentStore.Conventions.CreateSerializer()
        serializer.Deserialize(new RavenJTokenReader(ravenJObject), toType)

    let deserialize<'T> (documentStore : IDocumentStore) ravenJObject =
        deserializeToType documentStore typeof<'T> ravenJObject

    let getCacheKey databaseName docKey = databaseName + "::" + docKey 

    let getDocument<'TDocument> (documentStore : IDocumentStore) (cache : MemoryCache) database docKey =
        let cacheKey = getCacheKey database docKey 
        let cacheEntry = cache.Get cacheKey
        match cacheEntry with
        | :? ProjectedDocument<obj> as projectedDoc ->
            let (doc, metadata, etag) = projectedDoc
            cacheHitCounter.Increment()
            async { return Some (doc :?> 'TDocument, metadata, etag) }
        | null -> 
            async {
                use session = documentStore.OpenAsyncSession(database)
                let! doc = session.LoadAsync<_>(docKey) |> Async.AwaitTask
                if Object.Equals(doc, null) then
                    return None
                else
                    let etag = session.Advanced.GetEtagFor(doc)
                    let metadata = session.Advanced.GetMetadataFor(doc)
                    return Some (doc, metadata, etag)
            }
        | entry -> failwith <| sprintf "Unexpected entry type %A" entry

    let getDocuments (documentStore : IDocumentStore) (cache : MemoryCache) (database : string) (request : seq<GetDocRequest>) : Async<seq<GetDocResponse>> = async {
        let requestCacheMatches =
            request
            |> Seq.map(fun (docKey, docType) -> 
                let cacheEntry = getCacheKey database docKey |> cache.Get
                match cacheEntry with
                | null -> (docKey, docType, None)
                | :? (obj * RavenJObject * Etag) as value -> 
                    cacheHitCounter.Increment()
                    let (doc, metadata, etag) = value
                    (docKey, docType, Some (doc, metadata, etag))
                | a -> failwith <| sprintf "Unexpected %A" a)

        let toFetch =
            requestCacheMatches
            |> Seq.collect (function
                | docKey, _, None -> Seq.singleton docKey
                | _ -> Seq.empty)
            |> Array.ofSeq

        let fetchTypes = 
             requestCacheMatches
            |> Seq.collect (function
                | docKey, docType, None -> Seq.singleton (docKey, docType)
                | _ -> Seq.empty)
            |> Map.ofSeq

        let commands = documentStore.AsyncDatabaseCommands.ForDatabase(database)
        let! rawDocs = commands.GetAsync(toFetch, Array.empty) |> Async.AwaitTask

        let serializer = documentStore.Conventions.CreateSerializer()

        let rawDocMap =
            rawDocs.Results
            |> Raven.Client.Connection.SerializationHelper.ToJsonDocuments
            |> Seq.collect (function 
                | null -> Seq.empty
                | jsonDoc ->
                    let docKey = jsonDoc.Key
                    let docType = fetchTypes |> Map.find docKey
                    let actualDoc = deserializeToType documentStore docType jsonDoc.DataAsJson
                    Seq.singleton (docKey, (actualDoc, jsonDoc.Metadata, jsonDoc.Etag)))
            |> Map.ofSeq

        return 
            requestCacheMatches
            |> Seq.map (function
                | docKey, docType, None ->
                    (docKey, docType, rawDocMap |> Map.tryFind docKey)
                | x -> x)
    }

    let emptyMetadataForType (documentStore : IDocumentStore) (documentType : Type) = 
        let entityName = documentStore.Conventions.GetTypeTagName(documentType)
        let metadata = new RavenJObject()
        metadata.Add("Raven-Entity-Name", new RavenJValue(entityName))
        metadata.Add("Raven-Clr-Type", new RavenJValue(documentType.FullName))
        metadata

    let emptyMetadata<'T> (documentStore : IDocumentStore) = 
        emptyMetadataForType documentStore typeof<'T><commit_msg>Change cache hit counter to a meter<commit_after>﻿namespace Eventful.Raven

open System
open System.Runtime.Caching
open Raven.Json.Linq
open Raven.Client
open Raven.Abstractions.Data

type GetDocRequest = string * Type 
type GetDocResponse = string * Type * (obj * RavenJObject * Etag) option

module RavenOperations =
    let log = Common.Logging.LogManager.GetLogger(typeof<BatchWrite>)

    let cacheHitCounter = Metrics.Metric.Meter("RavenOperations Cache Hit", Metrics.Unit.Items)

    let serializeDocument<'T> (documentStore : IDocumentStore) (doc : 'T) =
        let serializer = documentStore.Conventions.CreateSerializer()
        RavenJObject.FromObject(doc, serializer)

    let deserializeToType (documentStore : IDocumentStore) (toType : Type) ravenJObject =
        let serializer = documentStore.Conventions.CreateSerializer()
        serializer.Deserialize(new RavenJTokenReader(ravenJObject), toType)

    let deserialize<'T> (documentStore : IDocumentStore) ravenJObject =
        deserializeToType documentStore typeof<'T> ravenJObject

    let getCacheKey databaseName docKey = databaseName + "::" + docKey 

    let getDocument<'TDocument> (documentStore : IDocumentStore) (cache : MemoryCache) database docKey =
        let cacheKey = getCacheKey database docKey 
        let cacheEntry = cache.Get cacheKey
        match cacheEntry with
        | :? ProjectedDocument<obj> as projectedDoc ->
            let (doc, metadata, etag) = projectedDoc
            cacheHitCounter.Mark()
            async { return Some (doc :?> 'TDocument, metadata, etag) }
        | null -> 
            async {
                use session = documentStore.OpenAsyncSession(database)
                let! doc = session.LoadAsync<_>(docKey) |> Async.AwaitTask
                if Object.Equals(doc, null) then
                    return None
                else
                    let etag = session.Advanced.GetEtagFor(doc)
                    let metadata = session.Advanced.GetMetadataFor(doc)
                    return Some (doc, metadata, etag)
            }
        | entry -> failwith <| sprintf "Unexpected entry type %A" entry

    let getDocuments (documentStore : IDocumentStore) (cache : MemoryCache) (database : string) (request : seq<GetDocRequest>) : Async<seq<GetDocResponse>> = async {
        let requestCacheMatches =
            request
            |> Seq.map(fun (docKey, docType) -> 
                let cacheEntry = getCacheKey database docKey |> cache.Get
                match cacheEntry with
                | null -> (docKey, docType, None)
                | :? (obj * RavenJObject * Etag) as value -> 
                    cacheHitCounter.Mark()
                    let (doc, metadata, etag) = value
                    (docKey, docType, Some (doc, metadata, etag))
                | a -> failwith <| sprintf "Unexpected %A" a)

        let toFetch =
            requestCacheMatches
            |> Seq.collect (function
                | docKey, _, None -> Seq.singleton docKey
                | _ -> Seq.empty)
            |> Array.ofSeq

        let fetchTypes = 
             requestCacheMatches
            |> Seq.collect (function
                | docKey, docType, None -> Seq.singleton (docKey, docType)
                | _ -> Seq.empty)
            |> Map.ofSeq

        let commands = documentStore.AsyncDatabaseCommands.ForDatabase(database)
        let! rawDocs = commands.GetAsync(toFetch, Array.empty) |> Async.AwaitTask

        let serializer = documentStore.Conventions.CreateSerializer()

        let rawDocMap =
            rawDocs.Results
            |> Raven.Client.Connection.SerializationHelper.ToJsonDocuments
            |> Seq.collect (function 
                | null -> Seq.empty
                | jsonDoc ->
                    let docKey = jsonDoc.Key
                    let docType = fetchTypes |> Map.find docKey
                    let actualDoc = deserializeToType documentStore docType jsonDoc.DataAsJson
                    Seq.singleton (docKey, (actualDoc, jsonDoc.Metadata, jsonDoc.Etag)))
            |> Map.ofSeq

        return 
            requestCacheMatches
            |> Seq.map (function
                | docKey, docType, None ->
                    (docKey, docType, rawDocMap |> Map.tryFind docKey)
                | x -> x)
    }

    let emptyMetadataForType (documentStore : IDocumentStore) (documentType : Type) = 
        let entityName = documentStore.Conventions.GetTypeTagName(documentType)
        let metadata = new RavenJObject()
        metadata.Add("Raven-Entity-Name", new RavenJValue(entityName))
        metadata.Add("Raven-Clr-Type", new RavenJValue(documentType.FullName))
        metadata

    let emptyMetadata<'T> (documentStore : IDocumentStore) = 
        emptyMetadataForType documentStore typeof<'T>
{'fraction_non_alphanumeric': 0.088125, 'fraction_numerical': 0.0, 'mean_word_length': 2.1427168576104747, 'pattern_counts': {'":': 2, '<': 31, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2875217', 'n_tokens_mistral': 2625, 'n_tokens_neox': 2490, 'n_words': 613}