<commit_before>﻿namespace Myriad

open System
open System.Collections.Concurrent
open System.Runtime.InteropServices

type MyriadCache(dimensions : Dimension seq, collection : Cluster seq) = 
    
    let cache = new ConcurrentDictionary<String, LockFreeList<ClusterSet>>()
    
    // Dimension Id -> weight
    let weights = dimensions |> Seq.mapi (fun i d -> d.Id, int (2.0 ** float i)) |> Map.ofSeq
        
    let compareMeasures (x : Cluster) (y : Cluster) = 
        let xWeight = x.Measures |> Seq.sumBy (fun m -> weights.[m.DimensionId])  
        let yWeight = y.Measures |> Seq.sumBy (fun m -> weights.[m.DimensionId])  
        yWeight.CompareTo(xWeight)

    /// If the cluster's measures are a subset of the context's measures, then it is a match
    let matchMeasures (context : Context) (cluster : Cluster) = Set.isSubset (cluster.Measures) (context.Measures)<commit_msg>Start of data source abstraction<commit_after>﻿namespace Myriad

open System
open System.Collections.Concurrent
open System.Runtime.InteropServices

type IMyriadDataSource =
    abstract GetDimensions : unit -> Dimension seq

type MyriadCache(dimensions : Dimension seq, collection : Cluster seq) = 
    
    let cache = new ConcurrentDictionary<String, LockFreeList<ClusterSet>>()
    
    // Dimension Id -> weight
    let weights = dimensions |> Seq.mapi (fun i d -> d.Id, int (2.0 ** float i)) |> Map.ofSeq
        
    let compareMeasures (x : Cluster) (y : Cluster) = 
        let xWeight = x.Measures |> Seq.sumBy (fun m -> weights.[m.DimensionId])  
        let yWeight = y.Measures |> Seq.sumBy (fun m -> weights.[m.DimensionId])  
        yWeight.CompareTo(xWeight)

    /// If the cluster's measures are a subset of the context's measures, then it is a match
    let matchMeasures (context : Context) (cluster : Cluster) = Set.isSubset (cluster.Measures) (context.Measures)
{'fraction_non_alphanumeric': 0.10897435897435898, 'fraction_numerical': 0.002136752136752137, 'mean_word_length': 3.247165532879819, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6111885', 'n_tokens_mistral': 582, 'n_tokens_neox': 559, 'n_words': 174}