<commit_before>﻿using BitSharp.Common;
using BitSharp.Common.ExtensionMethods;
using BitSharp.Storage.ExtensionMethods;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace BitSharp.Storage
{
    public class BoundedCache<TKey, TValue> : UnboundedCache<TKey, TValue>
    {
        // known keys
        private readonly ReaderWriterLockSlim knownKeysLock;
        private ConcurrentSet<TKey> knownKeys;

        private readonly IBoundedStorage<TKey, TValue> _dataStorage;

        public BoundedCache(string name, IBoundedStorage<TKey, TValue> dataStorage, long maxFlushMemorySize, long maxCacheMemorySize, Func<TValue, long> sizeEstimator)
            : base(name, dataStorage, maxFlushMemorySize, maxCacheMemorySize, sizeEstimator)
        {
            this.knownKeysLock = new ReaderWriterLockSlim();
            this.knownKeys = new ConcurrentSet<TKey>();

            this._dataStorage = dataStorage;

            this.OnAddition += key => AddKnownKey(key);
            this.OnModification += (key, value) => AddKnownKey(key);
            this.OnRetrieved += (key, value) => AddKnownKey(key);
            this.OnMissing += key => RemoveKnownKey(key);

            // load existing keys from storage
            LoadKeyFromStorage();
        }

        public IBoundedStorage<TKey, TValue> DataStorage { get { return this._dataStorage; } }

        // get count of known items
        public int Count
        {
            get { return this.knownKeys.Count; }
        }

        public bool ContainsKey(TKey key)
        {
            return this.knownKeys.Contains(key);
        }

        // get all known item keys
        public IEnumerable<TKey> GetAllKeys()
        {
            return this.knownKeys;
        }

        // get all known item keys, reads everything from storage
        public IEnumerable<TKey> GetAllKeysFromStorage()
        {
            var returnedKeys = new HashSet<TKey>();

            foreach (var key in this._dataStorage.ReadAllKeys())
            {
                returnedKeys.Add(key);
                AddKnownKey(key);
                yield return key;
            }

            // ensure that any keys not returned from storage are returned as well, pending items
            var pendingKeys = this.knownKeys.Except(returnedKeys);
            foreach (var key in pendingKeys)
                yield return key;
        }

        public void FillCache()
        {
            foreach (var value in StreamAllValues())
            {
                var valueSize = this.sizeEstimator(value.Value);

                if (this.MemoryCacheSize + valueSize < this.MaxCacheMemorySize)
                {
                    CacheValue(value.Key, value.Value);
                }
                else
                {
                    break;
                }
            }
        }

        // get all values, reads everything from storage<commit_msg>in bounded cache, use known keys to drop extraneous creates<commit_after>﻿using BitSharp.Common;
using BitSharp.Common.ExtensionMethods;
using BitSharp.Storage.ExtensionMethods;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace BitSharp.Storage
{
    public class BoundedCache<TKey, TValue> : UnboundedCache<TKey, TValue>
    {
        // known keys
        private readonly ReaderWriterLockSlim knownKeysLock;
        private ConcurrentSet<TKey> knownKeys;

        private readonly IBoundedStorage<TKey, TValue> _dataStorage;

        public BoundedCache(string name, IBoundedStorage<TKey, TValue> dataStorage, long maxFlushMemorySize, long maxCacheMemorySize, Func<TValue, long> sizeEstimator)
            : base(name, dataStorage, maxFlushMemorySize, maxCacheMemorySize, sizeEstimator)
        {
            this.knownKeysLock = new ReaderWriterLockSlim();
            this.knownKeys = new ConcurrentSet<TKey>();

            this._dataStorage = dataStorage;

            this.OnAddition += key => AddKnownKey(key);
            this.OnModification += (key, value) => AddKnownKey(key);
            this.OnRetrieved += (key, value) => AddKnownKey(key);
            this.OnMissing += key => RemoveKnownKey(key);

            // load existing keys from storage
            LoadKeyFromStorage();
        }

        public IBoundedStorage<TKey, TValue> DataStorage { get { return this._dataStorage; } }

        // get count of known items
        public int Count
        {
            get { return this.knownKeys.Count; }
        }

        public bool ContainsKey(TKey key)
        {
            return this.knownKeys.Contains(key);
        }

        // get all known item keys
        public IEnumerable<TKey> GetAllKeys()
        {
            return this.knownKeys;
        }

        // get all known item keys, reads everything from storage
        public IEnumerable<TKey> GetAllKeysFromStorage()
        {
            var returnedKeys = new HashSet<TKey>();

            foreach (var key in this._dataStorage.ReadAllKeys())
            {
                returnedKeys.Add(key);
                AddKnownKey(key);
                yield return key;
            }

            // ensure that any keys not returned from storage are returned as well, pending items
            var pendingKeys = this.knownKeys.Except(returnedKeys);
            foreach (var key in pendingKeys)
                yield return key;
        }

        public override void CreateValue(TKey key, TValue value)
        {
            if (!this.ContainsKey(key))
                base.CreateValue(key, value);
        }

        public void FillCache()
        {
            foreach (var value in StreamAllValues())
            {
                var valueSize = this.sizeEstimator(value.Value);

                if (this.MemoryCacheSize + valueSize < this.MaxCacheMemorySize)
                {
                    CacheValue(value.Key, value.Value);
                }
                else
                {
                    break;
                }
            }
        }

        // get all values, reads everything from storage
{'fraction_non_alphanumeric': 0.08019460138104206, 'fraction_numerical': 0.0, 'mean_word_length': 1.9477335800185014, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7226184', 'n_tokens_mistral': 1682, 'n_tokens_neox': 1585, 'n_words': 425}