<commit_before>using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BTDB.Buffer;
using BTDB.StreamLayer;

namespace BTDB.KVDBLayer
{
    internal class ChunkStorageInKV : IChunkStorage
    {
        struct StorageValue
        {
            internal uint FileId;
            internal uint FileOfs;
            internal uint ContentLengthCompressedIsLeaf;
            internal uint ContentLength
            {
                get { return ContentLengthCompressedIsLeaf / 4; }
                set { ContentLengthCompressedIsLeaf = (ContentLengthCompressedIsLeaf & 1) + value * 4; }
            }

            internal bool Compressed
            {
                get { return (ContentLengthCompressedIsLeaf & 2) != 0; }
                set
                {
                    if (value) ContentLengthCompressedIsLeaf |= 2u; else ContentLengthCompressedIsLeaf &= ~2u;
                }
            }

            internal bool Leaf
            {
                get { return (ContentLengthCompressedIsLeaf & 1) != 0; }
                set
                {
                    if (value) ContentLengthCompressedIsLeaf |= 1u; else ContentLengthCompressedIsLeaf &= ~1u;
                }
            }
        }

        readonly long _subDBId;
        readonly IFileCollectionWithFileInfos _fileCollection;
        readonly long _maxFileSize;
        volatile int _keyLen;
        readonly ConcurrentDictionary<ByteStructs.Key20, StorageValue> _dict20 = new ConcurrentDictionary<ByteStructs.Key20, StorageValue>(new ByteStructs.Key20EqualityComparer());
        readonly object _pureValueFileLock = new object();
        IFileCollectionFile _pureValueFile;
        AbstractBufferedWriter _pureValueFileWriter;
        IFileCollectionFile _hashIndexFile;
        AbstractBufferedWriter _hashIndexWriter;

        public ChunkStorageInKV(long subDBId, IFileCollectionWithFileInfos fileCollection, long maxFileSize)
        {
            _subDBId = subDBId;
            _fileCollection = fileCollection;
            _maxFileSize = maxFileSize;
            _keyLen = -1;
            LoadFiles();
        }

        void LoadFiles()
        {
            var hashKeyIndexFiles = new List<KeyValuePair<uint, long>>();
            var fileIdOfLastPureValues = 0u;
            var generationOfLastPureValues = 0L;
            foreach (var pair in _fileCollection.FileInfos)
            {
                if (pair.Value.SubDBId != _subDBId) continue;
                if (pair.Value.FileType == KVFileType.PureValuesWithId)
                {
                    if (generationOfLastPureValues < pair.Value.Generation)
                    {
                        generationOfLastPureValues = pair.Value.Generation;
                        fileIdOfLastPureValues = pair.Key;
                    }
                }
                else if (pair.Value.FileType == KVFileType.HashKeyIndex)
                {
                    hashKeyIndexFiles.Add(new KeyValuePair<uint, long>(pair.Key, pair.Value.Generation));
                }
            }
            if (hashKeyIndexFiles.Count == 0)
                return;
            hashKeyIndexFiles.Sort((x, y) => x.Value < y.Value ? -1 : x.Value > y.Value ? 1 : 0);
            TryLoadHashKeyIndex(hashKeyIndexFiles[hashKeyIndexFiles.Count - 1].Key);
        }

        bool TryLoadHashKeyIndex(uint hashKeyIndexFileId)
        {
            var reader = _fileCollection.GetFile(hashKeyIndexFileId).GetExclusiveReader();
            _keyLen = (int) ((IHashKeyIndex)_fileCollection.FileInfoByIdx(hashKeyIndexFileId)).KeyLen;
            HashKeyIndex.SkipHeader(reader);
            var keyBuf = ByteBuffer.NewSync(new byte[_keyLen]);
            while (!reader.Eof)
            {
                var value = new StorageValue();
                value.FileId = reader.ReadVUInt32();
                value.FileOfs = reader.ReadVUInt32();
                value.ContentLengthCompressedIsLeaf = reader.ReadVUInt32();
                reader.ReadBlock(keyBuf);
                _dict20.TryAdd(new ByteStructs.Key20(keyBuf), value);
            }
            return true;
        }

        void CheckOrInitKeyLen(int keyLen)
        {
            if (_keyLen == -1)
            {
                if (keyLen != 20) throw new ArgumentException("Length of Key must be 20 bytes");
                Interlocked.CompareExchange(ref _keyLen, keyLen, -1);
            }
            if (_keyLen != keyLen)
            {
                throw new ArgumentException("Key length is different from stored");
            }
        }

        public IChunkStorageTransaction StartTransaction()
        {
            return new ChunkStorageTransaction(this);
        }

        class ChunkStorageTransaction : IChunkStorageTransaction
        {
            readonly ChunkStorageInKV _chunkStorageInKV;

            public ChunkStorageTransaction(ChunkStorageInKV chunkStorageInKV)
            {<commit_msg>Disable invalid warning<commit_after>using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BTDB.Buffer;
using BTDB.StreamLayer;

namespace BTDB.KVDBLayer
{
    internal class ChunkStorageInKV : IChunkStorage
    {
        struct StorageValue
        {
            internal uint FileId;
            internal uint FileOfs;
            internal uint ContentLengthCompressedIsLeaf;
            internal uint ContentLength
            {
                get { return ContentLengthCompressedIsLeaf / 4; }
                set { ContentLengthCompressedIsLeaf = (ContentLengthCompressedIsLeaf & 1) + value * 4; }
            }

            internal bool Compressed
            {
                get { return (ContentLengthCompressedIsLeaf & 2) != 0; }
                set
                {
                    if (value) ContentLengthCompressedIsLeaf |= 2u; else ContentLengthCompressedIsLeaf &= ~2u;
                }
            }

            internal bool Leaf
            {
                get { return (ContentLengthCompressedIsLeaf & 1) != 0; }
                set
                {
                    if (value) ContentLengthCompressedIsLeaf |= 1u; else ContentLengthCompressedIsLeaf &= ~1u;
                }
            }
        }

        readonly long _subDBId;
        readonly IFileCollectionWithFileInfos _fileCollection;
        readonly long _maxFileSize;
        volatile int _keyLen;
        readonly ConcurrentDictionary<ByteStructs.Key20, StorageValue> _dict20 = new ConcurrentDictionary<ByteStructs.Key20, StorageValue>(new ByteStructs.Key20EqualityComparer());
        readonly object _pureValueFileLock = new object();
        IFileCollectionFile _pureValueFile;
        AbstractBufferedWriter _pureValueFileWriter;
        IFileCollectionFile _hashIndexFile;
        AbstractBufferedWriter _hashIndexWriter;

        public ChunkStorageInKV(long subDBId, IFileCollectionWithFileInfos fileCollection, long maxFileSize)
        {
            _subDBId = subDBId;
            _fileCollection = fileCollection;
            _maxFileSize = maxFileSize;
            _keyLen = -1;
            LoadFiles();
        }

        void LoadFiles()
        {
            var hashKeyIndexFiles = new List<KeyValuePair<uint, long>>();
            var fileIdOfLastPureValues = 0u;
            var generationOfLastPureValues = 0L;
            foreach (var pair in _fileCollection.FileInfos)
            {
                if (pair.Value.SubDBId != _subDBId) continue;
                if (pair.Value.FileType == KVFileType.PureValuesWithId)
                {
                    if (generationOfLastPureValues < pair.Value.Generation)
                    {
                        generationOfLastPureValues = pair.Value.Generation;
                        fileIdOfLastPureValues = pair.Key;
                    }
                }
                else if (pair.Value.FileType == KVFileType.HashKeyIndex)
                {
                    hashKeyIndexFiles.Add(new KeyValuePair<uint, long>(pair.Key, pair.Value.Generation));
                }
            }
            if (hashKeyIndexFiles.Count == 0)
                return;
            hashKeyIndexFiles.Sort((x, y) => x.Value < y.Value ? -1 : x.Value > y.Value ? 1 : 0);
            TryLoadHashKeyIndex(hashKeyIndexFiles[hashKeyIndexFiles.Count - 1].Key);
        }

        bool TryLoadHashKeyIndex(uint hashKeyIndexFileId)
        {
            var reader = _fileCollection.GetFile(hashKeyIndexFileId).GetExclusiveReader();
            _keyLen = (int) ((IHashKeyIndex)_fileCollection.FileInfoByIdx(hashKeyIndexFileId)).KeyLen;
            HashKeyIndex.SkipHeader(reader);
            var keyBuf = ByteBuffer.NewSync(new byte[_keyLen]);
            while (!reader.Eof)
            {
                var value = new StorageValue();
                value.FileId = reader.ReadVUInt32();
                value.FileOfs = reader.ReadVUInt32();
                value.ContentLengthCompressedIsLeaf = reader.ReadVUInt32();
                reader.ReadBlock(keyBuf);
                _dict20.TryAdd(new ByteStructs.Key20(keyBuf), value);
            }
            return true;
        }

        void CheckOrInitKeyLen(int keyLen)
        {
            if (_keyLen == -1)
            {
                if (keyLen != 20) throw new ArgumentException("Length of Key must be 20 bytes");
#pragma warning disable 420
                Interlocked.CompareExchange(ref _keyLen, keyLen, -1);
#pragma warning restore 420
            }
            if (_keyLen != keyLen)
            {
                throw new ArgumentException("Key length is different from stored");
            }
        }

        public IChunkStorageTransaction StartTransaction()
        {
            return new ChunkStorageTransaction(this);
        }

        class ChunkStorageTransaction : IChunkStorageTransaction
        {
            readonly ChunkStorageInKV _chunkStorageInKV;

            public ChunkStorageTransaction(ChunkStorageInKV chunkStorageInKV)
            {
{'fraction_non_alphanumeric': 0.07263345901964675, 'fraction_numerical': 0.009128795395911887, 'mean_word_length': 1.7834852250759459, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6640004', 'n_tokens_mistral': 2703, 'n_tokens_neox': 2534, 'n_words': 530}