<commit_before>﻿/*
Copyright 2015-2018 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
 */

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
using NonBlocking;
using TWCore.Reflection;

namespace TWCore.Serialization.NSerializer
{
    public partial class DeserializersTable
    {
        internal static readonly Dictionary<byte, (MethodInfo Method, MethodAccessorDelegate Accessor)> ReadValues = new Dictionary<byte, (MethodInfo Method, MethodAccessorDelegate Accessor)>();
        internal static readonly Dictionary<Type, MethodInfo> ReadValuesFromType = new Dictionary<Type, MethodInfo>();
        internal static readonly ConcurrentDictionary<Type, DeserializerTypeDescriptor> Descriptors = new ConcurrentDictionary<Type, DeserializerTypeDescriptor>();
        internal static readonly MethodInfo StreamReadByteMethod = typeof(DeserializersTable).GetMethod("StreamReadByte", BindingFlags.NonPublic | BindingFlags.Instance);
        internal static readonly MethodInfo StreamReadIntMethod = typeof(DeserializersTable).GetMethod("StreamReadInt", BindingFlags.NonPublic | BindingFlags.Instance);
        private static readonly byte[] EmptyBytes = new byte[0];
        private readonly byte[] _buffer = new byte[16];
        private readonly object[] _parameters = new object[1];
        internal readonly DeserializerCache<object> ObjectCache = new DeserializerCache<object>();
        private readonly DeserializerCache<Type> _typeCache = new DeserializerCache<Type>();
        private readonly DeserializerCache<DateTimeOffset> _dateTimeOffsetCache = new DeserializerCache<DateTimeOffset>();
        private readonly DeserializerCache<DateTime> _dateTimeCache = new DeserializerCache<DateTime>();
        private readonly DeserializerCache<Guid> _guidCache = new DeserializerCache<Guid>();
        private readonly DeserializerCache<decimal> _decimalCache = new DeserializerCache<decimal>();
        private readonly DeserializerCache<double> _doubleCache = new DeserializerCache<double>();
        private readonly DeserializerCache<float> _floatCache = new DeserializerCache<float>();
        private readonly DeserializerCache<long> _longCache = new DeserializerCache<long>();
        private readonly DeserializerCache<ulong> _uLongCache = new DeserializerCache<ulong>();
        private readonly DeserializerStringCache _stringCache8 = new DeserializerStringCache();
        private readonly DeserializerStringCache _stringCache16 = new DeserializerStringCache();
        private readonly DeserializerStringCache _stringCache32 = new DeserializerStringCache();
        private readonly DeserializerStringCache _stringCache = new DeserializerStringCache();
        private readonly DeserializerCache<TimeSpan> _timespanCache = new DeserializerCache<TimeSpan>();
        private readonly Dictionary<Type, DeserializerMetadataOfTypeRuntime> _metadataInTypes = new Dictionary<Type, DeserializerMetadataOfTypeRuntime>();
        protected Stream Stream;
        protected BinaryReader Reader;

        #region Attributes
        public class DeserializerMethodAttribute : Attribute
        {
            public byte[] ByteTypes { get; }
            public Type ReturnType { get; set; }
            public DeserializerMethodAttribute(params byte[] byteTypes)
            {
                ByteTypes = byteTypes;
            }
        }
        #endregion

        #region Statics
        static DeserializersTable()
        {
            var methods = typeof(DeserializersTable).GetMethods();
            foreach (var method in methods)
            {
                var attr = method.GetAttribute<DeserializerMethodAttribute>();
                if (attr == null) continue;
                foreach (var byteType in attr.ByteTypes)
                    ReadValues[byteType] = (method, Factory.Accessors.BuildMethodAccessor(method));
                ReadValuesFromType[attr.ReturnType] = method;
            }
        }
        #endregion

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public object Deserialize(Stream stream)
        {
            Stream = stream;
            Reader = new BinaryReader(stream, Encoding.UTF8, true);
            if (stream.ReadByte() != DataBytesDefinition.Start)
                throw new FormatException("The stream is not in NSerializer format.");

            var value = ReadValue(StreamReadByte());
            while (StreamReadByte() != DataBytesDefinition.End)
            {
            }

            _dateTimeOffsetCache.Clear();
            _dateTimeCache.Clear();
            _guidCache.Clear();
            _decimalCache.Clear();
            _doubleCache.Clear();
            _floatCache.Clear();
            _longCache.Clear();
            _uLongCache.Clear();
            _stringCache8.Clear();
            _stringCache16.Clear();
            _stringCache32.Clear();
            _stringCache.Clear();
            _timespanCache.Clear();
            _metadataInTypes.Clear();
            ObjectCache.Clear();
            _typeCache.Clear();
            Stream = null;
            Reader = null;
            return value;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private object ReadValue(byte type)
        {
            if (type == DataBytesDefinition.ValueNull) return null;
            if (type == DataBytesDefinition.RefObject)
                return ObjectCache.Get(StreamReadInt());
            if (ReadValues.TryGetValue(type, out var mTuple))
            {
                _parameters[0] = type;
                return mTuple.Accessor(this, _parameters);
            }

            DeserializerMetadataOfTypeRuntime metadata = default;

            if (type == DataBytesDefinition.TypeStart)
            {
                var length = StreamReadInt();
                var typeBytes = new byte[length];
                Stream.Read(typeBytes, 0, length);
                var typeData = Encoding.UTF8.GetString(typeBytes, 0, length);
                var fsCol1 = typeData.IndexOf(";", StringComparison.Ordinal);
                var fsCol2 = typeData.IndexOf(";", fsCol1 + 1, StringComparison.Ordinal);
                var fsCol3 = typeData.IndexOf(";", fsCol2 + 1, StringComparison.Ordinal);
                var fsCol4 = typeData.IndexOf(";", fsCol3 + 1, StringComparison.Ordinal);
                var vTypeString = typeData.Substring(0, fsCol1);
                var isArrayString = typeData.Substring(fsCol1 + 1, 1);
                var isListString = typeData.Substring(fsCol2 + 1, 1);
                var isDictionaryString = typeData.Substring(fsCol3 + 1, 1);
                var propertiesString = typeData.Substring(fsCol4 + 1);

                var valueType = Core.GetType(vTypeString);
                var isArray = isArrayString == "1";
                var isList = isListString == "1";
                var isDictionary = isDictionaryString == "1";
                var properties = propertiesString.Split(";", StringSplitOptions.RemoveEmptyEntries);
                var runtimeMeta = new DeserializerMetaDataOfType(valueType, isArray, isList, isDictionary, properties);
                var descriptor = Descriptors.GetOrAdd(valueType, vType => new DeserializerTypeDescriptor(vType));
                metadata = new DeserializerMetadataOfTypeRuntime(runtimeMeta, descriptor);
                _metadataInTypes[valueType] = metadata;
                _typeCache.Set(valueType);
            }
            else if (type == DataBytesDefinition.RefType)
            {
                var valueType = _typeCache.Get(StreamReadInt());
                metadata = _metadataInTypes[valueType];
            }
            if (metadata.Descriptor.IsNSerializable)
            {
                var value = (INSerializable)metadata.Descriptor.Activator();
                value.Fill(this, metadata.MetaDataOfType);
                return value;
            }
            if (metadata.EqualToDefinition)
                return metadata.Descriptor.DeserializeFunc(this);
            return FillObject(metadata.Descriptor, metadata.MetaDataOfType);
        }<commit_msg>changes<commit_after>﻿/*
Copyright 2015-2018 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
 */

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
using NonBlocking;
using TWCore.Reflection;

namespace TWCore.Serialization.NSerializer
{
    public partial class DeserializersTable
    {
        internal static readonly Dictionary<byte, (MethodInfo Method, MethodAccessorDelegate Accessor)> ReadValues = new Dictionary<byte, (MethodInfo Method, MethodAccessorDelegate Accessor)>();
        internal static readonly Dictionary<Type, MethodInfo> ReadValuesFromType = new Dictionary<Type, MethodInfo>();
        internal static readonly ConcurrentDictionary<Type, DeserializerTypeDescriptor> Descriptors = new ConcurrentDictionary<Type, DeserializerTypeDescriptor>();
        internal static readonly MethodInfo StreamReadByteMethod = typeof(DeserializersTable).GetMethod("StreamReadByte", BindingFlags.NonPublic | BindingFlags.Instance);
        internal static readonly MethodInfo StreamReadIntMethod = typeof(DeserializersTable).GetMethod("StreamReadInt", BindingFlags.NonPublic | BindingFlags.Instance);
        private static readonly byte[] EmptyBytes = new byte[0];
        private readonly byte[] _buffer = new byte[16];
        private readonly object[] _parameters = new object[1];
        internal readonly DeserializerCache<object> ObjectCache = new DeserializerCache<object>();
        private readonly DeserializerCache<Type> _typeCache = new DeserializerCache<Type>();
        private readonly DeserializerCache<DateTimeOffset> _dateTimeOffsetCache = new DeserializerCache<DateTimeOffset>();
        private readonly DeserializerCache<DateTime> _dateTimeCache = new DeserializerCache<DateTime>();
        private readonly DeserializerCache<Guid> _guidCache = new DeserializerCache<Guid>();
        private readonly DeserializerCache<decimal> _decimalCache = new DeserializerCache<decimal>();
        private readonly DeserializerCache<double> _doubleCache = new DeserializerCache<double>();
        private readonly DeserializerCache<float> _floatCache = new DeserializerCache<float>();
        private readonly DeserializerCache<long> _longCache = new DeserializerCache<long>();
        private readonly DeserializerCache<ulong> _uLongCache = new DeserializerCache<ulong>();
        private readonly DeserializerStringCache _stringCache8 = new DeserializerStringCache();
        private readonly DeserializerStringCache _stringCache16 = new DeserializerStringCache();
        private readonly DeserializerStringCache _stringCache32 = new DeserializerStringCache();
        private readonly DeserializerStringCache _stringCache = new DeserializerStringCache();
        private readonly DeserializerCache<TimeSpan> _timespanCache = new DeserializerCache<TimeSpan>();
        private readonly Dictionary<Type, DeserializerMetadataOfTypeRuntime> _metadataInTypes = new Dictionary<Type, DeserializerMetadataOfTypeRuntime>();
        protected Stream Stream;
        protected BinaryReader Reader;

        #region Attributes
        public class DeserializerMethodAttribute : Attribute
        {
            public byte[] ByteTypes { get; }
            public Type ReturnType { get; set; }
            public DeserializerMethodAttribute(params byte[] byteTypes)
            {
                ByteTypes = byteTypes;
            }
        }
        #endregion

        #region Statics
        static DeserializersTable()
        {
            var methods = typeof(DeserializersTable).GetMethods();
            foreach (var method in methods)
            {
                var attr = method.GetAttribute<DeserializerMethodAttribute>();
                if (attr == null) continue;
                foreach (var byteType in attr.ByteTypes)
                    ReadValues[byteType] = (method, Factory.Accessors.BuildMethodAccessor(method));
                ReadValuesFromType[attr.ReturnType] = method;
            }
        }
        #endregion

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public object Deserialize(Stream stream)
        {
            Stream = stream;
            Reader = new BinaryReader(stream, Encoding.UTF8, true);
            if (stream.ReadByte() != DataBytesDefinition.Start)
                throw new FormatException("The stream is not in NSerializer format.");

            var value = ReadValue(StreamReadByte());
            while (StreamReadByte() != DataBytesDefinition.End)
            {
            }

            _dateTimeOffsetCache.Clear();
            _dateTimeCache.Clear();
            _guidCache.Clear();
            _decimalCache.Clear();
            _doubleCache.Clear();
            _floatCache.Clear();
            _longCache.Clear();
            _uLongCache.Clear();
            _stringCache8.Clear();
            _stringCache16.Clear();
            _stringCache32.Clear();
            _stringCache.Clear();
            _timespanCache.Clear();
            _metadataInTypes.Clear();
            ObjectCache.Clear();
            _typeCache.Clear();
            Stream = null;
            Reader = null;
            return value;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private object ReadValue(byte type)
        {
            if (type == DataBytesDefinition.ValueNull) return null;
            if (type == DataBytesDefinition.RefObject)
                return ObjectCache.Get(StreamReadInt());
            if (ReadValues.TryGetValue(type, out var mTuple))
            {
                _parameters[0] = type;
                return mTuple.Accessor(this, _parameters);
            }

            DeserializerMetadataOfTypeRuntime metadata = default;

            if (type == DataBytesDefinition.TypeStart)
            {
                var length = StreamReadInt();
                var typeBytes = new byte[length];
                Stream.Read(typeBytes, 0, length);
                var typeData = Encoding.UTF8.GetString(typeBytes, 0, length);
                var fsCol1 = typeData.IndexOf(";", StringComparison.Ordinal);
                var fsCol2 = typeData.IndexOf(";", fsCol1 + 1, StringComparison.Ordinal);
                var fsCol3 = typeData.IndexOf(";", fsCol2 + 1, StringComparison.Ordinal);
                var fsCol4 = typeData.IndexOf(";", fsCol3 + 1, StringComparison.Ordinal);
                var vTypeString = typeData.Substring(0, fsCol1);
                var isArrayString = typeData.Substring(fsCol1 + 1, 1);
                var isListString = typeData.Substring(fsCol2 + 1, 1);
                var isDictionaryString = typeData.Substring(fsCol3 + 1, 1);
                var propertiesString = typeData.Substring(fsCol4 + 1);

                var valueType = Core.GetType(vTypeString);
                var isArray = isArrayString == "1";
                var isList = isListString == "1";
                var isDictionary = isDictionaryString == "1";
                var properties = propertiesString.Split(";", StringSplitOptions.RemoveEmptyEntries);
                var runtimeMeta = new DeserializerMetaDataOfType(valueType, isArray, isList, isDictionary, properties);
                var descriptor = Descriptors.GetOrAdd(valueType, vType => new DeserializerTypeDescriptor(vType));
                metadata = new DeserializerMetadataOfTypeRuntime(runtimeMeta, descriptor);
                _metadataInTypes[valueType] = metadata;
                _typeCache.Set(valueType);
            }
            else if (type == DataBytesDefinition.RefType)
            {
                var valueType = _typeCache.Get(StreamReadInt());
                metadata = _metadataInTypes[valueType];
            }
            if (metadata.Descriptor.IsNSerializable)
            {
                var value = (INSerializable)metadata.Descriptor.Activator();
                value.Fill(this, metadata.MetaDataOfType);
                return value;
            }
            if (metadata.EqualToDefinition)
                return metadata.Descriptor.DeserializeFunc(this);
            return FillObject(metadata.Descriptor, metadata.MetaDataOfType);
        }
        
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private object InnerReadValue(byte type)
        {
            if (type == DataBytesDefinition.ValueNull) return null;
            if (type == DataBytesDefinition.RefObject)
                return ObjectCache.Get(StreamReadInt());
            
            DeserializerMetadataOfTypeRuntime metadata = default;

            if (type == DataBytesDefinition.TypeStart)
            {
                var length = StreamReadInt();
                var typeBytes = new byte[length];
                Stream.Read(typeBytes, 0, length);
                var typeData = Encoding.UTF8.GetString(typeBytes, 0, length);
                var fsCol1 = typeData.IndexOf(";", StringComparison.Ordinal);
                var fsCol2 = typeData.IndexOf(";", fsCol1 + 1, StringComparison.Ordinal);
                var fsCol3 = typeData.IndexOf(";", fsCol2 + 1, StringComparison.Ordinal);
                var fsCol4 = typeData.IndexOf(";", fsCol3 + 1, StringComparison.Ordinal);
                var vTypeString = typeData.Substring(0, fsCol1);
                var isArrayString = typeData.Substring(fsCol1 + 1, 1);
                var isListString = typeData.Substring(fsCol2 + 1, 1);
                var isDictionaryString = typeData.Substring(fsCol3 + 1, 1);
                var propertiesString = typeData.Substring(fsCol4 + 1);

                var valueType = Core.GetType(vTypeString);
                var isArray = isArrayString == "1";
                var isList = isListString == "1";
                var isDictionary = isDictionaryString == "1";
                var properties = propertiesString.Split(";", StringSplitOptions.RemoveEmptyEntries);
                var runtimeMeta = new DeserializerMetaDataOfType(valueType, isArray, isList, isDictionary, properties);
                var descriptor = Descriptors.GetOrAdd(valueType, vType => new DeserializerTypeDescriptor(vType));
                metadata = new DeserializerMetadataOfTypeRuntime(runtimeMeta, descriptor);
                _metadataInTypes[valueType] = metadata;
                _typeCache.Set(valueType);
            }
            else if (type == DataBytesDefinition.RefType)
            {
                var valueType = _typeCache.Get(StreamReadInt());
                metadata = _metadataInTypes[valueType];
            }
            if (metadata.Descriptor.IsNSerializable)
            {
                var value = (INSerializable)metadata.Descriptor.Activator();
                value.Fill(this, metadata.MetaDataOfType);
                return value;
            }
            if (metadata.EqualToDefinition)
                return metadata.Descriptor.DeserializeFunc(this);
            return FillObject(metadata.Descriptor, metadata.MetaDataOfType);
        }
{'fraction_non_alphanumeric': 0.08290437537121362, 'fraction_numerical': 0.007077806374975252, 'mean_word_length': 2.5434935110487547, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5323003', 'n_tokens_mistral': 5009, 'n_tokens_neox': 4797, 'n_words': 1226}