<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections;
using System.Collections.Concurrent;
using System.IO;
using System.Linq.Expressions;
using System.Reflection;
using System.Reflection.Emit;

namespace MessageShark {
    public static partial class CustomBinary {

        public static byte[] EncodeLength(int length, int tag) {
            return tag <= 15 ? new byte[] { (byte)((tag << 4) | length) } : new byte[] { (byte)length, (byte)tag };
        }
       
        public static Func<object> ExpressionNew(Type type) {
            Func<object> func;
            if (DictFuncNew.TryGetValue(type, out func)) return func;
            var method = Expression.Lambda<Func<object>>(Expression.New(type.GetConstructor(Type.EmptyTypes))).Compile();
            DictFuncNew[type] = method;
            return method;
        }

        static bool IsListType(this Type type) {
            return ListType.IsAssignableFrom(type) || type.Name == "IList`1";
        }

        static bool IsDictionaryType(this Type type) {
            return DictType.IsAssignableFrom(type) || type.Name == "IDictionary`2";
        }

        static bool IsClassType(this Type type) {
            return !type.IsCollectionType();
        }

        static bool IsCollectionType(this Type type) {
            return type.IsListType() || type.IsDictionaryType();
        }

        public static object CreateInstance(Type type) {
            return ExpressionNew(type)();
        }

        static int IsBufferedTypeInt(this Type type) {
            if (type.IsEnum) type = EnumType;
            return BufferedTypes.Contains(type) || !PrimitiveWriterMethods.ContainsKey(type) ? 1 : 0;
        }

        static bool IsComplexType(this Type type) {
            if (type.IsEnum) type = EnumType;
            return
                !PrimitiveWriterMethods.ContainsKey(type) ||
                DictType.IsAssignableFrom(type)
                || ListType.IsAssignableFrom(type);
        }

        static IEnumerable<PropertyInfo> GetTypeProperties(Type type) {
            IEnumerable<PropertyInfo> props;
            if (!TypeProperties.TryGetValue(type, out props))
                TypeProperties[type] = props =
                    type.GetProperties(PropertyBinding)
                    .OrderBy(p => p.Name);
            return props;
        }

        internal static ISerializer<T> GetSerializer<T>() where T : class {
            var type = typeof(T);
            var serializer = default(object);
            if (!SerializerTypes.TryGetValue(type, out serializer)) {
                var serializerType = GenerateSerializer(type);
                serializer = SerializerTypes[type] = Activator.CreateInstance(serializerType);
            }
            return (serializer as ISerializer<T>);
        }
        
        public static bool IsNextTagForPropertyTag(int tag, byte[] buffer, int startIndex) {
            if (startIndex > buffer.Length - 1) return false;
            var readByte = buffer[startIndex];
            var isNextTag = (tag <= 15 && tag == ((readByte & MSB) >> 4)) || (tag > 15 && tag == buffer[startIndex + 1]);
            return isNextTag;
        }
        
        public static void MoveToNextBytes(int tag, ref int startIndex) {
            if (tag <= 15) startIndex++;
            else startIndex += 2;
        }

        public static object CreateInstanceForConcreteTypeTag(byte[] buffer, Type type, ref int startIndex, int tag) {
            if (tag <= 15) startIndex++;
            else startIndex += 2;
            return CreateInstance(SetBaseToConcreteType(buffer, type, ref startIndex));
        }

        public static object CreateInstanceForConcreteType(byte[] buffer, Type type, ref int startIndex) {<commit_msg>Removed unnecessary incrementing of tag for dynamic types<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections;
using System.Collections.Concurrent;
using System.IO;
using System.Linq.Expressions;
using System.Reflection;
using System.Reflection.Emit;

namespace MessageShark {
    public static partial class CustomBinary {

        public static byte[] EncodeLength(int length, int tag) {
            return tag <= 15 ? new byte[] { (byte)((tag << 4) | length) } : new byte[] { (byte)length, (byte)tag };
        }
       
        public static Func<object> ExpressionNew(Type type) {
            Func<object> func;
            if (DictFuncNew.TryGetValue(type, out func)) return func;
            var method = Expression.Lambda<Func<object>>(Expression.New(type.GetConstructor(Type.EmptyTypes))).Compile();
            DictFuncNew[type] = method;
            return method;
        }

        static bool IsListType(this Type type) {
            return ListType.IsAssignableFrom(type) || type.Name == "IList`1";
        }

        static bool IsDictionaryType(this Type type) {
            return DictType.IsAssignableFrom(type) || type.Name == "IDictionary`2";
        }

        static bool IsClassType(this Type type) {
            return !type.IsCollectionType();
        }

        static bool IsCollectionType(this Type type) {
            return type.IsListType() || type.IsDictionaryType();
        }

        public static object CreateInstance(Type type) {
            return ExpressionNew(type)();
        }

        static int IsBufferedTypeInt(this Type type) {
            if (type.IsEnum) type = EnumType;
            return BufferedTypes.Contains(type) || !PrimitiveWriterMethods.ContainsKey(type) ? 1 : 0;
        }

        static bool IsComplexType(this Type type) {
            if (type.IsEnum) type = EnumType;
            return
                !PrimitiveWriterMethods.ContainsKey(type) ||
                DictType.IsAssignableFrom(type)
                || ListType.IsAssignableFrom(type);
        }

        static IEnumerable<PropertyInfo> GetTypeProperties(Type type) {
            IEnumerable<PropertyInfo> props;
            if (!TypeProperties.TryGetValue(type, out props))
                TypeProperties[type] = props =
                    type.GetProperties(PropertyBinding)
                    .OrderBy(p => p.Name);
            return props;
        }

        internal static ISerializer<T> GetSerializer<T>() where T : class {
            var type = typeof(T);
            var serializer = default(object);
            if (!SerializerTypes.TryGetValue(type, out serializer)) {
                var serializerType = GenerateSerializer(type);
                serializer = SerializerTypes[type] = Activator.CreateInstance(serializerType);
            }
            return (serializer as ISerializer<T>);
        }
        
        public static bool IsNextTagForPropertyTag(int tag, byte[] buffer, int startIndex) {
            if (startIndex > buffer.Length - 1) return false;
            var readByte = buffer[startIndex];
            var isNextTag = (tag <= 15 && tag == ((readByte & MSB) >> 4)) || (tag > 15 && tag == buffer[startIndex + 1]);
            return isNextTag;
        }
        
        public static void MoveToNextBytes(int tag, ref int startIndex) {
            if (tag <= 15) startIndex++;
            else startIndex += 2;
        }

        public static object CreateInstanceForConcreteTypeTag(byte[] buffer, Type type, ref int startIndex, int tag) {
            return CreateInstance(SetBaseToConcreteType(buffer, type, ref startIndex));
        }

        public static object CreateInstanceForConcreteType(byte[] buffer, Type type, ref int startIndex) {
{'fraction_non_alphanumeric': 0.09893435074332324, 'fraction_numerical': 0.004867780555190107, 'mean_word_length': 2.2640618291112067, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6072458', 'n_tokens_mistral': 2016, 'n_tokens_neox': 1948, 'n_words': 518}